From 67dee0adc09534483ce2627ffee629feb5133ae7 Mon Sep 17 00:00:00 2001 From: imsheridan Date: Fri, 6 Apr 2018 03:26:26 +0800 Subject: [PATCH 001/796] Fix math equation rendering format in api definitions --- tensorflow/core/api_def/base_api/api_def_Exp.pbtxt | 2 +- .../core/api_def/base_api/api_def_GatherNd.pbtxt | 2 +- .../api_def/base_api/api_def_MatrixExponential.pbtxt | 2 +- .../api_def/base_api/api_def_MatrixLogarithm.pbtxt | 2 +- .../core/api_def/base_api/api_def_Polygamma.pbtxt | 2 +- .../core/api_def/base_api/api_def_ReduceJoin.pbtxt | 2 +- .../core/api_def/base_api/api_def_ScatterNdAdd.pbtxt | 4 ++-- .../base_api/api_def_ScatterNdNonAliasingAdd.pbtxt | 4 ++-- .../core/api_def/base_api/api_def_ScatterNdSub.pbtxt | 4 ++-- .../api_def/base_api/api_def_ScatterNdUpdate.pbtxt | 4 ++-- tensorflow/core/api_def/base_api/api_def_Softmax.pbtxt | 2 +- .../api_def/base_api/api_def_SparseApplyAdagrad.pbtxt | 4 ++-- .../base_api/api_def_SparseApplyCenteredRMSProp.pbtxt | 6 +++--- .../api_def/base_api/api_def_SparseApplyFtrl.pbtxt | 10 +++++----- .../api_def/base_api/api_def_SparseApplyMomentum.pbtxt | 4 ++-- .../base_api/api_def_SparseApplyProximalAdagrad.pbtxt | 8 ++++---- .../api_def_SparseApplyProximalGradientDescent.pbtxt | 4 ++-- .../api_def/base_api/api_def_SparseApplyRMSProp.pbtxt | 6 +++--- .../api_def/base_api/api_def_UnsortedSegmentSum.pbtxt | 2 +- tensorflow/core/api_def/base_api/api_def_Zeta.pbtxt | 2 +- 20 files changed, 38 insertions(+), 38 deletions(-) diff --git a/tensorflow/core/api_def/base_api/api_def_Exp.pbtxt b/tensorflow/core/api_def/base_api/api_def_Exp.pbtxt index dd1e3d5dfc..01ac3d433a 100644 --- a/tensorflow/core/api_def/base_api/api_def_Exp.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_Exp.pbtxt @@ -1,4 +1,4 @@ op { graph_op_name: "Exp" - summary: "Computes exponential of x element-wise. \\\\(y = e^x\\\\)." + summary: "Computes exponential of x element-wise. \\(y = e^x\\)." } diff --git a/tensorflow/core/api_def/base_api/api_def_GatherNd.pbtxt b/tensorflow/core/api_def/base_api/api_def_GatherNd.pbtxt index 6cd76ff340..342a1f6b05 100644 --- a/tensorflow/core/api_def/base_api/api_def_GatherNd.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_GatherNd.pbtxt @@ -25,7 +25,7 @@ END (K-1)-dimensional tensor of indices into `params`, where each element defines a slice of `params`: - output[i_0, ..., i_{K-2}] = params[indices[i0, ..., i_{K-2}]] + output[\\(i_0, ..., i_{K-2}\\)] = params[indices[\\(i_0, ..., i_{K-2}\\)]] Whereas in @{tf.gather} `indices` defines slices into the first dimension of `params`, in `tf.gather_nd`, `indices` defines slices into the diff --git a/tensorflow/core/api_def/base_api/api_def_MatrixExponential.pbtxt b/tensorflow/core/api_def/base_api/api_def_MatrixExponential.pbtxt index 0d680f6531..d7b56aec87 100644 --- a/tensorflow/core/api_def/base_api/api_def_MatrixExponential.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_MatrixExponential.pbtxt @@ -18,7 +18,7 @@ END } summary: "Computes the matrix exponential of one or more square matrices:" description: < l1 else 0.0 -accum = accum_new +$$accum_new = accum + grad * grad$$ +$$linear += grad + (accum_{new}^{-lr_{power}} - accum^{-lr_{power}} / lr * var$$ +$$quadratic = 1.0 / (accum_{new}^{lr_{power}} * lr) + 2 * l2$$ +$$var = (sign(linear) * l1 - linear) / quadratic\ if\ |linear| > l1\ else\ 0.0$$ +$$accum = accum_{new}$$ END } diff --git a/tensorflow/core/api_def/base_api/api_def_SparseApplyMomentum.pbtxt b/tensorflow/core/api_def/base_api/api_def_SparseApplyMomentum.pbtxt index 8d9ac9ea3f..17dbb488de 100644 --- a/tensorflow/core/api_def/base_api/api_def_SparseApplyMomentum.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_SparseApplyMomentum.pbtxt @@ -64,7 +64,7 @@ Set use_nesterov = True if you want to use Nesterov momentum. That is for rows we have grad for, we update var and accum as follows: -accum = accum * momentum + grad -var -= lr * accum +$$accum = accum * momentum + grad$$ +$$var -= lr * accum$$ END } diff --git a/tensorflow/core/api_def/base_api/api_def_SparseApplyProximalAdagrad.pbtxt b/tensorflow/core/api_def/base_api/api_def_SparseApplyProximalAdagrad.pbtxt index 80541b91c7..0b24f2ddd1 100644 --- a/tensorflow/core/api_def/base_api/api_def_SparseApplyProximalAdagrad.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_SparseApplyProximalAdagrad.pbtxt @@ -58,9 +58,9 @@ END summary: "Sparse update entries in \'*var\' and \'*accum\' according to FOBOS algorithm." description: < Date: Tue, 10 Apr 2018 21:10:51 +0800 Subject: [PATCH 002/796] Remove breaking ``` for math equations --- tensorflow/core/api_def/base_api/api_def_ScatterNdAdd.pbtxt | 2 -- .../api_def/base_api/api_def_ScatterNdNonAliasingAdd.pbtxt | 4 +--- tensorflow/core/api_def/base_api/api_def_ScatterNdSub.pbtxt | 4 +--- .../core/api_def/base_api/api_def_ScatterNdUpdate.pbtxt | 4 +--- .../core/api_def/base_api/api_def_UnsortedSegmentSum.pbtxt | 2 +- 5 files changed, 4 insertions(+), 12 deletions(-) diff --git a/tensorflow/core/api_def/base_api/api_def_ScatterNdAdd.pbtxt b/tensorflow/core/api_def/base_api/api_def_ScatterNdAdd.pbtxt index ee0578c2ec..a9a7646314 100644 --- a/tensorflow/core/api_def/base_api/api_def_ScatterNdAdd.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_ScatterNdAdd.pbtxt @@ -50,9 +50,7 @@ dimension of `ref`. `updates` is `Tensor` of rank `Q-1+P-K` with shape: -``` $$[d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]].$$ -``` For example, say we want to add 4 scattered elements to a rank-1 tensor to 8 elements. In Python, that addition would look like this: diff --git a/tensorflow/core/api_def/base_api/api_def_ScatterNdNonAliasingAdd.pbtxt b/tensorflow/core/api_def/base_api/api_def_ScatterNdNonAliasingAdd.pbtxt index 1e4f99006a..35116e5f6a 100644 --- a/tensorflow/core/api_def/base_api/api_def_ScatterNdNonAliasingAdd.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_ScatterNdNonAliasingAdd.pbtxt @@ -37,7 +37,7 @@ respect to both `input` and `updates`. `input` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`. `indices` must be integer tensor, containing indices into `input`. -It must be shape `\\([d_0, ..., d_{Q-2}, K]\\)` where `0 < K <= P`. +It must be shape \\([d_0, ..., d_{Q-2}, K]\\) where `0 < K <= P`. The innermost dimension of `indices` (with length `K`) corresponds to indices into elements (if `K = P`) or `(P-K)`-dimensional slices @@ -45,9 +45,7 @@ indices into elements (if `K = P`) or `(P-K)`-dimensional slices `updates` is `Tensor` of rank `Q-1+P-K` with shape: -``` $$[d_0, ..., d_{Q-2}, input.shape[K], ..., input.shape[P-1]].$$ -``` For example, say we want to add 4 scattered elements to a rank-1 tensor to 8 elements. In Python, that addition would look like this: diff --git a/tensorflow/core/api_def/base_api/api_def_ScatterNdSub.pbtxt b/tensorflow/core/api_def/base_api/api_def_ScatterNdSub.pbtxt index e8fdd71785..99e5c4908b 100644 --- a/tensorflow/core/api_def/base_api/api_def_ScatterNdSub.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_ScatterNdSub.pbtxt @@ -42,7 +42,7 @@ within a given variable according to `indices`. `ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`. `indices` must be integer tensor, containing indices into `ref`. -It must be shape `\\([d_0, ..., d_{Q-2}, K]\\)` where `0 < K <= P`. +It must be shape \\([d_0, ..., d_{Q-2}, K]\\) where `0 < K <= P`. The innermost dimension of `indices` (with length `K`) corresponds to indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th @@ -50,9 +50,7 @@ dimension of `ref`. `updates` is `Tensor` of rank `Q-1+P-K` with shape: -``` $$[d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]].$$ -``` For example, say we want to subtract 4 scattered elements from a rank-1 tensor with 8 elements. In Python, that subtraction would look like this: diff --git a/tensorflow/core/api_def/base_api/api_def_ScatterNdUpdate.pbtxt b/tensorflow/core/api_def/base_api/api_def_ScatterNdUpdate.pbtxt index 556a5d559b..cb57c171b9 100644 --- a/tensorflow/core/api_def/base_api/api_def_ScatterNdUpdate.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_ScatterNdUpdate.pbtxt @@ -42,7 +42,7 @@ variable according to `indices`. `ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`. `indices` must be integer tensor, containing indices into `ref`. -It must be shape `\\([d_0, ..., d_{Q-2}, K]\\)` where `0 < K <= P`. +It must be shape \\([d_0, ..., d_{Q-2}, K]\\) where `0 < K <= P`. The innermost dimension of `indices` (with length `K`) corresponds to indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th @@ -50,9 +50,7 @@ dimension of `ref`. `updates` is `Tensor` of rank `Q-1+P-K` with shape: -``` $$[d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]].$$ -``` For example, say we want to update 4 scattered elements to a rank-1 tensor to 8 elements. In Python, that update would look like this: diff --git a/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentSum.pbtxt b/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentSum.pbtxt index ac1499346c..9aeabd030d 100644 --- a/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentSum.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentSum.pbtxt @@ -20,7 +20,7 @@ Read @{$math_ops#Segmentation$the section on segmentation} for an explanation of segments. Computes a tensor such that -`\\(output[i] = sum_{j...} data[j...]\\)` where the sum is over tuples `j...` such +\\(output[i] = sum_{j...} data[j...]\\) where the sum is over tuples `j...` such that `segment_ids[j...] == i`. Unlike `SegmentSum`, `segment_ids` need not be sorted and need not cover all values in the full range of valid values. -- GitLab From de6200e7f58b616d6169cc35946e85323da66053 Mon Sep 17 00:00:00 2001 From: eqy Date: Sun, 15 Apr 2018 23:52:04 -0700 Subject: [PATCH 003/796] fix command line example package path --- tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md b/tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md index 495014c6fc..f8327daa08 100644 --- a/tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md +++ b/tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md @@ -41,7 +41,7 @@ FlatBuffer to perform floating-point inference. ``` bazel run --config=opt \ - third_party/tensorflow/contrib/lite/toco:toco -- \ + //tensorflow/contrib/lite/toco:toco -- \ --savedmodel_directory=/tmp/saved_model \ --output_file=/tmp/foo.tflite ``` -- GitLab From 8dc3b3c453180211f4be5302f957664004e1ec04 Mon Sep 17 00:00:00 2001 From: apantykhin Date: Mon, 16 Apr 2018 20:40:51 +0400 Subject: [PATCH 004/796] add checking for input values in GANHead constructor --- .../gan/python/estimator/python/head_impl.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/gan/python/estimator/python/head_impl.py b/tensorflow/contrib/gan/python/estimator/python/head_impl.py index a21358c50b..652ffee30a 100644 --- a/tensorflow/contrib/gan/python/estimator/python/head_impl.py +++ b/tensorflow/contrib/gan/python/estimator/python/head_impl.py @@ -25,6 +25,7 @@ from tensorflow.contrib.gan.python import train as tfgan_train from tensorflow.python.estimator import model_fn as model_fn_lib from tensorflow.python.estimator.canned import head from tensorflow.python.framework import ops +from tensorflow.python.training import optimizer __all__ = [ 'GANHead', @@ -90,9 +91,24 @@ class GANHead(head._Head): # pylint: disable=protected-access name: name of the head. If provided, summary and metrics keys will be suffixed by `"/" + name`. """ + + if not callable(generator_loss_fn): + raise TypeError('generator_loss_fn must be callable.') + if not callable(discriminator_loss_fn): + raise TypeError('discriminator_loss_fn must be callable.') + if not isinstance(generator_optimizer, optimizer.Optimizer): + raise TypeError('generator_optimizer must be Optimizer.') + if not isinstance(discriminator_optimizer, optimizer.Optimizer): + raise TypeError('discriminator_optimizer must be Optimizer.') + if not use_loss_summaries in [True, False, None]: + raise ValueError('use_loss_summaries must be True, False or None.') + if get_hooks_fn is not None and not callable(get_hooks_fn): + raise TypeError('get_hooks_fn must be callable.') + if name is not None and not isinstance(name, str): + raise TypeError('name must be string.') + if get_hooks_fn is None: get_hooks_fn = tfgan_train.get_sequential_train_hooks() - # TODO(joelshor): Validate inputs. if use_loss_summaries in [True, False]: generator_loss_fn = functools.partial( -- GitLab From 7e2929f0e429ba6f47365f034317138066dc2adb Mon Sep 17 00:00:00 2001 From: gracehoney <31743510+aaroey@users.noreply.github.com> Date: Fri, 20 Apr 2018 12:44:05 -0700 Subject: [PATCH 005/796] Roll forward the custom optimizers change --- .../core/grappler/optimizers/meta_optimizer.cc | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc index 22799311bc..3f8d42b98f 100644 --- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc @@ -156,6 +156,19 @@ Status MetaOptimizer::InitializeOptimizersByName( VLOG(2) << "Can't register an optimizer by name: " << optimizer_name; } } + for (const auto& optimizer_config : cfg_.custom_optimizers()) { + auto custom_optimizer = CustomGraphOptimizerRegistry::CreateByNameOrNull( + optimizer_config.name()); + if (custom_optimizer) { + VLOG(2) << "Registered custom configurable graph optimizer: " + << optimizer_config.name(); + TF_RETURN_IF_ERROR(custom_optimizer->Init(&optimizer_config)); + optimizers->push_back(std::move(custom_optimizer)); + } else { + VLOG(2) << "Can't register an optimizer by name: " + << optimizer_config.name(); + } + } return Status::OK(); } @@ -164,7 +177,8 @@ Status MetaOptimizer::OptimizeGraph(Cluster* cluster, const GrapplerItem& item, VLOG(2) << "Optimize GrapplerItem: item.id=" << item.id; std::vector> optimizers; - bool register_by_name = !cfg_.optimizers().empty(); + bool register_by_name = + (!cfg_.optimizers().empty() || !cfg_.custom_optimizers().empty()); TF_RETURN_IF_ERROR(register_by_name ? InitializeOptimizersByName(&optimizers) : InitializeOptimizers(&optimizers)); @@ -321,7 +335,7 @@ bool MetaOptimizerEnabled(const RewriterConfig& cfg) { cfg.auto_parallel().enable() || cfg.memory_optimization() != RewriterConfig::NO_MEM_OPT || cfg.debug_stripper() == RewriterConfig::ON || - !cfg.optimizers().empty(); + !cfg.optimizers().empty() || !cfg.custom_optimizers().empty(); } Status RunMetaOptimizer(const GrapplerItem& item, const RewriterConfig& cfg, -- GitLab From 78da41f8f16871cd1328218cbabcfc82dbecf8a3 Mon Sep 17 00:00:00 2001 From: Sami Kama Date: Wed, 9 May 2018 14:12:54 -0700 Subject: [PATCH 006/796] Subgraph to graphdef --- .../contrib/tensorrt/convert/convert_nodes.cc | 60 +++++++++++++++++++ .../contrib/tensorrt/convert/convert_nodes.h | 4 ++ 2 files changed, 64 insertions(+) diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc index 3767596f8c..9b9ce51097 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc +++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc @@ -53,8 +53,11 @@ limitations under the License. namespace tensorflow { namespace tensorrt { namespace convert { +using ::tensorflow::str_util::Split; + using ::tensorflow::strings::StrAppend; using ::tensorflow::strings::StrCat; + namespace { inline tensorflow::Status ConvertDType(tensorflow::DataType tf_dtype, @@ -2723,6 +2726,63 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef( return tensorflow::Status::OK(); } +// This needs to be called before TensorRT nodes inserted in order to correctly +// get sizes from the original graph +tensorflow::Status ConvertSegmentToGraphDef( + tensorflow::tensorrt::convert::SubGraphParams& params, + tensorflow::GraphDef* segment_def, + std::unordered_map *input_placeholder_map + ) { + //std::unordered_map input_placeholder_map; + for (size_t i = 0; i < params.input_inds.size(); ++i) { + auto& inputs = params.input_inds.at(i); + auto input_node = params.graph.FindNodeId(inputs.first); + if (input_node) { + tensorflow::DataType input_type = tensorflow::DT_FLOAT; + tensorflow::PartialTensorShape partial_shape; + + if (params.graph_properties.HasOutputProperties(input_node->name())) { + auto output_params = + params.graph_properties.GetOutputProperties(input_node->name()); + auto out_shape = output_params.at(inputs.second); + input_type = out_shape.dtype(); + std::vector dims; + for (const auto d : out_shape.shape().dim()) { + dims.push_back(d.size()); + } + tensorflow::PartialTensorShape::MakePartialShape( + dims.data(), dims.size(), &partial_shape); + } + tensorflow::NodeDef dummy_placeholder; + string node_name("InputPH_"); + StrAppend(&node_name, i); + input_placeholder_map->insert({input_node->name(),node_name}); + tensorflow::NodeDefBuilder dph_builder(node_name, "Placeholder"); + auto status = dph_builder.Attr("shape", partial_shape) + .Attr("dtype", input_type) + .Finalize(&dummy_placeholder); + auto seg_node = segment_def->add_node(); + seg_node->CopyFrom(dummy_placeholder); + } + } + for (const auto node_id : params.subgraph_node_ids) { + const auto node = params.graph.FindNodeId(node_id); + if (node) { + auto snode = segment_def->add_node(); + snode->CopyFrom(node->def()); + // check node inputs to see if it was connected to input node and update + // it to point to placeholder if necessary + for (int i = 0; i < snode->input_size(); ++i) { + auto node_input = Split(snode->input(i), ":"); + string node_input_name = node_input[0]; + auto it = input_placeholder_map->find(node_input_name); + if (it != input_placeholder_map->end()) { + snode->set_input(i, it->second); + } + } + } + } +} } // namespace convert } // namespace tensorrt } // namespace tensorflow diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h index 3f6592cd25..903867fa7f 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h +++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h @@ -85,6 +85,10 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(SubGraphParams& params); tensorflow::Status InjectCalibrationNode(SubGraphParams& params); tensorflow::Status ConvertCalibrationNodeToEngineNode(tensorflow::Graph& graph, tensorflow::Node* c_node); +tensorflow::Status ConvertSegmentToGraphDef( + tensorflow::tensorrt::convert::SubGraphParams& params, + tensorflow::GraphDef* segment_def, + std::unordered_map input_placeholder_map); } // namespace convert } // namespace tensorrt } // namespace tensorflow -- GitLab From 7991324a664c4c187c6e7e76d1c7588d79530c33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Wed, 30 May 2018 17:40:14 +0800 Subject: [PATCH 007/796] ENH: add gradient for sparse_slice --- tensorflow/core/kernels/BUILD | 7 + .../core/kernels/sparse_slice_grad_op.cc | 120 ++++++++++++++++++ tensorflow/core/ops/sparse_ops.cc | 14 ++ 3 files changed, 141 insertions(+) create mode 100644 tensorflow/core/kernels/sparse_slice_grad_op.cc diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 1bf6eafb58..dc1155077b 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -3886,6 +3886,7 @@ cc_library( ":sparse_reduce_op", ":sparse_reorder_op", ":sparse_reshape_op", + ":sparse_slice_grad_op", ":sparse_slice_op", ":sparse_softmax", ":sparse_sparse_binary_op_shared", @@ -3971,6 +3972,12 @@ tf_kernel_library( ], ) +tf_kernel_library( + name = "sparse_slice_grad_op", + prefix = "sparse_slice_grad_op", + deps = SPARSE_DEPS, +) + tf_kernel_library( name = "sparse_slice_op", prefix = "sparse_slice_op", diff --git a/tensorflow/core/kernels/sparse_slice_grad_op.cc b/tensorflow/core/kernels/sparse_slice_grad_op.cc new file mode 100644 index 0000000000..8d2c597c75 --- /dev/null +++ b/tensorflow/core/kernels/sparse_slice_grad_op.cc @@ -0,0 +1,120 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_util.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/util/sparse/sparse_tensor.h" + +namespace tensorflow { + +template +class SparseSliceGradOp : public OpKernel { + public: + explicit SparseSliceGradOp(OpKernelConstruction *ctx) : OpKernel(ctx) {} + + void Compute(OpKernelContext *ctx) override { + const Tensor *backprop_val_grad, *input_indices, *output_indices, *input_start; + OP_REQUIRES_OK(ctx, ctx->input("backprop_val_grad", &backprop_val_grad)); + OP_REQUIRES_OK(ctx, ctx->input("input_indices", &input_indices)); + OP_REQUIRES_OK(ctx, ctx->input("input_start", &input_start)); + OP_REQUIRES_OK(ctx, ctx->input("output_indices", &output_indices)); + + OP_REQUIRES(ctx, + TensorShapeUtils::IsMatrix(input_indices->shape()) && + TensorShapeUtils::IsMatrix(output_indices->shape()), + errors::InvalidArgument( + "Input indices should be matrices but received shapes: ", + input_indices->shape().DebugString(), " and ", + output_indices->shape().DebugString())); + OP_REQUIRES( + ctx, TensorShapeUtils::IsVector(backprop_val_grad->shape()), + errors::InvalidArgument( + "Input backprop_val_grad should be a vector but received shape: ", + backprop_val_grad->shape().DebugString())); + OP_REQUIRES( + ctx, + input_indices->dim_size(1) == output_indices->dim_size(1), + errors::InvalidArgument("The input and output should have the same " + "ndims: got: ", input_indices->dim_size(1), " and ", + output_indices->dim_size(1))); + OP_REQUIRES( + ctx, output_indices->dim_size(0) <= input_indices->dim_size(0), + errors::InvalidArgument("# rows of output_indices should be not greater " + "than of input_indices, got ", + output_indices->dim_size(0), " and ", + input_indices->dim_size(0))); + OP_REQUIRES( + ctx, backprop_val_grad->NumElements() == output_indices->dim_size(0), + errors::InvalidArgument("# elements of backprop_val_grad and # rows of " + "output_indices should match (#nnz of sum): got ", + backprop_val_grad->NumElements(), " and ", + output_indices->dim_size(0))); + OP_REQUIRES(ctx, TensorShapeUtils::IsVector(input_start->shape()), + errors::InvalidArgument( + "Input start should be a vector but received shape ", + input_start->shape().DebugString())); + + const int num_dims = input_indices->dim_size(1); + const int64 input_nnz = input_indices->dim_size(0); + + Tensor *val_grad; + OP_REQUIRES_OK(ctx, + ctx->allocate_output(0, TensorShape({input_nnz}), &val_grad)); + + T *val_grad_flat = val_grad->flat().data(); + const T *backprop_val_grad_flat = backprop_val_grad->flat().data(); + memset(val_grad_flat, 0, sizeof(T) * input_nnz); + + // Fill gradients for position where indices of input and ouput are same. + const auto input_indices_mat = input_indices->matrix(); + const auto output_indices_mat = output_indices->matrix(); + const auto input_start_flat = input_start->flat(); + int64 j = 0; + for (int64 i = 0; i < input_nnz && j < backprop_val_grad->NumElements(); + ++i) { + bool isSame = true; + for (int d = 0; d < num_dims; ++d) { + const int64 a = input_indices_mat(i, d); + const int64 b = output_indices_mat(j, d); + const int64 offset = input_start_flat(d); + if (a != b + offset) { + isSame = false; + break; + } + } + if (isSame) { + val_grad_flat[i] = backprop_val_grad_flat[j]; + ++j; + } + } + OP_REQUIRES( + ctx, backprop_val_grad->NumElements() == j, + errors::Internal("Elements of backprop_val_grad aren't eaten up :", + "all: ", backprop_val_grad->NumElements(), + " , used: ", output_indices->dim_size(0))); + } +}; + +#define REGISTER_KERNELS(type) \ + REGISTER_KERNEL_BUILDER( \ + Name("SparseSliceGrad").Device(DEVICE_CPU).TypeConstraint("T"), \ + SparseSliceGradOp) + +TF_CALL_NUMBER_TYPES(REGISTER_KERNELS); +#undef REGISTER_KERNELS +} // namespace tensorflow diff --git a/tensorflow/core/ops/sparse_ops.cc b/tensorflow/core/ops/sparse_ops.cc index acc8c782ef..bc0cb2095d 100644 --- a/tensorflow/core/ops/sparse_ops.cc +++ b/tensorflow/core/ops/sparse_ops.cc @@ -302,6 +302,20 @@ REGISTER_OP("SparseSplit") return Status::OK(); }); +REGISTER_OP("SparseSliceGrad") + .Input("backprop_val_grad: T") + .Input("input_indices: int64") + .Input("input_start: int64") + .Input("output_indices: int64") + .Output("val_grad: T") + .Attr("T: numbertype") + .SetShapeFn([](InferenceContext* c) { + ShapeHandle indices; + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 2, &indices)); + c->set_output(0, c->Vector(c->Dim(indices, 0))); + return Status::OK(); + }); + REGISTER_OP("SparseSlice") .Input("indices: int64") .Input("values: T") -- GitLab From c47442d74edf0de11cad2975662a21bd27b9bf68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Thu, 31 May 2018 10:45:28 +0800 Subject: [PATCH 008/796] ENH: add gradient in python side --- tensorflow/python/kernel_tests/BUILD | 1 + .../kernel_tests/sparse_slice_op_test.py | 22 ++++++++++++-- tensorflow/python/ops/sparse_grad.py | 30 +++++++++++++++++++ 3 files changed, 51 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index 83b353600a..82f3357515 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -892,6 +892,7 @@ tf_py_test( "//third_party/py/numpy", "//tensorflow/python:client_testlib", "//tensorflow/python:framework", + "//tensorflow/python:sparse_grad", "//tensorflow/python:sparse_ops", ], ) diff --git a/tensorflow/python/kernel_tests/sparse_slice_op_test.py b/tensorflow/python/kernel_tests/sparse_slice_op_test.py index da116601f8..38eed897cf 100644 --- a/tensorflow/python/kernel_tests/sparse_slice_op_test.py +++ b/tensorflow/python/kernel_tests/sparse_slice_op_test.py @@ -21,13 +21,15 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import sparse_tensor +from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import sparse_ops +import tensorflow.python.ops.sparse_grad # pylint: disable=unused-import from tensorflow.python.platform import test class SparseSliceOpTest(test.TestCase): - def _SparseTensor_4x6(self): + def _SparseTensor_4x6(self, val_dtype=np.int64): # [0 | |2 | |4 |5 ] # [ |11| |13|14| ] # [20| | |23| |25] @@ -37,7 +39,7 @@ class SparseSliceOpTest(test.TestCase): [2, 3], [2, 5], [3, 0], [3, 2], [3, 3], [3, 5]]).astype( np.int64) val = np.array([0, 2, 4, 5, 11, 13, 14, 20, 23, 25, 30, 32, 33, 35]).astype( - np.int64) + val_dtype) shape = np.array([4, 6]).astype(np.int64) return sparse_tensor.SparseTensor(ind, val, shape) @@ -244,6 +246,22 @@ class SparseSliceOpTest(test.TestCase): self.assertAllEqual(sparse_tensor5.values.eval(), [5, 25, 35]) self.assertAllEqual(sparse_tensor5.dense_shape.eval(), [4, 1]) + def testGradients(self): + sp_input = self._SparseTensor_4x6(val_dtype=np.float32) + start_and_size = [([0, 0], [4, 2]), + ([0, 2], [5, 2]), + ([0, 4], [5, 3])] + + with self.test_session(use_gpu=False): + for start, size in start_and_size: + sp_output = sparse_ops.sparse_slice(sp_input, start, size) + nnz_in = len(sp_input.values.eval()) + nnz_out = len(sp_output.values.eval()) + + err = gradient_checker.compute_gradient_error( + [sp_input.values], [(nnz_in,)], sp_output.values, (nnz_out,)) + self.assertLess(err, 1e-3) + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/ops/sparse_grad.py b/tensorflow/python/ops/sparse_grad.py index 97353d6c74..3ed94738e0 100644 --- a/tensorflow/python/ops/sparse_grad.py +++ b/tensorflow/python/ops/sparse_grad.py @@ -116,6 +116,36 @@ def _SparseReduceSumGrad(op, out_grad): None, None) +@ops.RegisterGradient("SparseSlice") +def _SparseSliceGrad(op, *grads): + """The backward operator for the SparseSlice op. + + The SparseAdd op calculates A + B, where A, B, and the sum are all represented + as `SparseTensor` objects. This op takes in the upstream gradient w.r.t. + non-empty values of the sum, and outputs the gradients w.r.t. the non-empty + values of A and B. + + Args: + op: the SparseAdd op + *grads: the incoming gradients, one element per output of `op` + + Returns: + Gradient for each of the 6 input tensors of SparseAdd: + (a_indices, a_values, a_shape, b_indices, b_values, b_shape, thresh) + The gradients for the indices, shapes, and the threshold are None. + """ + backprop_val_grad = grads[1] + input_indices = op.inputs[0] + input_start = op.inputs[3] + output_indices = op.outputs[0] + + val_grad = gen_sparse_ops.sparse_slice_grad( + backprop_val_grad, input_indices, input_start, output_indices) + val_grad.set_shape(op.inputs[1].get_shape()) + # (indices, values, shape, start, size) + return (None, val_grad, None, None, None) + + @ops.RegisterGradient("SparseTensorDenseMatMul") def _SparseTensorDenseMatMulGrad(op, grad): """Gradients for the dense tensor in the SparseTensorDenseMatMul op. -- GitLab From 86191c9f267fbd157c199c410e8d46574d034782 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Thu, 31 May 2018 12:27:39 +0800 Subject: [PATCH 009/796] DOC: add document --- .../base_api/api_def_SparseSliceGrad.pbtxt | 40 +++++++++++++++++++ .../python_api/api_def_SparseSliceGrad.pbtxt | 4 ++ tensorflow/python/ops/sparse_grad.py | 15 ++++--- 3 files changed, 51 insertions(+), 8 deletions(-) create mode 100644 tensorflow/core/api_def/base_api/api_def_SparseSliceGrad.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_SparseSliceGrad.pbtxt diff --git a/tensorflow/core/api_def/base_api/api_def_SparseSliceGrad.pbtxt b/tensorflow/core/api_def/base_api/api_def_SparseSliceGrad.pbtxt new file mode 100644 index 0000000000..51af6adcf1 --- /dev/null +++ b/tensorflow/core/api_def/base_api/api_def_SparseSliceGrad.pbtxt @@ -0,0 +1,40 @@ +op { + graph_op_name: "SparseSliceGrad" + in_arg { + name: "backprop_val_grad" + description: <