From 49a3eb441a7530d82b23128c5e0b00c4267ad0f5 Mon Sep 17 00:00:00 2001 From: naurril Date: Fri, 4 Aug 2017 22:48:03 +0800 Subject: [PATCH 0001/3149] Add int64_t typemap for swig --- tensorflow/python/platform/base.i | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tensorflow/python/platform/base.i b/tensorflow/python/platform/base.i index aa33eb52c9..dbefca2be9 100644 --- a/tensorflow/python/platform/base.i +++ b/tensorflow/python/platform/base.i @@ -106,6 +106,10 @@ limitations under the License. $1 = &temp; } +%typemap(out) int64_t { + $result = PyLong_FromLongLong($1); +} + %typemap(out) string { $result = PyBytes_FromStringAndSize($1.data(), $1.size()); } -- GitLab From 3b6d5d9df888181e5fb5a49885c0d4eb0b58c5eb Mon Sep 17 00:00:00 2001 From: Dan Jarvis Date: Thu, 23 Nov 2017 12:43:24 -0500 Subject: [PATCH 0002/3149] Update .gitignore files for Android Studio Without these, Git shows lots of noisy build/IDE config files. These files are based on these two sources (plus the list of added files that Git shows when I build in Android Studio): https://github.com/github/gitignore/blob/master/Android.gitignore https://github.com/github/gitignore/pull/2103/files --- tensorflow/contrib/lite/java/demo/.gitignore | 28 ++++++++++++++++--- tensorflow/examples/android/.gitignore | 29 ++++++++++++++++++++ 2 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 tensorflow/examples/android/.gitignore diff --git a/tensorflow/contrib/lite/java/demo/.gitignore b/tensorflow/contrib/lite/java/demo/.gitignore index 39fb081a42..d245ab6109 100644 --- a/tensorflow/contrib/lite/java/demo/.gitignore +++ b/tensorflow/contrib/lite/java/demo/.gitignore @@ -1,9 +1,29 @@ +# This file is based on https://github.com/github/gitignore/blob/master/Android.gitignore *.iml +.idea/compiler.xml +.idea/copyright +.idea/dictionaries +.idea/gradle.xml +.idea/libraries +.idea/inspectionProfiles +.idea/misc.xml +.idea/modules.xml +.idea/runConfigurations.xml +.idea/tasks.xml +.idea/workspace.xml .gradle -/local.properties -/.idea/workspace.xml -/.idea/libraries +local.properties .DS_Store -/build +build/ +gradleBuild/ +*.apk +*.ap_ +*.dex +*.class +bin/ +gen/ +out/ +*.log +.navigation/ /captures .externalNativeBuild diff --git a/tensorflow/examples/android/.gitignore b/tensorflow/examples/android/.gitignore new file mode 100644 index 0000000000..d245ab6109 --- /dev/null +++ b/tensorflow/examples/android/.gitignore @@ -0,0 +1,29 @@ +# This file is based on https://github.com/github/gitignore/blob/master/Android.gitignore +*.iml +.idea/compiler.xml +.idea/copyright +.idea/dictionaries +.idea/gradle.xml +.idea/libraries +.idea/inspectionProfiles +.idea/misc.xml +.idea/modules.xml +.idea/runConfigurations.xml +.idea/tasks.xml +.idea/workspace.xml +.gradle +local.properties +.DS_Store +build/ +gradleBuild/ +*.apk +*.ap_ +*.dex +*.class +bin/ +gen/ +out/ +*.log +.navigation/ +/captures +.externalNativeBuild -- GitLab From 2c858368c8c4b7e82c8d134786026a62a72d2676 Mon Sep 17 00:00:00 2001 From: Randy West Date: Mon, 18 Dec 2017 18:22:03 -0500 Subject: [PATCH 0003/3149] Compute test accuracy in batches to avoid OOM on GPUs. Reported here: https://github.com/tensorflow/tensorflow/issues/136 Alternative to this for mnist_deep.py: https://github.com/tensorflow/tensorflow/pull/157 --- tensorflow/examples/tutorials/mnist/mnist_deep.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tensorflow/examples/tutorials/mnist/mnist_deep.py b/tensorflow/examples/tutorials/mnist/mnist_deep.py index 1e0294db27..2699738735 100644 --- a/tensorflow/examples/tutorials/mnist/mnist_deep.py +++ b/tensorflow/examples/tutorials/mnist/mnist_deep.py @@ -34,6 +34,8 @@ from tensorflow.examples.tutorials.mnist import input_data import tensorflow as tf +import numpy + FLAGS = None @@ -164,8 +166,13 @@ def main(_): print('step %d, training accuracy %g' % (i, train_accuracy)) train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5}) - print('test accuracy %g' % accuracy.eval(feed_dict={ - x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0})) + # compute in batches to avoid OOM on GPUs + accuracy_l = [] + for i in range(50): + batch = mnist.test.next_batch(500, shuffle=False) + accuracy_l.append(accuracy.eval(feed_dict={x: batch[0], y_: batch[1], keep_prob: 1.0})) + print('test accuracy %g' % numpy.mean(accuracy_l)) + if __name__ == '__main__': parser = argparse.ArgumentParser() -- GitLab From 3f18817317940253e6ec0e6b412492c5add5927b Mon Sep 17 00:00:00 2001 From: Randy West Date: Mon, 18 Dec 2017 23:18:30 -0500 Subject: [PATCH 0004/3149] Fix basic arithmetic fail + make loop pythonic --- tensorflow/examples/tutorials/mnist/mnist_deep.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/examples/tutorials/mnist/mnist_deep.py b/tensorflow/examples/tutorials/mnist/mnist_deep.py index 2699738735..47d2777813 100644 --- a/tensorflow/examples/tutorials/mnist/mnist_deep.py +++ b/tensorflow/examples/tutorials/mnist/mnist_deep.py @@ -168,7 +168,7 @@ def main(_): # compute in batches to avoid OOM on GPUs accuracy_l = [] - for i in range(50): + for _ in range(20): batch = mnist.test.next_batch(500, shuffle=False) accuracy_l.append(accuracy.eval(feed_dict={x: batch[0], y_: batch[1], keep_prob: 1.0})) print('test accuracy %g' % numpy.mean(accuracy_l)) -- GitLab From 67dee0adc09534483ce2627ffee629feb5133ae7 Mon Sep 17 00:00:00 2001 From: imsheridan Date: Fri, 6 Apr 2018 03:26:26 +0800 Subject: [PATCH 0005/3149] Fix math equation rendering format in api definitions --- tensorflow/core/api_def/base_api/api_def_Exp.pbtxt | 2 +- .../core/api_def/base_api/api_def_GatherNd.pbtxt | 2 +- .../api_def/base_api/api_def_MatrixExponential.pbtxt | 2 +- .../api_def/base_api/api_def_MatrixLogarithm.pbtxt | 2 +- .../core/api_def/base_api/api_def_Polygamma.pbtxt | 2 +- .../core/api_def/base_api/api_def_ReduceJoin.pbtxt | 2 +- .../core/api_def/base_api/api_def_ScatterNdAdd.pbtxt | 4 ++-- .../base_api/api_def_ScatterNdNonAliasingAdd.pbtxt | 4 ++-- .../core/api_def/base_api/api_def_ScatterNdSub.pbtxt | 4 ++-- .../api_def/base_api/api_def_ScatterNdUpdate.pbtxt | 4 ++-- tensorflow/core/api_def/base_api/api_def_Softmax.pbtxt | 2 +- .../api_def/base_api/api_def_SparseApplyAdagrad.pbtxt | 4 ++-- .../base_api/api_def_SparseApplyCenteredRMSProp.pbtxt | 6 +++--- .../api_def/base_api/api_def_SparseApplyFtrl.pbtxt | 10 +++++----- .../api_def/base_api/api_def_SparseApplyMomentum.pbtxt | 4 ++-- .../base_api/api_def_SparseApplyProximalAdagrad.pbtxt | 8 ++++---- .../api_def_SparseApplyProximalGradientDescent.pbtxt | 4 ++-- .../api_def/base_api/api_def_SparseApplyRMSProp.pbtxt | 6 +++--- .../api_def/base_api/api_def_UnsortedSegmentSum.pbtxt | 2 +- tensorflow/core/api_def/base_api/api_def_Zeta.pbtxt | 2 +- 20 files changed, 38 insertions(+), 38 deletions(-) diff --git a/tensorflow/core/api_def/base_api/api_def_Exp.pbtxt b/tensorflow/core/api_def/base_api/api_def_Exp.pbtxt index dd1e3d5dfc..01ac3d433a 100644 --- a/tensorflow/core/api_def/base_api/api_def_Exp.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_Exp.pbtxt @@ -1,4 +1,4 @@ op { graph_op_name: "Exp" - summary: "Computes exponential of x element-wise. \\\\(y = e^x\\\\)." + summary: "Computes exponential of x element-wise. \\(y = e^x\\)." } diff --git a/tensorflow/core/api_def/base_api/api_def_GatherNd.pbtxt b/tensorflow/core/api_def/base_api/api_def_GatherNd.pbtxt index 6cd76ff340..342a1f6b05 100644 --- a/tensorflow/core/api_def/base_api/api_def_GatherNd.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_GatherNd.pbtxt @@ -25,7 +25,7 @@ END (K-1)-dimensional tensor of indices into `params`, where each element defines a slice of `params`: - output[i_0, ..., i_{K-2}] = params[indices[i0, ..., i_{K-2}]] + output[\\(i_0, ..., i_{K-2}\\)] = params[indices[\\(i_0, ..., i_{K-2}\\)]] Whereas in @{tf.gather} `indices` defines slices into the first dimension of `params`, in `tf.gather_nd`, `indices` defines slices into the diff --git a/tensorflow/core/api_def/base_api/api_def_MatrixExponential.pbtxt b/tensorflow/core/api_def/base_api/api_def_MatrixExponential.pbtxt index 0d680f6531..d7b56aec87 100644 --- a/tensorflow/core/api_def/base_api/api_def_MatrixExponential.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_MatrixExponential.pbtxt @@ -18,7 +18,7 @@ END } summary: "Computes the matrix exponential of one or more square matrices:" description: < l1 else 0.0 -accum = accum_new +$$accum_new = accum + grad * grad$$ +$$linear += grad + (accum_{new}^{-lr_{power}} - accum^{-lr_{power}} / lr * var$$ +$$quadratic = 1.0 / (accum_{new}^{lr_{power}} * lr) + 2 * l2$$ +$$var = (sign(linear) * l1 - linear) / quadratic\ if\ |linear| > l1\ else\ 0.0$$ +$$accum = accum_{new}$$ END } diff --git a/tensorflow/core/api_def/base_api/api_def_SparseApplyMomentum.pbtxt b/tensorflow/core/api_def/base_api/api_def_SparseApplyMomentum.pbtxt index 8d9ac9ea3f..17dbb488de 100644 --- a/tensorflow/core/api_def/base_api/api_def_SparseApplyMomentum.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_SparseApplyMomentum.pbtxt @@ -64,7 +64,7 @@ Set use_nesterov = True if you want to use Nesterov momentum. That is for rows we have grad for, we update var and accum as follows: -accum = accum * momentum + grad -var -= lr * accum +$$accum = accum * momentum + grad$$ +$$var -= lr * accum$$ END } diff --git a/tensorflow/core/api_def/base_api/api_def_SparseApplyProximalAdagrad.pbtxt b/tensorflow/core/api_def/base_api/api_def_SparseApplyProximalAdagrad.pbtxt index 80541b91c7..0b24f2ddd1 100644 --- a/tensorflow/core/api_def/base_api/api_def_SparseApplyProximalAdagrad.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_SparseApplyProximalAdagrad.pbtxt @@ -58,9 +58,9 @@ END summary: "Sparse update entries in \'*var\' and \'*accum\' according to FOBOS algorithm." description: < Date: Tue, 10 Apr 2018 21:10:51 +0800 Subject: [PATCH 0006/3149] Remove breaking ``` for math equations --- tensorflow/core/api_def/base_api/api_def_ScatterNdAdd.pbtxt | 2 -- .../api_def/base_api/api_def_ScatterNdNonAliasingAdd.pbtxt | 4 +--- tensorflow/core/api_def/base_api/api_def_ScatterNdSub.pbtxt | 4 +--- .../core/api_def/base_api/api_def_ScatterNdUpdate.pbtxt | 4 +--- .../core/api_def/base_api/api_def_UnsortedSegmentSum.pbtxt | 2 +- 5 files changed, 4 insertions(+), 12 deletions(-) diff --git a/tensorflow/core/api_def/base_api/api_def_ScatterNdAdd.pbtxt b/tensorflow/core/api_def/base_api/api_def_ScatterNdAdd.pbtxt index ee0578c2ec..a9a7646314 100644 --- a/tensorflow/core/api_def/base_api/api_def_ScatterNdAdd.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_ScatterNdAdd.pbtxt @@ -50,9 +50,7 @@ dimension of `ref`. `updates` is `Tensor` of rank `Q-1+P-K` with shape: -``` $$[d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]].$$ -``` For example, say we want to add 4 scattered elements to a rank-1 tensor to 8 elements. In Python, that addition would look like this: diff --git a/tensorflow/core/api_def/base_api/api_def_ScatterNdNonAliasingAdd.pbtxt b/tensorflow/core/api_def/base_api/api_def_ScatterNdNonAliasingAdd.pbtxt index 1e4f99006a..35116e5f6a 100644 --- a/tensorflow/core/api_def/base_api/api_def_ScatterNdNonAliasingAdd.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_ScatterNdNonAliasingAdd.pbtxt @@ -37,7 +37,7 @@ respect to both `input` and `updates`. `input` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`. `indices` must be integer tensor, containing indices into `input`. -It must be shape `\\([d_0, ..., d_{Q-2}, K]\\)` where `0 < K <= P`. +It must be shape \\([d_0, ..., d_{Q-2}, K]\\) where `0 < K <= P`. The innermost dimension of `indices` (with length `K`) corresponds to indices into elements (if `K = P`) or `(P-K)`-dimensional slices @@ -45,9 +45,7 @@ indices into elements (if `K = P`) or `(P-K)`-dimensional slices `updates` is `Tensor` of rank `Q-1+P-K` with shape: -``` $$[d_0, ..., d_{Q-2}, input.shape[K], ..., input.shape[P-1]].$$ -``` For example, say we want to add 4 scattered elements to a rank-1 tensor to 8 elements. In Python, that addition would look like this: diff --git a/tensorflow/core/api_def/base_api/api_def_ScatterNdSub.pbtxt b/tensorflow/core/api_def/base_api/api_def_ScatterNdSub.pbtxt index e8fdd71785..99e5c4908b 100644 --- a/tensorflow/core/api_def/base_api/api_def_ScatterNdSub.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_ScatterNdSub.pbtxt @@ -42,7 +42,7 @@ within a given variable according to `indices`. `ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`. `indices` must be integer tensor, containing indices into `ref`. -It must be shape `\\([d_0, ..., d_{Q-2}, K]\\)` where `0 < K <= P`. +It must be shape \\([d_0, ..., d_{Q-2}, K]\\) where `0 < K <= P`. The innermost dimension of `indices` (with length `K`) corresponds to indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th @@ -50,9 +50,7 @@ dimension of `ref`. `updates` is `Tensor` of rank `Q-1+P-K` with shape: -``` $$[d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]].$$ -``` For example, say we want to subtract 4 scattered elements from a rank-1 tensor with 8 elements. In Python, that subtraction would look like this: diff --git a/tensorflow/core/api_def/base_api/api_def_ScatterNdUpdate.pbtxt b/tensorflow/core/api_def/base_api/api_def_ScatterNdUpdate.pbtxt index 556a5d559b..cb57c171b9 100644 --- a/tensorflow/core/api_def/base_api/api_def_ScatterNdUpdate.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_ScatterNdUpdate.pbtxt @@ -42,7 +42,7 @@ variable according to `indices`. `ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`. `indices` must be integer tensor, containing indices into `ref`. -It must be shape `\\([d_0, ..., d_{Q-2}, K]\\)` where `0 < K <= P`. +It must be shape \\([d_0, ..., d_{Q-2}, K]\\) where `0 < K <= P`. The innermost dimension of `indices` (with length `K`) corresponds to indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th @@ -50,9 +50,7 @@ dimension of `ref`. `updates` is `Tensor` of rank `Q-1+P-K` with shape: -``` $$[d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]].$$ -``` For example, say we want to update 4 scattered elements to a rank-1 tensor to 8 elements. In Python, that update would look like this: diff --git a/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentSum.pbtxt b/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentSum.pbtxt index ac1499346c..9aeabd030d 100644 --- a/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentSum.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentSum.pbtxt @@ -20,7 +20,7 @@ Read @{$math_ops#Segmentation$the section on segmentation} for an explanation of segments. Computes a tensor such that -`\\(output[i] = sum_{j...} data[j...]\\)` where the sum is over tuples `j...` such +\\(output[i] = sum_{j...} data[j...]\\) where the sum is over tuples `j...` such that `segment_ids[j...] == i`. Unlike `SegmentSum`, `segment_ids` need not be sorted and need not cover all values in the full range of valid values. -- GitLab From de6200e7f58b616d6169cc35946e85323da66053 Mon Sep 17 00:00:00 2001 From: eqy Date: Sun, 15 Apr 2018 23:52:04 -0700 Subject: [PATCH 0007/3149] fix command line example package path --- tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md b/tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md index 495014c6fc..f8327daa08 100644 --- a/tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md +++ b/tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md @@ -41,7 +41,7 @@ FlatBuffer to perform floating-point inference. ``` bazel run --config=opt \ - third_party/tensorflow/contrib/lite/toco:toco -- \ + //tensorflow/contrib/lite/toco:toco -- \ --savedmodel_directory=/tmp/saved_model \ --output_file=/tmp/foo.tflite ``` -- GitLab From 8dc3b3c453180211f4be5302f957664004e1ec04 Mon Sep 17 00:00:00 2001 From: apantykhin Date: Mon, 16 Apr 2018 20:40:51 +0400 Subject: [PATCH 0008/3149] add checking for input values in GANHead constructor --- .../gan/python/estimator/python/head_impl.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/gan/python/estimator/python/head_impl.py b/tensorflow/contrib/gan/python/estimator/python/head_impl.py index a21358c50b..652ffee30a 100644 --- a/tensorflow/contrib/gan/python/estimator/python/head_impl.py +++ b/tensorflow/contrib/gan/python/estimator/python/head_impl.py @@ -25,6 +25,7 @@ from tensorflow.contrib.gan.python import train as tfgan_train from tensorflow.python.estimator import model_fn as model_fn_lib from tensorflow.python.estimator.canned import head from tensorflow.python.framework import ops +from tensorflow.python.training import optimizer __all__ = [ 'GANHead', @@ -90,9 +91,24 @@ class GANHead(head._Head): # pylint: disable=protected-access name: name of the head. If provided, summary and metrics keys will be suffixed by `"/" + name`. """ + + if not callable(generator_loss_fn): + raise TypeError('generator_loss_fn must be callable.') + if not callable(discriminator_loss_fn): + raise TypeError('discriminator_loss_fn must be callable.') + if not isinstance(generator_optimizer, optimizer.Optimizer): + raise TypeError('generator_optimizer must be Optimizer.') + if not isinstance(discriminator_optimizer, optimizer.Optimizer): + raise TypeError('discriminator_optimizer must be Optimizer.') + if not use_loss_summaries in [True, False, None]: + raise ValueError('use_loss_summaries must be True, False or None.') + if get_hooks_fn is not None and not callable(get_hooks_fn): + raise TypeError('get_hooks_fn must be callable.') + if name is not None and not isinstance(name, str): + raise TypeError('name must be string.') + if get_hooks_fn is None: get_hooks_fn = tfgan_train.get_sequential_train_hooks() - # TODO(joelshor): Validate inputs. if use_loss_summaries in [True, False]: generator_loss_fn = functools.partial( -- GitLab From 6f1a30c2589e2b9adf3d34a4d87fb280b2e6b149 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 3 Apr 2018 23:44:22 +0000 Subject: [PATCH 0009/3149] Add RecordInput documentation to tf.contrib.framework This fix tries to address the issue raised in 10622 where the RecordInput is not available in the documentation. As was suggested in 10622, this fix exposes RecordInput to tf.contrib (tf.contrib.framework) so that docs could shown up. This fix fixes 10622. Signed-off-by: Yong Tang --- tensorflow/contrib/framework/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/contrib/framework/__init__.py b/tensorflow/contrib/framework/__init__.py index 11397e86bd..5435c98978 100644 --- a/tensorflow/contrib/framework/__init__.py +++ b/tensorflow/contrib/framework/__init__.py @@ -119,6 +119,7 @@ from tensorflow.python.framework.smart_cond import smart_constant_value from tensorflow.python.framework.tensor_spec import BoundedTensorSpec from tensorflow.python.framework.tensor_spec import TensorSpec from tensorflow.python.ops.array_ops import broadcast_to +from tensorflow.python.ops.data_flow_ops import RecordInput from tensorflow.python.ops.init_ops import convolutional_delta_orthogonal from tensorflow.python.ops.init_ops import convolutional_orthogonal_1d from tensorflow.python.ops.init_ops import convolutional_orthogonal_2d -- GitLab From 5dc8f313b1808ae893edb3b8b03ba06ce4968437 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 3 Apr 2018 23:47:02 +0000 Subject: [PATCH 0010/3149] Add `@@RecordInput` so that docs could be exposed After this change and run with: ``` bazel run tools/docs:generate -- --src_dir="$(pwd)/docs_src/" --output_dir=/tmp/tfdocs/ ``` The documentation correctly shown up in: `/tmp/tfdocs/api_docs/python/tf/contrib/framework/RecordInput.md` Signed-off-by: Yong Tang --- tensorflow/contrib/framework/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/contrib/framework/__init__.py b/tensorflow/contrib/framework/__init__.py index 5435c98978..ffcfd6961e 100644 --- a/tensorflow/contrib/framework/__init__.py +++ b/tensorflow/contrib/framework/__init__.py @@ -100,6 +100,8 @@ See the @{$python/contrib.framework} guide. @@BoundedTensorSpec @@TensorSpec + +@@RecordInput """ from __future__ import absolute_import -- GitLab From 7e2929f0e429ba6f47365f034317138066dc2adb Mon Sep 17 00:00:00 2001 From: gracehoney <31743510+aaroey@users.noreply.github.com> Date: Fri, 20 Apr 2018 12:44:05 -0700 Subject: [PATCH 0011/3149] Roll forward the custom optimizers change --- .../core/grappler/optimizers/meta_optimizer.cc | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc index 22799311bc..3f8d42b98f 100644 --- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc @@ -156,6 +156,19 @@ Status MetaOptimizer::InitializeOptimizersByName( VLOG(2) << "Can't register an optimizer by name: " << optimizer_name; } } + for (const auto& optimizer_config : cfg_.custom_optimizers()) { + auto custom_optimizer = CustomGraphOptimizerRegistry::CreateByNameOrNull( + optimizer_config.name()); + if (custom_optimizer) { + VLOG(2) << "Registered custom configurable graph optimizer: " + << optimizer_config.name(); + TF_RETURN_IF_ERROR(custom_optimizer->Init(&optimizer_config)); + optimizers->push_back(std::move(custom_optimizer)); + } else { + VLOG(2) << "Can't register an optimizer by name: " + << optimizer_config.name(); + } + } return Status::OK(); } @@ -164,7 +177,8 @@ Status MetaOptimizer::OptimizeGraph(Cluster* cluster, const GrapplerItem& item, VLOG(2) << "Optimize GrapplerItem: item.id=" << item.id; std::vector> optimizers; - bool register_by_name = !cfg_.optimizers().empty(); + bool register_by_name = + (!cfg_.optimizers().empty() || !cfg_.custom_optimizers().empty()); TF_RETURN_IF_ERROR(register_by_name ? InitializeOptimizersByName(&optimizers) : InitializeOptimizers(&optimizers)); @@ -321,7 +335,7 @@ bool MetaOptimizerEnabled(const RewriterConfig& cfg) { cfg.auto_parallel().enable() || cfg.memory_optimization() != RewriterConfig::NO_MEM_OPT || cfg.debug_stripper() == RewriterConfig::ON || - !cfg.optimizers().empty(); + !cfg.optimizers().empty() || !cfg.custom_optimizers().empty(); } Status RunMetaOptimizer(const GrapplerItem& item, const RewriterConfig& cfg, -- GitLab From 6583c9a693b122a49f17e7ec99463c6c3b7dbe98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Tue, 24 Apr 2018 20:22:24 +0800 Subject: [PATCH 0012/3149] CLN: move _safe_embedding_lookup_sparse to embedding_ops and make it public --- tensorflow/python/BUILD | 2 + .../python/feature_column/feature_column.py | 161 +----------------- tensorflow/python/ops/embedding_ops.py | 157 +++++++++++++++++ tensorflow/python/ops/nn.py | 1 + 4 files changed, 163 insertions(+), 158 deletions(-) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index bb32f4bbe0..6e2e546984 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -1845,6 +1845,8 @@ py_library( ":math_ops", ":platform", ":resource_variable_ops", + ":sparse_ops", + ":tensor_shape", ":variables", ], ) diff --git a/tensorflow/python/feature_column/feature_column.py b/tensorflow/python/feature_column/feature_column.py index c16c3cda48..f48634d0c7 100644 --- a/tensorflow/python/feature_column/feature_column.py +++ b/tensorflow/python/feature_column/feature_column.py @@ -2058,7 +2058,7 @@ def _create_categorical_column_weighted_sum(column, initializer=init_ops.zeros_initializer(), trainable=trainable, collections=weight_collections) - return _safe_embedding_lookup_sparse( + return embedding_ops.safe_embedding_lookup_sparse( weight, id_tensor, sparse_weights=weight_tensor, @@ -2479,7 +2479,7 @@ class _EmbeddingColumn( }) # Return embedding lookup result. - return _safe_embedding_lookup_sparse( + return embedding_ops.safe_embedding_lookup_sparse( embedding_weights=embedding_weights, sparse_ids=sparse_ids, sparse_weights=sparse_weights, @@ -2612,7 +2612,7 @@ class _SharedEmbeddingColumn( }) # Return embedding lookup result. - return _safe_embedding_lookup_sparse( + return embedding_ops.safe_embedding_lookup_sparse( embedding_weights=embedding_weights, sparse_ids=sparse_ids, sparse_weights=sparse_weights, @@ -3065,161 +3065,6 @@ def _collect_leaf_level_keys(cross): return leaf_level_keys -# TODO(zakaria): Move this to embedding_ops and make it public. -def _safe_embedding_lookup_sparse(embedding_weights, - sparse_ids, - sparse_weights=None, - combiner='mean', - default_id=None, - name=None, - partition_strategy='div', - max_norm=None): - """Lookup embedding results, accounting for invalid IDs and empty features. - - The partitioned embedding in `embedding_weights` must all be the same shape - except for the first dimension. The first dimension is allowed to vary as the - vocabulary size is not necessarily a multiple of `P`. `embedding_weights` - may be a `PartitionedVariable` as returned by using `tf.get_variable()` with a - partitioner. - - Invalid IDs (< 0) are pruned from input IDs and weights, as well as any IDs - with non-positive weight. For an entry with no features, the embedding vector - for `default_id` is returned, or the 0-vector if `default_id` is not supplied. - - The ids and weights may be multi-dimensional. Embeddings are always aggregated - along the last dimension. - - Args: - embedding_weights: A list of `P` float `Tensor`s or values representing - partitioned embedding `Tensor`s. Alternatively, a `PartitionedVariable` - created by partitioning along dimension 0. The total unpartitioned - shape should be `[e_0, e_1, ..., e_m]`, where `e_0` represents the - vocab size and `e_1, ..., e_m` are the embedding dimensions. - sparse_ids: `SparseTensor` of shape `[d_0, d_1, ..., d_n]` containing the - ids. `d_0` is typically batch size. - sparse_weights: `SparseTensor` of same shape as `sparse_ids`, containing - float weights corresponding to `sparse_ids`, or `None` if all weights - are be assumed to be 1.0. - combiner: A string specifying how to combine embedding results for each - entry. Currently "mean", "sqrtn" and "sum" are supported, with "mean" - the default. - default_id: The id to use for an entry with no features. - name: A name for this operation (optional). - partition_strategy: A string specifying the partitioning strategy. - Currently `"div"` and `"mod"` are supported. Default is `"div"`. - max_norm: If not `None`, all embeddings are l2-normalized to max_norm before - combining. - - - Returns: - Dense `Tensor` of shape `[d_0, d_1, ..., d_{n-1}, e_1, ..., e_m]`. - - Raises: - ValueError: if `embedding_weights` is empty. - """ - if embedding_weights is None: - raise ValueError('Missing embedding_weights %s.' % embedding_weights) - if isinstance(embedding_weights, variables.PartitionedVariable): - embedding_weights = list(embedding_weights) # get underlying Variables. - if not isinstance(embedding_weights, list): - embedding_weights = [embedding_weights] - if len(embedding_weights) < 1: - raise ValueError('Missing embedding_weights %s.' % embedding_weights) - - dtype = sparse_weights.dtype if sparse_weights is not None else None - embedding_weights = [ - ops.convert_to_tensor(w, dtype=dtype) for w in embedding_weights - ] - - with ops.name_scope(name, 'embedding_lookup', - embedding_weights + [sparse_ids, - sparse_weights]) as scope: - # Reshape higher-rank sparse ids and weights to linear segment ids. - original_shape = sparse_ids.dense_shape - original_rank_dim = sparse_ids.dense_shape.get_shape()[0] - original_rank = ( - array_ops.size(original_shape) - if original_rank_dim.value is None - else original_rank_dim.value) - sparse_ids = sparse_ops.sparse_reshape(sparse_ids, [ - math_ops.reduce_prod( - array_ops.slice(original_shape, [0], [original_rank - 1])), - array_ops.gather(original_shape, original_rank - 1)]) - if sparse_weights is not None: - sparse_weights = sparse_tensor_lib.SparseTensor( - sparse_ids.indices, - sparse_weights.values, sparse_ids.dense_shape) - - # Prune invalid ids and weights. - sparse_ids, sparse_weights = _prune_invalid_ids(sparse_ids, sparse_weights) - if combiner != 'sum': - sparse_ids, sparse_weights = _prune_invalid_weights( - sparse_ids, sparse_weights) - - # Fill in dummy values for empty features, if necessary. - sparse_ids, is_row_empty = sparse_ops.sparse_fill_empty_rows(sparse_ids, - default_id or - 0) - if sparse_weights is not None: - sparse_weights, _ = sparse_ops.sparse_fill_empty_rows(sparse_weights, 1.0) - - result = embedding_ops.embedding_lookup_sparse( - embedding_weights, - sparse_ids, - sparse_weights, - combiner=combiner, - partition_strategy=partition_strategy, - name=None if default_id is None else scope, - max_norm=max_norm) - - if default_id is None: - # Broadcast is_row_empty to the same shape as embedding_lookup_result, - # for use in Select. - is_row_empty = array_ops.tile( - array_ops.reshape(is_row_empty, [-1, 1]), - array_ops.stack([1, array_ops.shape(result)[1]])) - - result = array_ops.where(is_row_empty, - array_ops.zeros_like(result), - result, - name=scope) - - # Reshape back from linear ids back into higher-dimensional dense result. - final_result = array_ops.reshape( - result, - array_ops.concat([ - array_ops.slice( - math_ops.cast(original_shape, dtypes.int32), [0], - [original_rank - 1]), - array_ops.slice(array_ops.shape(result), [1], [-1]) - ], 0)) - final_result.set_shape(tensor_shape.unknown_shape( - (original_rank_dim - 1).value).concatenate(result.get_shape()[1:])) - return final_result - - -def _prune_invalid_ids(sparse_ids, sparse_weights): - """Prune invalid IDs (< 0) from the input ids and weights.""" - is_id_valid = math_ops.greater_equal(sparse_ids.values, 0) - if sparse_weights is not None: - is_id_valid = math_ops.logical_and( - is_id_valid, - array_ops.ones_like(sparse_weights.values, dtype=dtypes.bool)) - sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_id_valid) - if sparse_weights is not None: - sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_id_valid) - return sparse_ids, sparse_weights - - -def _prune_invalid_weights(sparse_ids, sparse_weights): - """Prune invalid weights (< 0) from the input ids and weights.""" - if sparse_weights is not None: - is_weights_valid = math_ops.greater(sparse_weights.values, 0) - sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_weights_valid) - sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_weights_valid) - return sparse_ids, sparse_weights - - class _IndicatorColumn(_DenseColumn, _SequenceDenseColumn, collections.namedtuple('_IndicatorColumn', ['categorical_column'])): diff --git a/tensorflow/python/ops/embedding_ops.py b/tensorflow/python/ops/embedding_ops.py index 9e46739bc1..a8cfeca119 100644 --- a/tensorflow/python/ops/embedding_ops.py +++ b/tensorflow/python/ops/embedding_ops.py @@ -23,6 +23,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import array_ops from tensorflow.python.ops import clip_ops # Imports gradient definitions. @@ -30,6 +31,7 @@ from tensorflow.python.ops import data_flow_grad # pylint: disable=unused-impor from tensorflow.python.ops import data_flow_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import sparse_ops from tensorflow.python.ops import variables from tensorflow.python.platform import tf_logging as logging from tensorflow.python.util.tf_export import tf_export @@ -480,3 +482,158 @@ def embedding_lookup_sparse(params, assert False, "Unrecognized combiner" return embeddings + + +@tf_export("nn.safe_embedding_lookup_sparse") +def safe_embedding_lookup_sparse(embedding_weights, + sparse_ids, + sparse_weights=None, + combiner='mean', + default_id=None, + name=None, + partition_strategy='div', + max_norm=None): + """Lookup embedding results, accounting for invalid IDs and empty features. + + The partitioned embedding in `embedding_weights` must all be the same shape + except for the first dimension. The first dimension is allowed to vary as the + vocabulary size is not necessarily a multiple of `P`. `embedding_weights` + may be a `PartitionedVariable` as returned by using `tf.get_variable()` with a + partitioner. + + Invalid IDs (< 0) are pruned from input IDs and weights, as well as any IDs + with non-positive weight. For an entry with no features, the embedding vector + for `default_id` is returned, or the 0-vector if `default_id` is not supplied. + + The ids and weights may be multi-dimensional. Embeddings are always aggregated + along the last dimension. + + Args: + embedding_weights: A list of `P` float `Tensor`s or values representing + partitioned embedding `Tensor`s. Alternatively, a `PartitionedVariable` + created by partitioning along dimension 0. The total unpartitioned + shape should be `[e_0, e_1, ..., e_m]`, where `e_0` represents the + vocab size and `e_1, ..., e_m` are the embedding dimensions. + sparse_ids: `SparseTensor` of shape `[d_0, d_1, ..., d_n]` containing the + ids. `d_0` is typically batch size. + sparse_weights: `SparseTensor` of same shape as `sparse_ids`, containing + float weights corresponding to `sparse_ids`, or `None` if all weights + are be assumed to be 1.0. + combiner: A string specifying how to combine embedding results for each + entry. Currently "mean", "sqrtn" and "sum" are supported, with "mean" + the default. + default_id: The id to use for an entry with no features. + name: A name for this operation (optional). + partition_strategy: A string specifying the partitioning strategy. + Currently `"div"` and `"mod"` are supported. Default is `"div"`. + max_norm: If not `None`, all embeddings are l2-normalized to max_norm before + combining. + + + Returns: + Dense `Tensor` of shape `[d_0, d_1, ..., d_{n-1}, e_1, ..., e_m]`. + + Raises: + ValueError: if `embedding_weights` is empty. + """ + if embedding_weights is None: + raise ValueError('Missing embedding_weights %s.' % embedding_weights) + if isinstance(embedding_weights, variables.PartitionedVariable): + embedding_weights = list(embedding_weights) # get underlying Variables. + if not isinstance(embedding_weights, list): + embedding_weights = [embedding_weights] + if len(embedding_weights) < 1: + raise ValueError('Missing embedding_weights %s.' % embedding_weights) + + dtype = sparse_weights.dtype if sparse_weights is not None else None + embedding_weights = [ + ops.convert_to_tensor(w, dtype=dtype) for w in embedding_weights + ] + + with ops.name_scope(name, 'embedding_lookup', + embedding_weights + [sparse_ids, + sparse_weights]) as scope: + # Reshape higher-rank sparse ids and weights to linear segment ids. + original_shape = sparse_ids.dense_shape + original_rank_dim = sparse_ids.dense_shape.get_shape()[0] + original_rank = ( + array_ops.size(original_shape) + if original_rank_dim.value is None + else original_rank_dim.value) + sparse_ids = sparse_ops.sparse_reshape(sparse_ids, [ + math_ops.reduce_prod( + array_ops.slice(original_shape, [0], [original_rank - 1])), + array_ops.gather(original_shape, original_rank - 1)]) + if sparse_weights is not None: + sparse_weights = sparse_tensor.SparseTensor( + sparse_ids.indices, + sparse_weights.values, sparse_ids.dense_shape) + + # Prune invalid ids and weights. + sparse_ids, sparse_weights = _prune_invalid_ids(sparse_ids, sparse_weights) + if combiner != 'sum': + sparse_ids, sparse_weights = _prune_invalid_weights( + sparse_ids, sparse_weights) + + # Fill in dummy values for empty features, if necessary. + sparse_ids, is_row_empty = sparse_ops.sparse_fill_empty_rows(sparse_ids, + default_id or + 0) + if sparse_weights is not None: + sparse_weights, _ = sparse_ops.sparse_fill_empty_rows(sparse_weights, 1.0) + + result = embedding_lookup_sparse( + embedding_weights, + sparse_ids, + sparse_weights, + combiner=combiner, + partition_strategy=partition_strategy, + name=None if default_id is None else scope, + max_norm=max_norm) + + if default_id is None: + # Broadcast is_row_empty to the same shape as embedding_lookup_result, + # for use in Select. + is_row_empty = array_ops.tile( + array_ops.reshape(is_row_empty, [-1, 1]), + array_ops.stack([1, array_ops.shape(result)[1]])) + + result = array_ops.where(is_row_empty, + array_ops.zeros_like(result), + result, + name=scope) + + # Reshape back from linear ids back into higher-dimensional dense result. + final_result = array_ops.reshape( + result, + array_ops.concat([ + array_ops.slice( + math_ops.cast(original_shape, dtypes.int32), [0], + [original_rank - 1]), + array_ops.slice(array_ops.shape(result), [1], [-1]) + ], 0)) + final_result.set_shape(tensor_shape.unknown_shape( + (original_rank_dim - 1).value).concatenate(result.get_shape()[1:])) + return final_result + + +def _prune_invalid_ids(sparse_ids, sparse_weights): + """Prune invalid IDs (< 0) from the input ids and weights.""" + is_id_valid = math_ops.greater_equal(sparse_ids.values, 0) + if sparse_weights is not None: + is_id_valid = math_ops.logical_and( + is_id_valid, + array_ops.ones_like(sparse_weights.values, dtype=dtypes.bool)) + sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_id_valid) + if sparse_weights is not None: + sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_id_valid) + return sparse_ids, sparse_weights + + +def _prune_invalid_weights(sparse_ids, sparse_weights): + """Prune invalid weights (< 0) from the input ids and weights.""" + if sparse_weights is not None: + is_weights_valid = math_ops.greater(sparse_weights.values, 0) + sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_weights_valid) + sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_weights_valid) + return sparse_ids, sparse_weights diff --git a/tensorflow/python/ops/nn.py b/tensorflow/python/ops/nn.py index 1d0d9a52a1..fb896bf042 100644 --- a/tensorflow/python/ops/nn.py +++ b/tensorflow/python/ops/nn.py @@ -79,6 +79,7 @@ See the @{$python/nn} guide. @@weighted_cross_entropy_with_logits @@embedding_lookup @@embedding_lookup_sparse +@@safe_embedding_lookup_sparse @@dynamic_rnn @@bidirectional_dynamic_rnn @@raw_rnn -- GitLab From 608508c35a4b87a17b9f07364e6fbeae2fa948c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Tue, 24 Apr 2018 20:32:38 +0800 Subject: [PATCH 0013/3149] CLN: move the corresponding test case --- .../python/layers/embedding_ops_test.py | 1 - tensorflow/python/kernel_tests/BUILD | 1 + .../python/kernel_tests/embedding_ops_test.py | 218 ++++++++++++++++++ 3 files changed, 219 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py index bf25144982..87f00f94a6 100644 --- a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py +++ b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py @@ -21,7 +21,6 @@ from __future__ import print_function import itertools import math -import sys import numpy as np diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index ba8f1fd3ca..2fbdde849b 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -2717,6 +2717,7 @@ cuda_py_test( "//tensorflow/python:embedding_ops", "//tensorflow/python:framework", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:init_ops", "//tensorflow/python:linalg_ops", "//tensorflow/python:math_ops", "//tensorflow/python:partitioned_variables", diff --git a/tensorflow/python/kernel_tests/embedding_ops_test.py b/tensorflow/python/kernel_tests/embedding_ops_test.py index e53ca1dcaa..55d75cb474 100644 --- a/tensorflow/python/kernel_tests/embedding_ops_test.py +++ b/tensorflow/python/kernel_tests/embedding_ops_test.py @@ -19,6 +19,7 @@ from __future__ import division from __future__ import print_function import itertools +import math import numpy as np from six.moves import xrange # pylint: disable=redefined-builtin @@ -31,6 +32,7 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import data_flow_ops from tensorflow.python.ops import embedding_ops from tensorflow.python.ops import gradient_checker +from tensorflow.python.ops import init_ops from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import partitioned_variables @@ -736,6 +738,222 @@ class EmbeddingLookupSparseTest(test.TestCase): x, sp_ids, sp_weights, combiner="mean") +class SafeEmbeddingLookupSparseTest(test.TestCase): + + def _random_weights(self, vocab_size=4, embed_dim=4, num_shards=1): + assert vocab_size > 0 + assert embed_dim > 0 + assert num_shards > 0 + assert num_shards <= vocab_size + + embedding_weights = partitioned_variables.create_partitioned_variables( + shape=[vocab_size, embed_dim], + slicing=[num_shards, 1], + initializer=init_ops.truncated_normal_initializer( + mean=0.0, stddev=1.0 / math.sqrt(vocab_size), dtype=dtypes.float32)) + for w in embedding_weights: + w.initializer.run() + embedding_weights = [w.eval() for w in embedding_weights] + return embedding_weights + + def _ids_and_weights_2d(self): + # Each row demonstrates a test case: + # Row 0: multiple valid ids, 1 invalid id, weighted mean + # Row 1: all ids are invalid (leaving no valid ids after pruning) + # Row 2: no ids to begin with + # Row 3: single id + # Row 4: all ids have <=0 weight + indices = [[0, 0], [0, 1], [0, 2], [1, 0], [3, 0], [4, 0], [4, 1]] + ids = [0, 1, -1, -1, 2, 0, 1] + weights = [1.0, 2.0, 1.0, 1.0, 3.0, 0.0, -0.5] + shape = [5, 4] + + sparse_ids = sparse_tensor.SparseTensor( + constant_op.constant(indices, dtypes.int64), + constant_op.constant(ids, dtypes.int64), + constant_op.constant(shape, dtypes.int64)) + + sparse_weights = sparse_tensor.SparseTensor( + constant_op.constant(indices, dtypes.int64), + constant_op.constant(weights, dtypes.float32), + constant_op.constant(shape, dtypes.int64)) + + return sparse_ids, sparse_weights + + def _ids_and_weights_3d(self): + # Each (2-D) index demonstrates a test case: + # Index 0, 0: multiple valid ids, 1 invalid id, weighted mean + # Index 0, 1: all ids are invalid (leaving no valid ids after pruning) + # Index 0, 2: no ids to begin with + # Index 1, 0: single id + # Index 1, 1: all ids have <=0 weight + # Index 1, 2: no ids to begin with + indices = [[0, 0, 0], [0, 0, 1], [0, 0, 2], [0, 1, 0], [1, 0, 0], [1, 1, 0], + [1, 1, 1]] + ids = [0, 1, -1, -1, 2, 0, 1] + weights = [1.0, 2.0, 1.0, 1.0, 3.0, 0.0, -0.5] + shape = [2, 3, 4] + + sparse_ids = sparse_tensor.SparseTensor( + constant_op.constant(indices, dtypes.int64), + constant_op.constant(ids, dtypes.int64), + constant_op.constant(shape, dtypes.int64)) + + sparse_weights = sparse_tensor.SparseTensor( + constant_op.constant(indices, dtypes.int64), + constant_op.constant(weights, dtypes.float32), + constant_op.constant(shape, dtypes.int64)) + + return sparse_ids, sparse_weights + + def test_safe_embedding_lookup_sparse_return_zero_vector(self): + with self.test_session(): + embedding_weights = self._random_weights() + sparse_ids, sparse_weights = self._ids_and_weights_2d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, sparse_weights).eval()) + + self.assertAllClose( + embedding_lookup_result, + [(1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / + 3.0, [0] * 4, [0] * 4, embedding_weights[0][2], [0] * 4]) + + def test_safe_embedding_lookup_sparse_return_special_vector(self): + with self.test_session(): + embedding_weights = self._random_weights() + sparse_ids, sparse_weights = self._ids_and_weights_2d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, sparse_weights, default_id=3).eval()) + + self.assertAllClose( + embedding_lookup_result, + [(1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / + 3.0, embedding_weights[0][3], embedding_weights[0][3], + embedding_weights[0][2], embedding_weights[0][3]]) + + def test_safe_embedding_lookup_sparse_no_weights(self): + with self.test_session(): + embedding_weights = self._random_weights() + sparse_ids, _ = self._ids_and_weights_2d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, None).eval()) + + self.assertAllClose( + embedding_lookup_result, + [(embedding_weights[0][0] + embedding_weights[0][1]) / 2.0, [0] * 4, + [0] * 4, embedding_weights[0][2], ( + embedding_weights[0][0] + embedding_weights[0][1]) / 2.0]) + + def test_safe_embedding_lookup_sparse_partitioned(self): + with self.test_session(): + embedding_weights = self._random_weights(num_shards=3) + sparse_ids, _ = self._ids_and_weights_2d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, None).eval()) + + embedding_weights = list(itertools.chain(*embedding_weights)) + self.assertAllClose(embedding_lookup_result, + [(embedding_weights[0] + embedding_weights[1]) / 2.0, + [0] * 4, [0] * 4, embedding_weights[2], + (embedding_weights[0] + embedding_weights[1]) / 2.0]) + + def test_safe_embedding_lookup_sparse_partitioned_inconsistent_weights(self): + with self.test_session(): + embedding_weights = self._random_weights(num_shards=3) + sparse_ids, sparse_weights = self._ids_and_weights_2d() + + embedding_weights[1] = embedding_weights[1].astype(np.float64) + self.assertRaises(TypeError, embedding_ops.safe_embedding_lookup_sparse, + embedding_weights, sparse_ids) + embedding_weights = [ + constant_op.constant(w, dtype=dtypes.float64) + for w in embedding_weights + ] + self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, + embedding_weights, sparse_ids, sparse_weights) + + def test_safe_embedding_lookup_sparse_3d_return_zero_vector(self): + with self.test_session(): + embedding_weights = self._random_weights() + sparse_ids, sparse_weights = self._ids_and_weights_3d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, sparse_weights).eval()) + + self.assertAllClose(embedding_lookup_result, [[ + (1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / 3.0, + [0] * 4, [0] * 4 + ], [embedding_weights[0][2], [0] * 4, [0] * 4]]) + + def test_safe_embedding_lookup_sparse_3d_return_special_vector(self): + with self.test_session(): + embedding_weights = self._random_weights() + sparse_ids, sparse_weights = self._ids_and_weights_3d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, sparse_weights, default_id=3).eval()) + + self.assertAllClose( + embedding_lookup_result, + [[(1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / + 3.0, embedding_weights[0][3], embedding_weights[0][3]], [ + embedding_weights[0][2], embedding_weights[0][3], + embedding_weights[0][3] + ]]) + + def test_safe_embedding_lookup_sparse_3d_no_weights(self): + with self.test_session(): + embedding_weights = self._random_weights() + sparse_ids, _ = self._ids_and_weights_3d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, None).eval()) + + self.assertAllClose(embedding_lookup_result, [[( + embedding_weights[0][0] + embedding_weights[0][1]) / 2.0, [0] * 4, [ + 0 + ] * 4], [ + embedding_weights[0][2], + (embedding_weights[0][0] + embedding_weights[0][1]) / 2.0, [0] * 4 + ]]) + + def test_safe_embedding_lookup_sparse_3d_partitioned(self): + with self.test_session(): + embedding_weights = self._random_weights(num_shards=3) + sparse_ids, _ = self._ids_and_weights_3d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, None).eval()) + + embedding_weights = list(itertools.chain(*embedding_weights)) + self.assertAllClose(embedding_lookup_result, [[ + (embedding_weights[0] + embedding_weights[1]) / 2.0, [0] * 4, [0] * 4 + ], [ + embedding_weights[2], + (embedding_weights[0] + embedding_weights[1]) / 2.0, [0] * 4 + ]]) + + def test_safe_embedding_lookup_sparse_3d_partitioned_inconsistent_weights( + self): + with self.test_session(): + embedding_weights = self._random_weights(num_shards=3) + sparse_ids, sparse_weights = self._ids_and_weights_3d() + + embedding_weights[1] = embedding_weights[1].astype(np.float64) + self.assertRaises(TypeError, embedding_ops.safe_embedding_lookup_sparse, + embedding_weights, sparse_ids) + embedding_weights = [ + constant_op.constant(w, dtype=dtypes.float64) + for w in embedding_weights + ] + self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, + embedding_weights, sparse_ids, sparse_weights) + + class DynamicStitchOpTest(test.TestCase): def testCint32Cpu(self): -- GitLab From 067c85fb66345e61aee9428cd645cca786ed2bf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Tue, 24 Apr 2018 20:52:33 +0800 Subject: [PATCH 0014/3149] CLN: delete duplicate codes --- .../layers/python/layers/embedding_ops.py | 117 ++---------------- .../python/layers/embedding_ops_test.py | 4 +- 2 files changed, 11 insertions(+), 110 deletions(-) diff --git a/tensorflow/contrib/layers/python/layers/embedding_ops.py b/tensorflow/contrib/layers/python/layers/embedding_ops.py index 49c3faf3b7..4353bf9c28 100644 --- a/tensorflow/contrib/layers/python/layers/embedding_ops.py +++ b/tensorflow/contrib/layers/python/layers/embedding_ops.py @@ -19,14 +19,12 @@ from __future__ import print_function from six.moves import xrange # pylint: disable=redefined-builtin -from tensorflow.contrib.framework.python.framework import tensor_util as contrib_tensor_util from tensorflow.contrib.layers.python.ops import sparse_feature_cross_op from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import array_ops from tensorflow.python.ops import clip_ops from tensorflow.python.ops import control_flow_ops @@ -100,112 +98,15 @@ def safe_embedding_lookup_sparse(embedding_weights, logging.warn("The default value of combiner will change from \"mean\" " "to \"sqrtn\" after 2016/11/01.") combiner = "mean" - if embedding_weights is None: - raise ValueError("Missing embedding_weights %s." % embedding_weights) - if isinstance(embedding_weights, variables.PartitionedVariable): - embedding_weights = list(embedding_weights) # get underlying Variables. - if not isinstance(embedding_weights, list): - embedding_weights = [embedding_weights] - if len(embedding_weights) < 1: - raise ValueError("Missing embedding_weights %s." % embedding_weights) - - dtype = sparse_weights.dtype if sparse_weights is not None else None - if isinstance(embedding_weights, variables.PartitionedVariable): - embedding_weights = list(embedding_weights) - embedding_weights = [ - ops.convert_to_tensor(w, dtype=dtype) for w in embedding_weights - ] - - contrib_tensor_util.assert_same_float_dtype(embedding_weights + - [sparse_weights]) - - with ops.name_scope(name, "embedding_lookup", - embedding_weights + [sparse_ids, - sparse_weights]) as scope: - # Reshape higher-rank sparse ids and weights to linear segment ids. - original_shape = sparse_ids.dense_shape - original_rank_dim = sparse_ids.dense_shape.get_shape()[0] - original_rank = ( - array_ops.size(original_shape) - if original_rank_dim.value is None - else original_rank_dim.value) - sparse_ids = sparse_ops.sparse_reshape(sparse_ids, [ - math_ops.reduce_prod( - array_ops.slice(original_shape, [0], [original_rank - 1])), - array_ops.gather(original_shape, original_rank - 1)]) - if sparse_weights is not None: - sparse_weights = sparse_tensor.SparseTensor( - sparse_ids.indices, - sparse_weights.values, sparse_ids.dense_shape) - - # Prune invalid ids and weights. - sparse_ids, sparse_weights = _prune_invalid_ids(sparse_ids, sparse_weights) - if combiner != "sum": - sparse_ids, sparse_weights = _prune_invalid_weights( - sparse_ids, sparse_weights) - - # Fill in dummy values for empty features, if necessary. - sparse_ids, is_row_empty = sparse_ops.sparse_fill_empty_rows(sparse_ids, - default_id or - 0) - if sparse_weights is not None: - sparse_weights, _ = sparse_ops.sparse_fill_empty_rows(sparse_weights, 1.0) - - result = embedding_ops.embedding_lookup_sparse( - embedding_weights, - sparse_ids, - sparse_weights, - combiner=combiner, - partition_strategy=partition_strategy, - name=None if default_id is None else scope, - max_norm=max_norm) - - if default_id is None: - # Broadcast is_row_empty to the same shape as embedding_lookup_result, - # for use in Select. - is_row_empty = array_ops.tile( - array_ops.reshape(is_row_empty, [-1, 1]), - array_ops.stack([1, array_ops.shape(result)[1]])) - - result = array_ops.where(is_row_empty, - array_ops.zeros_like(result), - result, - name=scope) - - # Reshape back from linear ids back into higher-dimensional dense result. - final_result = array_ops.reshape( - result, - array_ops.concat([ - array_ops.slice( - math_ops.cast(original_shape, dtypes.int32), [0], - [original_rank - 1]), - array_ops.slice(array_ops.shape(result), [1], [-1]) - ], 0)) - final_result.set_shape(tensor_shape.unknown_shape( - (original_rank_dim - 1).value).concatenate(result.get_shape()[1:])) - return final_result - - -def _prune_invalid_ids(sparse_ids, sparse_weights): - """Prune invalid IDs (< 0) from the input ids and weights.""" - is_id_valid = math_ops.greater_equal(sparse_ids.values, 0) - if sparse_weights is not None: - is_id_valid = math_ops.logical_and( - is_id_valid, - array_ops.ones_like(sparse_weights.values, dtype=dtypes.bool)) - sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_id_valid) - if sparse_weights is not None: - sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_id_valid) - return sparse_ids, sparse_weights - - -def _prune_invalid_weights(sparse_ids, sparse_weights): - """Prune invalid weights (< 0) from the input ids and weights.""" - if sparse_weights is not None: - is_weights_valid = math_ops.greater(sparse_weights.values, 0) - sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_weights_valid) - sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_weights_valid) - return sparse_ids, sparse_weights + return embedding_ops.safe_embedding_lookup_sparse( + embedding_weights=embedding_weights, + sparse_ids=sparse_ids, + sparse_weights=sparse_weights, + combiner=combiner, + default_id=default_id, + name=name, + partition_strategy=partition_strategy, + max_norm=max_norm) def scattered_embedding_lookup(params, diff --git a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py index 87f00f94a6..4d9849b4b1 100644 --- a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py +++ b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py @@ -168,7 +168,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase): sparse_ids, sparse_weights = self._ids_and_weights_2d() embedding_weights[1] = embedding_weights[1].astype(np.float64) - self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, + self.assertRaises(TypeError, embedding_ops.safe_embedding_lookup_sparse, embedding_weights, sparse_ids) embedding_weights = [ constant_op.constant(w, dtype=dtypes.float64) @@ -245,7 +245,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase): sparse_ids, sparse_weights = self._ids_and_weights_3d() embedding_weights[1] = embedding_weights[1].astype(np.float64) - self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, + self.assertRaises(TypeError, embedding_ops.safe_embedding_lookup_sparse, embedding_weights, sparse_ids) embedding_weights = [ constant_op.constant(w, dtype=dtypes.float64) -- GitLab From 91ad552a52242b3d382eee6a3382c79be36b7df7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Tue, 24 Apr 2018 20:57:47 +0800 Subject: [PATCH 0015/3149] CLN: delete test cases in contrib --- .../python/layers/embedding_ops_test.py | 217 ------------------ 1 file changed, 217 deletions(-) diff --git a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py index 4d9849b4b1..f7b7ade39d 100644 --- a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py +++ b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py @@ -20,7 +20,6 @@ from __future__ import division from __future__ import print_function import itertools -import math import numpy as np @@ -39,222 +38,6 @@ from tensorflow.python.platform import test from tensorflow.python.util import compat -class SafeEmbeddingLookupSparseTest(test.TestCase): - - def _random_weights(self, vocab_size=4, embed_dim=4, num_shards=1): - assert vocab_size > 0 - assert embed_dim > 0 - assert num_shards > 0 - assert num_shards <= vocab_size - - embedding_weights = partitioned_variables.create_partitioned_variables( - shape=[vocab_size, embed_dim], - slicing=[num_shards, 1], - initializer=init_ops.truncated_normal_initializer( - mean=0.0, stddev=1.0 / math.sqrt(vocab_size), dtype=dtypes.float32)) - for w in embedding_weights: - w.initializer.run() - embedding_weights = [w.eval() for w in embedding_weights] - return embedding_weights - - def _ids_and_weights_2d(self): - # Each row demonstrates a test case: - # Row 0: multiple valid ids, 1 invalid id, weighted mean - # Row 1: all ids are invalid (leaving no valid ids after pruning) - # Row 2: no ids to begin with - # Row 3: single id - # Row 4: all ids have <=0 weight - indices = [[0, 0], [0, 1], [0, 2], [1, 0], [3, 0], [4, 0], [4, 1]] - ids = [0, 1, -1, -1, 2, 0, 1] - weights = [1.0, 2.0, 1.0, 1.0, 3.0, 0.0, -0.5] - shape = [5, 4] - - sparse_ids = sparse_tensor_lib.SparseTensor( - constant_op.constant(indices, dtypes.int64), - constant_op.constant(ids, dtypes.int64), - constant_op.constant(shape, dtypes.int64)) - - sparse_weights = sparse_tensor_lib.SparseTensor( - constant_op.constant(indices, dtypes.int64), - constant_op.constant(weights, dtypes.float32), - constant_op.constant(shape, dtypes.int64)) - - return sparse_ids, sparse_weights - - def _ids_and_weights_3d(self): - # Each (2-D) index demonstrates a test case: - # Index 0, 0: multiple valid ids, 1 invalid id, weighted mean - # Index 0, 1: all ids are invalid (leaving no valid ids after pruning) - # Index 0, 2: no ids to begin with - # Index 1, 0: single id - # Index 1, 1: all ids have <=0 weight - # Index 1, 2: no ids to begin with - indices = [[0, 0, 0], [0, 0, 1], [0, 0, 2], [0, 1, 0], [1, 0, 0], [1, 1, 0], - [1, 1, 1]] - ids = [0, 1, -1, -1, 2, 0, 1] - weights = [1.0, 2.0, 1.0, 1.0, 3.0, 0.0, -0.5] - shape = [2, 3, 4] - - sparse_ids = sparse_tensor_lib.SparseTensor( - constant_op.constant(indices, dtypes.int64), - constant_op.constant(ids, dtypes.int64), - constant_op.constant(shape, dtypes.int64)) - - sparse_weights = sparse_tensor_lib.SparseTensor( - constant_op.constant(indices, dtypes.int64), - constant_op.constant(weights, dtypes.float32), - constant_op.constant(shape, dtypes.int64)) - - return sparse_ids, sparse_weights - - def test_safe_embedding_lookup_sparse_return_zero_vector(self): - with self.test_session(): - embedding_weights = self._random_weights() - sparse_ids, sparse_weights = self._ids_and_weights_2d() - - embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( - embedding_weights, sparse_ids, sparse_weights).eval()) - - self.assertAllClose( - embedding_lookup_result, - [(1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / - 3.0, [0] * 4, [0] * 4, embedding_weights[0][2], [0] * 4]) - - def test_safe_embedding_lookup_sparse_return_special_vector(self): - with self.test_session(): - embedding_weights = self._random_weights() - sparse_ids, sparse_weights = self._ids_and_weights_2d() - - embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( - embedding_weights, sparse_ids, sparse_weights, default_id=3).eval()) - - self.assertAllClose( - embedding_lookup_result, - [(1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / - 3.0, embedding_weights[0][3], embedding_weights[0][3], - embedding_weights[0][2], embedding_weights[0][3]]) - - def test_safe_embedding_lookup_sparse_no_weights(self): - with self.test_session(): - embedding_weights = self._random_weights() - sparse_ids, _ = self._ids_and_weights_2d() - - embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( - embedding_weights, sparse_ids, None).eval()) - - self.assertAllClose( - embedding_lookup_result, - [(embedding_weights[0][0] + embedding_weights[0][1]) / 2.0, [0] * 4, - [0] * 4, embedding_weights[0][2], ( - embedding_weights[0][0] + embedding_weights[0][1]) / 2.0]) - - def test_safe_embedding_lookup_sparse_partitioned(self): - with self.test_session(): - embedding_weights = self._random_weights(num_shards=3) - sparse_ids, _ = self._ids_and_weights_2d() - - embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( - embedding_weights, sparse_ids, None).eval()) - - embedding_weights = list(itertools.chain(*embedding_weights)) - self.assertAllClose(embedding_lookup_result, - [(embedding_weights[0] + embedding_weights[1]) / 2.0, - [0] * 4, [0] * 4, embedding_weights[2], - (embedding_weights[0] + embedding_weights[1]) / 2.0]) - - def test_safe_embedding_lookup_sparse_partitioned_inconsistent_weights(self): - with self.test_session(): - embedding_weights = self._random_weights(num_shards=3) - sparse_ids, sparse_weights = self._ids_and_weights_2d() - - embedding_weights[1] = embedding_weights[1].astype(np.float64) - self.assertRaises(TypeError, embedding_ops.safe_embedding_lookup_sparse, - embedding_weights, sparse_ids) - embedding_weights = [ - constant_op.constant(w, dtype=dtypes.float64) - for w in embedding_weights - ] - self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, - embedding_weights, sparse_ids, sparse_weights) - - def test_safe_embedding_lookup_sparse_3d_return_zero_vector(self): - with self.test_session(): - embedding_weights = self._random_weights() - sparse_ids, sparse_weights = self._ids_and_weights_3d() - - embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( - embedding_weights, sparse_ids, sparse_weights).eval()) - - self.assertAllClose(embedding_lookup_result, [[ - (1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / 3.0, - [0] * 4, [0] * 4 - ], [embedding_weights[0][2], [0] * 4, [0] * 4]]) - - def test_safe_embedding_lookup_sparse_3d_return_special_vector(self): - with self.test_session(): - embedding_weights = self._random_weights() - sparse_ids, sparse_weights = self._ids_and_weights_3d() - - embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( - embedding_weights, sparse_ids, sparse_weights, default_id=3).eval()) - - self.assertAllClose( - embedding_lookup_result, - [[(1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / - 3.0, embedding_weights[0][3], embedding_weights[0][3]], [ - embedding_weights[0][2], embedding_weights[0][3], - embedding_weights[0][3] - ]]) - - def test_safe_embedding_lookup_sparse_3d_no_weights(self): - with self.test_session(): - embedding_weights = self._random_weights() - sparse_ids, _ = self._ids_and_weights_3d() - - embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( - embedding_weights, sparse_ids, None).eval()) - - self.assertAllClose(embedding_lookup_result, [[( - embedding_weights[0][0] + embedding_weights[0][1]) / 2.0, [0] * 4, [ - 0 - ] * 4], [ - embedding_weights[0][2], - (embedding_weights[0][0] + embedding_weights[0][1]) / 2.0, [0] * 4 - ]]) - - def test_safe_embedding_lookup_sparse_3d_partitioned(self): - with self.test_session(): - embedding_weights = self._random_weights(num_shards=3) - sparse_ids, _ = self._ids_and_weights_3d() - - embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( - embedding_weights, sparse_ids, None).eval()) - - embedding_weights = list(itertools.chain(*embedding_weights)) - self.assertAllClose(embedding_lookup_result, [[ - (embedding_weights[0] + embedding_weights[1]) / 2.0, [0] * 4, [0] * 4 - ], [ - embedding_weights[2], - (embedding_weights[0] + embedding_weights[1]) / 2.0, [0] * 4 - ]]) - - def test_safe_embedding_lookup_sparse_3d_partitioned_inconsistent_weights( - self): - with self.test_session(): - embedding_weights = self._random_weights(num_shards=3) - sparse_ids, sparse_weights = self._ids_and_weights_3d() - - embedding_weights[1] = embedding_weights[1].astype(np.float64) - self.assertRaises(TypeError, embedding_ops.safe_embedding_lookup_sparse, - embedding_weights, sparse_ids) - embedding_weights = [ - constant_op.constant(w, dtype=dtypes.float64) - for w in embedding_weights - ] - self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, - embedding_weights, sparse_ids, sparse_weights) - - class ScatteredEmbeddingLookupTest(test.TestCase): def setUp(self): -- GitLab From 24a6350ad173865c16351825f251f2fde97b7d9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Thu, 26 Apr 2018 07:01:06 +0800 Subject: [PATCH 0016/3149] Revert "CLN: delete test cases in contrib" This reverts commit 91ad552a52242b3d382eee6a3382c79be36b7df7. --- .../python/layers/embedding_ops_test.py | 217 ++++++++++++++++++ 1 file changed, 217 insertions(+) diff --git a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py index f7b7ade39d..4d9849b4b1 100644 --- a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py +++ b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py @@ -20,6 +20,7 @@ from __future__ import division from __future__ import print_function import itertools +import math import numpy as np @@ -38,6 +39,222 @@ from tensorflow.python.platform import test from tensorflow.python.util import compat +class SafeEmbeddingLookupSparseTest(test.TestCase): + + def _random_weights(self, vocab_size=4, embed_dim=4, num_shards=1): + assert vocab_size > 0 + assert embed_dim > 0 + assert num_shards > 0 + assert num_shards <= vocab_size + + embedding_weights = partitioned_variables.create_partitioned_variables( + shape=[vocab_size, embed_dim], + slicing=[num_shards, 1], + initializer=init_ops.truncated_normal_initializer( + mean=0.0, stddev=1.0 / math.sqrt(vocab_size), dtype=dtypes.float32)) + for w in embedding_weights: + w.initializer.run() + embedding_weights = [w.eval() for w in embedding_weights] + return embedding_weights + + def _ids_and_weights_2d(self): + # Each row demonstrates a test case: + # Row 0: multiple valid ids, 1 invalid id, weighted mean + # Row 1: all ids are invalid (leaving no valid ids after pruning) + # Row 2: no ids to begin with + # Row 3: single id + # Row 4: all ids have <=0 weight + indices = [[0, 0], [0, 1], [0, 2], [1, 0], [3, 0], [4, 0], [4, 1]] + ids = [0, 1, -1, -1, 2, 0, 1] + weights = [1.0, 2.0, 1.0, 1.0, 3.0, 0.0, -0.5] + shape = [5, 4] + + sparse_ids = sparse_tensor_lib.SparseTensor( + constant_op.constant(indices, dtypes.int64), + constant_op.constant(ids, dtypes.int64), + constant_op.constant(shape, dtypes.int64)) + + sparse_weights = sparse_tensor_lib.SparseTensor( + constant_op.constant(indices, dtypes.int64), + constant_op.constant(weights, dtypes.float32), + constant_op.constant(shape, dtypes.int64)) + + return sparse_ids, sparse_weights + + def _ids_and_weights_3d(self): + # Each (2-D) index demonstrates a test case: + # Index 0, 0: multiple valid ids, 1 invalid id, weighted mean + # Index 0, 1: all ids are invalid (leaving no valid ids after pruning) + # Index 0, 2: no ids to begin with + # Index 1, 0: single id + # Index 1, 1: all ids have <=0 weight + # Index 1, 2: no ids to begin with + indices = [[0, 0, 0], [0, 0, 1], [0, 0, 2], [0, 1, 0], [1, 0, 0], [1, 1, 0], + [1, 1, 1]] + ids = [0, 1, -1, -1, 2, 0, 1] + weights = [1.0, 2.0, 1.0, 1.0, 3.0, 0.0, -0.5] + shape = [2, 3, 4] + + sparse_ids = sparse_tensor_lib.SparseTensor( + constant_op.constant(indices, dtypes.int64), + constant_op.constant(ids, dtypes.int64), + constant_op.constant(shape, dtypes.int64)) + + sparse_weights = sparse_tensor_lib.SparseTensor( + constant_op.constant(indices, dtypes.int64), + constant_op.constant(weights, dtypes.float32), + constant_op.constant(shape, dtypes.int64)) + + return sparse_ids, sparse_weights + + def test_safe_embedding_lookup_sparse_return_zero_vector(self): + with self.test_session(): + embedding_weights = self._random_weights() + sparse_ids, sparse_weights = self._ids_and_weights_2d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, sparse_weights).eval()) + + self.assertAllClose( + embedding_lookup_result, + [(1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / + 3.0, [0] * 4, [0] * 4, embedding_weights[0][2], [0] * 4]) + + def test_safe_embedding_lookup_sparse_return_special_vector(self): + with self.test_session(): + embedding_weights = self._random_weights() + sparse_ids, sparse_weights = self._ids_and_weights_2d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, sparse_weights, default_id=3).eval()) + + self.assertAllClose( + embedding_lookup_result, + [(1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / + 3.0, embedding_weights[0][3], embedding_weights[0][3], + embedding_weights[0][2], embedding_weights[0][3]]) + + def test_safe_embedding_lookup_sparse_no_weights(self): + with self.test_session(): + embedding_weights = self._random_weights() + sparse_ids, _ = self._ids_and_weights_2d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, None).eval()) + + self.assertAllClose( + embedding_lookup_result, + [(embedding_weights[0][0] + embedding_weights[0][1]) / 2.0, [0] * 4, + [0] * 4, embedding_weights[0][2], ( + embedding_weights[0][0] + embedding_weights[0][1]) / 2.0]) + + def test_safe_embedding_lookup_sparse_partitioned(self): + with self.test_session(): + embedding_weights = self._random_weights(num_shards=3) + sparse_ids, _ = self._ids_and_weights_2d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, None).eval()) + + embedding_weights = list(itertools.chain(*embedding_weights)) + self.assertAllClose(embedding_lookup_result, + [(embedding_weights[0] + embedding_weights[1]) / 2.0, + [0] * 4, [0] * 4, embedding_weights[2], + (embedding_weights[0] + embedding_weights[1]) / 2.0]) + + def test_safe_embedding_lookup_sparse_partitioned_inconsistent_weights(self): + with self.test_session(): + embedding_weights = self._random_weights(num_shards=3) + sparse_ids, sparse_weights = self._ids_and_weights_2d() + + embedding_weights[1] = embedding_weights[1].astype(np.float64) + self.assertRaises(TypeError, embedding_ops.safe_embedding_lookup_sparse, + embedding_weights, sparse_ids) + embedding_weights = [ + constant_op.constant(w, dtype=dtypes.float64) + for w in embedding_weights + ] + self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, + embedding_weights, sparse_ids, sparse_weights) + + def test_safe_embedding_lookup_sparse_3d_return_zero_vector(self): + with self.test_session(): + embedding_weights = self._random_weights() + sparse_ids, sparse_weights = self._ids_and_weights_3d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, sparse_weights).eval()) + + self.assertAllClose(embedding_lookup_result, [[ + (1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / 3.0, + [0] * 4, [0] * 4 + ], [embedding_weights[0][2], [0] * 4, [0] * 4]]) + + def test_safe_embedding_lookup_sparse_3d_return_special_vector(self): + with self.test_session(): + embedding_weights = self._random_weights() + sparse_ids, sparse_weights = self._ids_and_weights_3d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, sparse_weights, default_id=3).eval()) + + self.assertAllClose( + embedding_lookup_result, + [[(1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / + 3.0, embedding_weights[0][3], embedding_weights[0][3]], [ + embedding_weights[0][2], embedding_weights[0][3], + embedding_weights[0][3] + ]]) + + def test_safe_embedding_lookup_sparse_3d_no_weights(self): + with self.test_session(): + embedding_weights = self._random_weights() + sparse_ids, _ = self._ids_and_weights_3d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, None).eval()) + + self.assertAllClose(embedding_lookup_result, [[( + embedding_weights[0][0] + embedding_weights[0][1]) / 2.0, [0] * 4, [ + 0 + ] * 4], [ + embedding_weights[0][2], + (embedding_weights[0][0] + embedding_weights[0][1]) / 2.0, [0] * 4 + ]]) + + def test_safe_embedding_lookup_sparse_3d_partitioned(self): + with self.test_session(): + embedding_weights = self._random_weights(num_shards=3) + sparse_ids, _ = self._ids_and_weights_3d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, None).eval()) + + embedding_weights = list(itertools.chain(*embedding_weights)) + self.assertAllClose(embedding_lookup_result, [[ + (embedding_weights[0] + embedding_weights[1]) / 2.0, [0] * 4, [0] * 4 + ], [ + embedding_weights[2], + (embedding_weights[0] + embedding_weights[1]) / 2.0, [0] * 4 + ]]) + + def test_safe_embedding_lookup_sparse_3d_partitioned_inconsistent_weights( + self): + with self.test_session(): + embedding_weights = self._random_weights(num_shards=3) + sparse_ids, sparse_weights = self._ids_and_weights_3d() + + embedding_weights[1] = embedding_weights[1].astype(np.float64) + self.assertRaises(TypeError, embedding_ops.safe_embedding_lookup_sparse, + embedding_weights, sparse_ids) + embedding_weights = [ + constant_op.constant(w, dtype=dtypes.float64) + for w in embedding_weights + ] + self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, + embedding_weights, sparse_ids, sparse_weights) + + class ScatteredEmbeddingLookupTest(test.TestCase): def setUp(self): -- GitLab From 1c1b4d47707a439c157b5dcf3755e391730a328c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Thu, 26 Apr 2018 07:01:21 +0800 Subject: [PATCH 0017/3149] Revert "CLN: delete duplicate codes" This reverts commit 067c85fb66345e61aee9428cd645cca786ed2bf4. --- .../layers/python/layers/embedding_ops.py | 117 ++++++++++++++++-- .../python/layers/embedding_ops_test.py | 4 +- 2 files changed, 110 insertions(+), 11 deletions(-) diff --git a/tensorflow/contrib/layers/python/layers/embedding_ops.py b/tensorflow/contrib/layers/python/layers/embedding_ops.py index 4353bf9c28..49c3faf3b7 100644 --- a/tensorflow/contrib/layers/python/layers/embedding_ops.py +++ b/tensorflow/contrib/layers/python/layers/embedding_ops.py @@ -19,12 +19,14 @@ from __future__ import print_function from six.moves import xrange # pylint: disable=redefined-builtin +from tensorflow.contrib.framework.python.framework import tensor_util as contrib_tensor_util from tensorflow.contrib.layers.python.ops import sparse_feature_cross_op from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import array_ops from tensorflow.python.ops import clip_ops from tensorflow.python.ops import control_flow_ops @@ -98,15 +100,112 @@ def safe_embedding_lookup_sparse(embedding_weights, logging.warn("The default value of combiner will change from \"mean\" " "to \"sqrtn\" after 2016/11/01.") combiner = "mean" - return embedding_ops.safe_embedding_lookup_sparse( - embedding_weights=embedding_weights, - sparse_ids=sparse_ids, - sparse_weights=sparse_weights, - combiner=combiner, - default_id=default_id, - name=name, - partition_strategy=partition_strategy, - max_norm=max_norm) + if embedding_weights is None: + raise ValueError("Missing embedding_weights %s." % embedding_weights) + if isinstance(embedding_weights, variables.PartitionedVariable): + embedding_weights = list(embedding_weights) # get underlying Variables. + if not isinstance(embedding_weights, list): + embedding_weights = [embedding_weights] + if len(embedding_weights) < 1: + raise ValueError("Missing embedding_weights %s." % embedding_weights) + + dtype = sparse_weights.dtype if sparse_weights is not None else None + if isinstance(embedding_weights, variables.PartitionedVariable): + embedding_weights = list(embedding_weights) + embedding_weights = [ + ops.convert_to_tensor(w, dtype=dtype) for w in embedding_weights + ] + + contrib_tensor_util.assert_same_float_dtype(embedding_weights + + [sparse_weights]) + + with ops.name_scope(name, "embedding_lookup", + embedding_weights + [sparse_ids, + sparse_weights]) as scope: + # Reshape higher-rank sparse ids and weights to linear segment ids. + original_shape = sparse_ids.dense_shape + original_rank_dim = sparse_ids.dense_shape.get_shape()[0] + original_rank = ( + array_ops.size(original_shape) + if original_rank_dim.value is None + else original_rank_dim.value) + sparse_ids = sparse_ops.sparse_reshape(sparse_ids, [ + math_ops.reduce_prod( + array_ops.slice(original_shape, [0], [original_rank - 1])), + array_ops.gather(original_shape, original_rank - 1)]) + if sparse_weights is not None: + sparse_weights = sparse_tensor.SparseTensor( + sparse_ids.indices, + sparse_weights.values, sparse_ids.dense_shape) + + # Prune invalid ids and weights. + sparse_ids, sparse_weights = _prune_invalid_ids(sparse_ids, sparse_weights) + if combiner != "sum": + sparse_ids, sparse_weights = _prune_invalid_weights( + sparse_ids, sparse_weights) + + # Fill in dummy values for empty features, if necessary. + sparse_ids, is_row_empty = sparse_ops.sparse_fill_empty_rows(sparse_ids, + default_id or + 0) + if sparse_weights is not None: + sparse_weights, _ = sparse_ops.sparse_fill_empty_rows(sparse_weights, 1.0) + + result = embedding_ops.embedding_lookup_sparse( + embedding_weights, + sparse_ids, + sparse_weights, + combiner=combiner, + partition_strategy=partition_strategy, + name=None if default_id is None else scope, + max_norm=max_norm) + + if default_id is None: + # Broadcast is_row_empty to the same shape as embedding_lookup_result, + # for use in Select. + is_row_empty = array_ops.tile( + array_ops.reshape(is_row_empty, [-1, 1]), + array_ops.stack([1, array_ops.shape(result)[1]])) + + result = array_ops.where(is_row_empty, + array_ops.zeros_like(result), + result, + name=scope) + + # Reshape back from linear ids back into higher-dimensional dense result. + final_result = array_ops.reshape( + result, + array_ops.concat([ + array_ops.slice( + math_ops.cast(original_shape, dtypes.int32), [0], + [original_rank - 1]), + array_ops.slice(array_ops.shape(result), [1], [-1]) + ], 0)) + final_result.set_shape(tensor_shape.unknown_shape( + (original_rank_dim - 1).value).concatenate(result.get_shape()[1:])) + return final_result + + +def _prune_invalid_ids(sparse_ids, sparse_weights): + """Prune invalid IDs (< 0) from the input ids and weights.""" + is_id_valid = math_ops.greater_equal(sparse_ids.values, 0) + if sparse_weights is not None: + is_id_valid = math_ops.logical_and( + is_id_valid, + array_ops.ones_like(sparse_weights.values, dtype=dtypes.bool)) + sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_id_valid) + if sparse_weights is not None: + sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_id_valid) + return sparse_ids, sparse_weights + + +def _prune_invalid_weights(sparse_ids, sparse_weights): + """Prune invalid weights (< 0) from the input ids and weights.""" + if sparse_weights is not None: + is_weights_valid = math_ops.greater(sparse_weights.values, 0) + sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_weights_valid) + sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_weights_valid) + return sparse_ids, sparse_weights def scattered_embedding_lookup(params, diff --git a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py index 4d9849b4b1..87f00f94a6 100644 --- a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py +++ b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py @@ -168,7 +168,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase): sparse_ids, sparse_weights = self._ids_and_weights_2d() embedding_weights[1] = embedding_weights[1].astype(np.float64) - self.assertRaises(TypeError, embedding_ops.safe_embedding_lookup_sparse, + self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, embedding_weights, sparse_ids) embedding_weights = [ constant_op.constant(w, dtype=dtypes.float64) @@ -245,7 +245,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase): sparse_ids, sparse_weights = self._ids_and_weights_3d() embedding_weights[1] = embedding_weights[1].astype(np.float64) - self.assertRaises(TypeError, embedding_ops.safe_embedding_lookup_sparse, + self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, embedding_weights, sparse_ids) embedding_weights = [ constant_op.constant(w, dtype=dtypes.float64) -- GitLab From f2a0bc58db70cc792649672b81317288c4151ebb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Thu, 26 Apr 2018 20:52:03 +0800 Subject: [PATCH 0018/3149] BLD: update golden --- tensorflow/tools/api/golden/tensorflow.nn.pbtxt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tensorflow/tools/api/golden/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.pbtxt index 455590d866..d9e5b0d0fc 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.pbtxt @@ -260,6 +260,10 @@ tf_module { name: "relu_layer" argspec: "args=[\'x\', \'weights\', \'biases\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "safe_embedding_lookup_sparse" + argspec: "args=[\'embedding_weights\', \'sparse_ids\', \'sparse_weights\', \'combiner\', \'default_id\', \'name\', \'partition_strategy\', \'max_norm\'], varargs=None, keywords=None, defaults=[\'None\', \'mean\', \'None\', \'None\', \'div\', \'None\'], " + } member_method { name: "sampled_softmax_loss" argspec: "args=[\'weights\', \'biases\', \'labels\', \'inputs\', \'num_sampled\', \'num_classes\', \'num_true\', \'sampled_values\', \'remove_accidental_hits\', \'partition_strategy\', \'name\', \'seed\'], varargs=None, keywords=None, defaults=[\'1\', \'None\', \'True\', \'mod\', \'sampled_softmax_loss\', \'None\'], " -- GitLab From cd2ba0c063ffd89f0310a6ab6482a5607e590cb1 Mon Sep 17 00:00:00 2001 From: Dan Osipov Date: Sun, 18 Mar 2018 18:50:34 -0700 Subject: [PATCH 0019/3149] Document additional argument --- tensorflow/python/ops/image_ops_impl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py index 8524c08f81..cee948fe43 100644 --- a/tensorflow/python/ops/image_ops_impl.py +++ b/tensorflow/python/ops/image_ops_impl.py @@ -984,6 +984,7 @@ def resize_image_aspect_with_pad(image, target_height, target_width, 3-D Tensor of shape `[height, width, channels]`. target_height: Target height. target_width: Target width. + method: Method to use for resizing image. See `resize_images()` Raises: ValueError: if `target_height` or `target_width` are zero or negative. -- GitLab From 96dc82647d0eb5d1903242c2dde1cf9dd5bb36f0 Mon Sep 17 00:00:00 2001 From: Dan Osipov Date: Sat, 28 Apr 2018 08:28:33 -0700 Subject: [PATCH 0020/3149] Rename API method --- tensorflow/python/ops/image_ops.py | 2 +- tensorflow/python/ops/image_ops_impl.py | 6 +++--- tensorflow/python/ops/image_ops_test.py | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tensorflow/python/ops/image_ops.py b/tensorflow/python/ops/image_ops.py index f11b6dcea6..091ec61b1f 100644 --- a/tensorflow/python/ops/image_ops.py +++ b/tensorflow/python/ops/image_ops.py @@ -36,7 +36,7 @@ See the @{$python/image} guide. @@resize_bilinear @@resize_nearest_neighbor @@resize_image_with_crop_or_pad -@@resize_image_aspect_with_pad +@@resize_image_with_pad @@central_crop @@pad_to_bounding_box @@crop_to_bounding_box diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py index cee948fe43..5fe0b7a251 100644 --- a/tensorflow/python/ops/image_ops_impl.py +++ b/tensorflow/python/ops/image_ops_impl.py @@ -969,8 +969,8 @@ def resize_images(images, return images -@tf_export('image.resize_image_aspect_with_pad') -def resize_image_aspect_with_pad(image, target_height, target_width, +@tf_export('image.resize_image_with_pad') +def resize_image_with_pad(image, target_height, target_width, method=ResizeMethod.BILINEAR): """ Resizes and pads an image to a target width and height. @@ -996,7 +996,7 @@ def resize_image_aspect_with_pad(image, target_height, target_width, If `images` was 3-D, a 3-D float Tensor of shape `[new_height, new_width, channels]`. """ - with ops.name_scope(None, 'resize_image_aspect_with_pad', [image]): + with ops.name_scope(None, 'resize_image_with_pad', [image]): image = ops.convert_to_tensor(image, name='image') image_shape = image.get_shape() is_batch = True diff --git a/tensorflow/python/ops/image_ops_test.py b/tensorflow/python/ops/image_ops_test.py index 40a4d175ac..22d9ce4289 100644 --- a/tensorflow/python/ops/image_ops_test.py +++ b/tensorflow/python/ops/image_ops_test.py @@ -2458,9 +2458,9 @@ class ResizeImagesTest(test_util.TensorFlowTestCase): self.assertTrue(y.op.name.startswith("resize_images")) -class ResizeImageAspectWithPadTest(test_util.TensorFlowTestCase): +class ResizeImageWithPadTest(test_util.TensorFlowTestCase): - def _ResizeImageAspectWithPad(self, x, target_height, target_width, + def _ResizeImageWithPad(self, x, target_height, target_width, use_tensor_inputs): if use_tensor_inputs: target_height = ops.convert_to_tensor(target_height) @@ -2471,7 +2471,7 @@ class ResizeImageAspectWithPadTest(test_util.TensorFlowTestCase): x_tensor = x feed_dict = {} - y = image_ops.resize_image_aspect_with_pad(x_tensor, target_height, + y = image_ops.resize_image_with_pad(x_tensor, target_height, target_width) if not use_tensor_inputs: self.assertTrue(y.get_shape().is_fully_defined()) @@ -2491,7 +2491,7 @@ class ResizeImageAspectWithPadTest(test_util.TensorFlowTestCase): y = np.array(y).reshape(y_shape) for use_tensor_inputs in use_tensor_inputs_options: - y_tf = self._ResizeImageAspectWithPad(x, target_height, target_width, + y_tf = self._ResizeImageWithPad(x, target_height, target_width, use_tensor_inputs) self.assertAllClose(y, y_tf) @@ -2507,7 +2507,7 @@ class ResizeImageAspectWithPadTest(test_util.TensorFlowTestCase): for use_tensor_inputs in use_tensor_inputs_options: try: - self._ResizeImageAspectWithPad(x, target_height, target_width, + self._ResizeImageWithPad(x, target_height, target_width, use_tensor_inputs) except Exception as e: if err_msg not in str(e): @@ -2517,7 +2517,7 @@ class ResizeImageAspectWithPadTest(test_util.TensorFlowTestCase): def _assertShapeInference(self, pre_shape, height, width, post_shape): image = array_ops.placeholder(dtypes.float32, shape=pre_shape) - y = image_ops.resize_image_aspect_with_pad(image, height, width) + y = image_ops.resize_image_with_pad(image, height, width) self.assertEqual(y.get_shape().as_list(), post_shape) def testNoOp(self): -- GitLab From 533cb5caa4c88d3f76e1994e8f039ea04d342482 Mon Sep 17 00:00:00 2001 From: Dan Osipov Date: Sat, 28 Apr 2018 08:30:56 -0700 Subject: [PATCH 0021/3149] Remove assertions --- tensorflow/python/ops/image_ops_impl.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py index 5fe0b7a251..e174feedb5 100644 --- a/tensorflow/python/ops/image_ops_impl.py +++ b/tensorflow/python/ops/image_ops_impl.py @@ -1066,16 +1066,6 @@ def resize_image_with_pad(image, target_height, target_width, _, resized_height, resized_width, _ = _ImageDimensions(resized, rank=4) - assert_ops = [] - assert_ops += _assert( - equal_(resized_height, target_height), ValueError, - 'resized height is not correct.') - assert_ops += _assert( - equal_(resized_width, target_width), ValueError, - 'resized width is not correct.') - - resized = control_flow_ops.with_dependencies(assert_ops, resized) - if not is_batch: resized = array_ops.squeeze(resized, squeeze_dims=[0]) -- GitLab From 764ea231d9b649ad167fd1ffd4f4c5c4e79642c7 Mon Sep 17 00:00:00 2001 From: Dan Osipov Date: Sat, 28 Apr 2018 08:32:36 -0700 Subject: [PATCH 0022/3149] Update docstring --- tensorflow/python/ops/image_ops_impl.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py index e174feedb5..d5ac72bac6 100644 --- a/tensorflow/python/ops/image_ops_impl.py +++ b/tensorflow/python/ops/image_ops_impl.py @@ -976,8 +976,9 @@ def resize_image_with_pad(image, target_height, target_width, Resizes and pads an image to a target width and height. Resizes an image to a target width and height by keeping - the aspect ratio the same without distortion and padding - it evenly with zeros. + the aspect ratio the same without distortion. If the target + dimensions don't match the image dimensions, the image + is padded with zeroes prior to resizing. Args: image: 4-D Tensor of shape `[batch, height, width, channels]` or -- GitLab From 37f71fd3a983d0bacab4ecd96679f7d55202dd28 Mon Sep 17 00:00:00 2001 From: Nehal J Wani Date: Fri, 27 Apr 2018 15:12:59 -0500 Subject: [PATCH 0023/3149] Update is_windows() to allow mingw shells --- tensorflow/tools/pip_package/build_pip_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/pip_package/build_pip_package.sh b/tensorflow/tools/pip_package/build_pip_package.sh index 8f0cf8c3d1..9108eb4f6f 100755 --- a/tensorflow/tools/pip_package/build_pip_package.sh +++ b/tensorflow/tools/pip_package/build_pip_package.sh @@ -34,7 +34,7 @@ function cp_external() { PLATFORM="$(uname -s | tr 'A-Z' 'a-z')" function is_windows() { # On windows, the shell script is actually running in msys - if [[ "${PLATFORM}" =~ msys_nt* ]]; then + if [[ "${PLATFORM}" =~ (mingw64|msys)_nt* ]]; then true else false -- GitLab From 14a9c4b3c9ca8aa93aeff6b8bc4ef2413cea1d24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Fri, 4 May 2018 18:12:35 +0800 Subject: [PATCH 0024/3149] ENH: add UnsafeDiv op --- tensorflow/core/kernels/cwise_op_div.cc | 3 +++ tensorflow/core/kernels/cwise_ops.h | 24 ++++++++++++++++++++++++ tensorflow/core/ops/math_ops.cc | 3 +++ 3 files changed, 30 insertions(+) diff --git a/tensorflow/core/kernels/cwise_op_div.cc b/tensorflow/core/kernels/cwise_op_div.cc index b12652f7fb..97f457d7d7 100644 --- a/tensorflow/core/kernels/cwise_op_div.cc +++ b/tensorflow/core/kernels/cwise_op_div.cc @@ -24,6 +24,9 @@ REGISTER5(BinaryOp, CPU, "TruncateDiv", functor::safe_div, uint8, uint16, int16, int32, int64); REGISTER6(BinaryOp, CPU, "RealDiv", functor::div, float, Eigen::half, double, bfloat16, complex64, complex128); +REGISTER3(BinaryOp, CPU, "UnsafeDiv", functor::unsafe_div, + float, Eigen::half, double); + #if GOOGLE_CUDA REGISTER9(BinaryOp, GPU, "Div", functor::div, float, Eigen::half, double, uint8, uint16, int16, int64, complex64, complex128); diff --git a/tensorflow/core/kernels/cwise_ops.h b/tensorflow/core/kernels/cwise_ops.h index a80905d145..a54892c53f 100644 --- a/tensorflow/core/kernels/cwise_ops.h +++ b/tensorflow/core/kernels/cwise_ops.h @@ -153,6 +153,27 @@ struct functor_traits> { }; }; +template +struct unsafe_div_op { + EIGEN_EMPTY_STRUCT_CTOR(unsafe_div_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T operator()(const T& a, + const T& b) const { + if (TF_PREDICT_TRUE(b != 0)) { + return scalar_quotient_op()(a, b); + } else { + return 0; + } + } +}; + +template +struct functor_traits> { + enum { + Cost = functor_traits>::Cost + NumTraits::AddCost, + PacketAccess = false, + }; +}; + // scalar_left and scalar_right are template helpers to partially // apply a binary function. // @@ -714,6 +735,9 @@ struct safe_div : base +struct unsafe_div : base> {}; + template struct fmod : base> {}; diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc index 8f8443a46c..1a2e1d70c1 100644 --- a/tensorflow/core/ops/math_ops.cc +++ b/tensorflow/core/ops/math_ops.cc @@ -375,6 +375,9 @@ Returns x * y element-wise. REGISTER_OP("Div").BINARY_MORE().SetShapeFn( shape_inference::BroadcastBinaryOpShapeFn); +REGISTER_OP("UnsafeDiv").BINARY_MORE().SetShapeFn( + shape_inference::BroadcastBinaryOpShapeFn); + REGISTER_OP("FloorDiv") .BINARY_MORE() .SetShapeFn(shape_inference::BroadcastBinaryOpShapeFn); -- GitLab From 1bb62968ec5bd726f7cbc11a00b6001d64407ca9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Sat, 5 May 2018 12:29:20 +0800 Subject: [PATCH 0025/3149] TST: add test case --- tensorflow/python/ops/math_ops.py | 22 ++++++++++++++++++++++ tensorflow/python/ops/math_ops_test.py | 15 +++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py index 477f870060..29c14b975a 100644 --- a/tensorflow/python/ops/math_ops.py +++ b/tensorflow/python/ops/math_ops.py @@ -1034,6 +1034,28 @@ def div(x, y, name=None): return _div_python2(x, y, name) +def unsafe_div(x, y, name=None): + """Divide two values using Python 2 semantics. Used for Tensor.__div__. + + Args: + x: `Tensor` numerator of real numeric type. + y: `Tensor` denominator of real numeric type. + name: A name for the operation (optional). + Returns: + `x / y` returns the quotient of x and y. + """ + + with ops.name_scope(name, "unsafe_div", [x, y]) as name: + x = ops.convert_to_tensor(x, name="x") + y = ops.convert_to_tensor(y, name="y", dtype=x.dtype.base_dtype) + x_dtype = x.dtype.base_dtype + y_dtype = y.dtype.base_dtype + if x_dtype != y_dtype: + raise TypeError("x and y must have the same dtype, got %r != %r" % + (x_dtype, y_dtype)) + return gen_math_ops.unsafe_div(x, y, name=name) + + # TODO(aselle): This should be removed mod = gen_math_ops.floor_mod diff --git a/tensorflow/python/ops/math_ops_test.py b/tensorflow/python/ops/math_ops_test.py index 05bcee8801..2b78a433f7 100644 --- a/tensorflow/python/ops/math_ops_test.py +++ b/tensorflow/python/ops/math_ops_test.py @@ -474,5 +474,20 @@ class DivAndModTest(test_util.TensorFlowTestCase): self.assertAllEqual(tf_result, expanded_nums) +@test_util.with_c_api +class UnsafeDivTest(test_util.TensorFlowTestCase): + + def testBasic(self): + nums = np.arange(-10, 10, .25).reshape(80, 1) + divs = np.arange(-3, 3, .25).reshape(1, 24) + + np_result = np.true_divide(nums, divs) + np_result[:, divs[0] == 0] = 0 + + with self.test_session(): + tf_result = math_ops.unsafe_div(nums, divs).eval() + self.assertAllEqual(tf_result, np_result) + + if __name__ == "__main__": googletest.main() -- GitLab From 043b574402c58e1cf629242b3faad3ec071e5ce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Sat, 5 May 2018 17:16:04 +0800 Subject: [PATCH 0026/3149] ENH: add gradient function --- tensorflow/cc/gradients/math_grad.cc | 15 ++++++++++++++ tensorflow/cc/gradients/math_grad_test.cc | 10 +++++++++ tensorflow/core/ops/math_ops_test.cc | 3 ++- tensorflow/python/ops/math_grad.py | 18 ++++++++++++++++ tensorflow/python/ops/math_grad_test.py | 25 +++++++++++++++++++++++ 5 files changed, 70 insertions(+), 1 deletion(-) diff --git a/tensorflow/cc/gradients/math_grad.cc b/tensorflow/cc/gradients/math_grad.cc index 52c177212a..ea86fc0a7c 100644 --- a/tensorflow/cc/gradients/math_grad.cc +++ b/tensorflow/cc/gradients/math_grad.cc @@ -440,6 +440,21 @@ Status RealDivGrad(const Scope& scope, const Operation& op, } REGISTER_GRADIENT_OP("RealDiv", RealDivGrad); +Status UnsafeDivGrad(const Scope& scope, const Operation& op, + const std::vector& grad_inputs, + std::vector* grad_outputs) { + auto x_1 = ConjugateHelper(scope, op.input(0)); + auto x_2 = ConjugateHelper(scope, op.input(1)); + // y = x_1 / x_2 + // dy/dx_1 = 1/x_2 + // dy/dx_2 = -x_1/x_2^2 + auto gx_1 = UnsafeDiv(scope, grad_inputs[0], x_2); + auto gx_2 = Mul(scope, grad_inputs[0], + UnsafeDiv(scope, UnsafeDiv(scope, Neg(scope, x_1), x_2), x_2)); + return BinaryGradCommon(scope, op, grad_outputs, gx_1, gx_2); +} +REGISTER_GRADIENT_OP("UnsafeDiv", DivGrad); + Status SquaredDifferenceGrad(const Scope& scope, const Operation& op, const std::vector& grad_inputs, std::vector* grad_outputs) { diff --git a/tensorflow/cc/gradients/math_grad_test.cc b/tensorflow/cc/gradients/math_grad_test.cc index 1b4c7c2688..0cc398abcf 100644 --- a/tensorflow/cc/gradients/math_grad_test.cc +++ b/tensorflow/cc/gradients/math_grad_test.cc @@ -46,6 +46,7 @@ using ops::RealDiv; using ops::SquaredDifference; using ops::Sub; using ops::Sum; +using ops::UnsafeDiv; using ops::Where3; // TODO(andydavis) Test gradient function against numeric gradients output. @@ -856,6 +857,15 @@ TEST_F(NaryGradTest, RealDiv) { RunTest({x}, {x_shape}, {y}, {x_shape}); } +TEST_F(NaryGradTest, UnsafeDiv) { + TensorShape x_shape({3, 2, 5}); + auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(x_shape)); + // Test x / (1 + |x|) rather than x_1 / x_2 to avoid triggering large + // division errors in the numeric estimator used by the gradient checker. + auto y = UnsafeDiv(scope_, x, Add(scope_, Const(scope_, 1), Abs(scope_, x))); + RunTest({x}, {x_shape}, {y}, {x_shape}); +} + TEST_F(NaryGradTest, SquaredDifference) { TensorShape x1_shape({3, 2, 5}); TensorShape x2_shape({2, 5}); diff --git a/tensorflow/core/ops/math_ops_test.cc b/tensorflow/core/ops/math_ops_test.cc index 8f974d5367..33d91ec617 100644 --- a/tensorflow/core/ops/math_ops_test.cc +++ b/tensorflow/core/ops/math_ops_test.cc @@ -120,7 +120,8 @@ TEST(MathOpsTest, BroadcastBinaryOps_ShapeFn) { "Maximum", "Minimum", "Mod", "Mul", "NotEqual", "Pow", - "Sub", "SquaredDifference"}) { + "Sub", "SquaredDifference", + "UnsafeDiv"}) { ShapeInferenceTestOp op(op_name); INFER_OK(op, "?;?", "?"); INFER_OK(op, "[1,2];?", "?"); diff --git a/tensorflow/python/ops/math_grad.py b/tensorflow/python/ops/math_grad.py index 02e07dc7b1..339ae6039f 100644 --- a/tensorflow/python/ops/math_grad.py +++ b/tensorflow/python/ops/math_grad.py @@ -940,6 +940,24 @@ def _RealDivGrad(op, grad): grad * math_ops.realdiv(math_ops.realdiv(-x, y), y), ry), sy)) +@ops.RegisterGradient("UnsafeDiv") +def _UnsafeDivGrad(op, grad): + """UnsafeDiv op gradient.""" + x = op.inputs[0] + y = op.inputs[1] + sx = array_ops.shape(x) + sy = array_ops.shape(y) + rx, ry = gen_array_ops.broadcast_gradient_args(sx, sy) + x = math_ops.conj(x) + y = math_ops.conj(y) + return (array_ops.reshape( + math_ops.reduce_sum(math_ops.unsafe_div(grad, y), rx), sx), + array_ops.reshape( + math_ops.reduce_sum( + grad * math_ops.unsafe_div(math_ops.unsafe_div(-x, y), y), + ry), sy)) + + @ops.RegisterGradient("Pow") def _PowGrad(op, grad): """Returns grad * (y*x^(y-1), z*log(x)).""" diff --git a/tensorflow/python/ops/math_grad_test.py b/tensorflow/python/ops/math_grad_test.py index 04eeb00518..525cbd202c 100644 --- a/tensorflow/python/ops/math_grad_test.py +++ b/tensorflow/python/ops/math_grad_test.py @@ -25,6 +25,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker +from tensorflow.python.ops import gradients from tensorflow.python.ops import math_ops from tensorflow.python.platform import test @@ -208,5 +209,29 @@ class FloorModGradientTest(test.TestCase): self.assertLess(error, 1e-4) +class UnsafeDivGradientTest(test.TestCase): + + def testBasicGradient(self): + inputs = constant_op.constant(np.arange(-3, 3), + dtype=dtypes.float32) + outputs = math_ops.unsafe_div(inputs, 1 + abs(inputs)) + with self.test_session(): + error = gradient_checker.compute_gradient_error( + inputs, inputs.get_shape().as_list(), + outputs, outputs.get_shape().as_list()) + self.assertLess(error, 1e-4) + + def testGradientWithDenominatorIsZero(self): + x = constant_op.constant(np.arange(-3, 3), + dtype=dtypes.float32) + y = array_ops.zeros_like(x, + dtype=dtypes.float32) + outputs = math_ops.unsafe_div(x, y) + with self.test_session(): + dx, dy = gradients.gradients(outputs, [x, y]) + self.assertAllClose(dx.eval(), np.zeros(x.shape.as_list())) + self.assertAllClose(dy.eval(), np.zeros(y.shape.as_list())) + + if __name__ == "__main__": test.main() -- GitLab From 7ce6f5464f82c81383d3c4fddae01ed202902ebe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Sat, 5 May 2018 18:08:34 +0800 Subject: [PATCH 0027/3149] DOC: add document --- .../core/api_def/base_api/api_def_UnsafeDiv.pbtxt | 10 ++++++++++ .../core/api_def/python_api/api_def_UnsafeDiv.pbtxt | 4 ++++ tensorflow/python/ops/math_ops.py | 11 +++++++---- 3 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 tensorflow/core/api_def/base_api/api_def_UnsafeDiv.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_UnsafeDiv.pbtxt diff --git a/tensorflow/core/api_def/base_api/api_def_UnsafeDiv.pbtxt b/tensorflow/core/api_def/base_api/api_def_UnsafeDiv.pbtxt new file mode 100644 index 0000000000..c8115b59a2 --- /dev/null +++ b/tensorflow/core/api_def/base_api/api_def_UnsafeDiv.pbtxt @@ -0,0 +1,10 @@ +op { + graph_op_name: "UnsafeDiv" + visibility: HIDDEN + summary: "Returns 0 if the denominator is zero." + description: <