From 67dee0adc09534483ce2627ffee629feb5133ae7 Mon Sep 17 00:00:00 2001 From: imsheridan Date: Fri, 6 Apr 2018 03:26:26 +0800 Subject: [PATCH 001/911] Fix math equation rendering format in api definitions --- tensorflow/core/api_def/base_api/api_def_Exp.pbtxt | 2 +- .../core/api_def/base_api/api_def_GatherNd.pbtxt | 2 +- .../api_def/base_api/api_def_MatrixExponential.pbtxt | 2 +- .../api_def/base_api/api_def_MatrixLogarithm.pbtxt | 2 +- .../core/api_def/base_api/api_def_Polygamma.pbtxt | 2 +- .../core/api_def/base_api/api_def_ReduceJoin.pbtxt | 2 +- .../core/api_def/base_api/api_def_ScatterNdAdd.pbtxt | 4 ++-- .../base_api/api_def_ScatterNdNonAliasingAdd.pbtxt | 4 ++-- .../core/api_def/base_api/api_def_ScatterNdSub.pbtxt | 4 ++-- .../api_def/base_api/api_def_ScatterNdUpdate.pbtxt | 4 ++-- tensorflow/core/api_def/base_api/api_def_Softmax.pbtxt | 2 +- .../api_def/base_api/api_def_SparseApplyAdagrad.pbtxt | 4 ++-- .../base_api/api_def_SparseApplyCenteredRMSProp.pbtxt | 6 +++--- .../api_def/base_api/api_def_SparseApplyFtrl.pbtxt | 10 +++++----- .../api_def/base_api/api_def_SparseApplyMomentum.pbtxt | 4 ++-- .../base_api/api_def_SparseApplyProximalAdagrad.pbtxt | 8 ++++---- .../api_def_SparseApplyProximalGradientDescent.pbtxt | 4 ++-- .../api_def/base_api/api_def_SparseApplyRMSProp.pbtxt | 6 +++--- .../api_def/base_api/api_def_UnsortedSegmentSum.pbtxt | 2 +- tensorflow/core/api_def/base_api/api_def_Zeta.pbtxt | 2 +- 20 files changed, 38 insertions(+), 38 deletions(-) diff --git a/tensorflow/core/api_def/base_api/api_def_Exp.pbtxt b/tensorflow/core/api_def/base_api/api_def_Exp.pbtxt index dd1e3d5dfc..01ac3d433a 100644 --- a/tensorflow/core/api_def/base_api/api_def_Exp.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_Exp.pbtxt @@ -1,4 +1,4 @@ op { graph_op_name: "Exp" - summary: "Computes exponential of x element-wise. \\\\(y = e^x\\\\)." + summary: "Computes exponential of x element-wise. \\(y = e^x\\)." } diff --git a/tensorflow/core/api_def/base_api/api_def_GatherNd.pbtxt b/tensorflow/core/api_def/base_api/api_def_GatherNd.pbtxt index 6cd76ff340..342a1f6b05 100644 --- a/tensorflow/core/api_def/base_api/api_def_GatherNd.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_GatherNd.pbtxt @@ -25,7 +25,7 @@ END (K-1)-dimensional tensor of indices into `params`, where each element defines a slice of `params`: - output[i_0, ..., i_{K-2}] = params[indices[i0, ..., i_{K-2}]] + output[\\(i_0, ..., i_{K-2}\\)] = params[indices[\\(i_0, ..., i_{K-2}\\)]] Whereas in @{tf.gather} `indices` defines slices into the first dimension of `params`, in `tf.gather_nd`, `indices` defines slices into the diff --git a/tensorflow/core/api_def/base_api/api_def_MatrixExponential.pbtxt b/tensorflow/core/api_def/base_api/api_def_MatrixExponential.pbtxt index 0d680f6531..d7b56aec87 100644 --- a/tensorflow/core/api_def/base_api/api_def_MatrixExponential.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_MatrixExponential.pbtxt @@ -18,7 +18,7 @@ END } summary: "Computes the matrix exponential of one or more square matrices:" description: < l1 else 0.0 -accum = accum_new +$$accum_new = accum + grad * grad$$ +$$linear += grad + (accum_{new}^{-lr_{power}} - accum^{-lr_{power}} / lr * var$$ +$$quadratic = 1.0 / (accum_{new}^{lr_{power}} * lr) + 2 * l2$$ +$$var = (sign(linear) * l1 - linear) / quadratic\ if\ |linear| > l1\ else\ 0.0$$ +$$accum = accum_{new}$$ END } diff --git a/tensorflow/core/api_def/base_api/api_def_SparseApplyMomentum.pbtxt b/tensorflow/core/api_def/base_api/api_def_SparseApplyMomentum.pbtxt index 8d9ac9ea3f..17dbb488de 100644 --- a/tensorflow/core/api_def/base_api/api_def_SparseApplyMomentum.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_SparseApplyMomentum.pbtxt @@ -64,7 +64,7 @@ Set use_nesterov = True if you want to use Nesterov momentum. That is for rows we have grad for, we update var and accum as follows: -accum = accum * momentum + grad -var -= lr * accum +$$accum = accum * momentum + grad$$ +$$var -= lr * accum$$ END } diff --git a/tensorflow/core/api_def/base_api/api_def_SparseApplyProximalAdagrad.pbtxt b/tensorflow/core/api_def/base_api/api_def_SparseApplyProximalAdagrad.pbtxt index 80541b91c7..0b24f2ddd1 100644 --- a/tensorflow/core/api_def/base_api/api_def_SparseApplyProximalAdagrad.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_SparseApplyProximalAdagrad.pbtxt @@ -58,9 +58,9 @@ END summary: "Sparse update entries in \'*var\' and \'*accum\' according to FOBOS algorithm." description: < Date: Tue, 10 Apr 2018 21:10:51 +0800 Subject: [PATCH 002/911] Remove breaking ``` for math equations --- tensorflow/core/api_def/base_api/api_def_ScatterNdAdd.pbtxt | 2 -- .../api_def/base_api/api_def_ScatterNdNonAliasingAdd.pbtxt | 4 +--- tensorflow/core/api_def/base_api/api_def_ScatterNdSub.pbtxt | 4 +--- .../core/api_def/base_api/api_def_ScatterNdUpdate.pbtxt | 4 +--- .../core/api_def/base_api/api_def_UnsortedSegmentSum.pbtxt | 2 +- 5 files changed, 4 insertions(+), 12 deletions(-) diff --git a/tensorflow/core/api_def/base_api/api_def_ScatterNdAdd.pbtxt b/tensorflow/core/api_def/base_api/api_def_ScatterNdAdd.pbtxt index ee0578c2ec..a9a7646314 100644 --- a/tensorflow/core/api_def/base_api/api_def_ScatterNdAdd.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_ScatterNdAdd.pbtxt @@ -50,9 +50,7 @@ dimension of `ref`. `updates` is `Tensor` of rank `Q-1+P-K` with shape: -``` $$[d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]].$$ -``` For example, say we want to add 4 scattered elements to a rank-1 tensor to 8 elements. In Python, that addition would look like this: diff --git a/tensorflow/core/api_def/base_api/api_def_ScatterNdNonAliasingAdd.pbtxt b/tensorflow/core/api_def/base_api/api_def_ScatterNdNonAliasingAdd.pbtxt index 1e4f99006a..35116e5f6a 100644 --- a/tensorflow/core/api_def/base_api/api_def_ScatterNdNonAliasingAdd.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_ScatterNdNonAliasingAdd.pbtxt @@ -37,7 +37,7 @@ respect to both `input` and `updates`. `input` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`. `indices` must be integer tensor, containing indices into `input`. -It must be shape `\\([d_0, ..., d_{Q-2}, K]\\)` where `0 < K <= P`. +It must be shape \\([d_0, ..., d_{Q-2}, K]\\) where `0 < K <= P`. The innermost dimension of `indices` (with length `K`) corresponds to indices into elements (if `K = P`) or `(P-K)`-dimensional slices @@ -45,9 +45,7 @@ indices into elements (if `K = P`) or `(P-K)`-dimensional slices `updates` is `Tensor` of rank `Q-1+P-K` with shape: -``` $$[d_0, ..., d_{Q-2}, input.shape[K], ..., input.shape[P-1]].$$ -``` For example, say we want to add 4 scattered elements to a rank-1 tensor to 8 elements. In Python, that addition would look like this: diff --git a/tensorflow/core/api_def/base_api/api_def_ScatterNdSub.pbtxt b/tensorflow/core/api_def/base_api/api_def_ScatterNdSub.pbtxt index e8fdd71785..99e5c4908b 100644 --- a/tensorflow/core/api_def/base_api/api_def_ScatterNdSub.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_ScatterNdSub.pbtxt @@ -42,7 +42,7 @@ within a given variable according to `indices`. `ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`. `indices` must be integer tensor, containing indices into `ref`. -It must be shape `\\([d_0, ..., d_{Q-2}, K]\\)` where `0 < K <= P`. +It must be shape \\([d_0, ..., d_{Q-2}, K]\\) where `0 < K <= P`. The innermost dimension of `indices` (with length `K`) corresponds to indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th @@ -50,9 +50,7 @@ dimension of `ref`. `updates` is `Tensor` of rank `Q-1+P-K` with shape: -``` $$[d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]].$$ -``` For example, say we want to subtract 4 scattered elements from a rank-1 tensor with 8 elements. In Python, that subtraction would look like this: diff --git a/tensorflow/core/api_def/base_api/api_def_ScatterNdUpdate.pbtxt b/tensorflow/core/api_def/base_api/api_def_ScatterNdUpdate.pbtxt index 556a5d559b..cb57c171b9 100644 --- a/tensorflow/core/api_def/base_api/api_def_ScatterNdUpdate.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_ScatterNdUpdate.pbtxt @@ -42,7 +42,7 @@ variable according to `indices`. `ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`. `indices` must be integer tensor, containing indices into `ref`. -It must be shape `\\([d_0, ..., d_{Q-2}, K]\\)` where `0 < K <= P`. +It must be shape \\([d_0, ..., d_{Q-2}, K]\\) where `0 < K <= P`. The innermost dimension of `indices` (with length `K`) corresponds to indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th @@ -50,9 +50,7 @@ dimension of `ref`. `updates` is `Tensor` of rank `Q-1+P-K` with shape: -``` $$[d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]].$$ -``` For example, say we want to update 4 scattered elements to a rank-1 tensor to 8 elements. In Python, that update would look like this: diff --git a/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentSum.pbtxt b/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentSum.pbtxt index ac1499346c..9aeabd030d 100644 --- a/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentSum.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentSum.pbtxt @@ -20,7 +20,7 @@ Read @{$math_ops#Segmentation$the section on segmentation} for an explanation of segments. Computes a tensor such that -`\\(output[i] = sum_{j...} data[j...]\\)` where the sum is over tuples `j...` such +\\(output[i] = sum_{j...} data[j...]\\) where the sum is over tuples `j...` such that `segment_ids[j...] == i`. Unlike `SegmentSum`, `segment_ids` need not be sorted and need not cover all values in the full range of valid values. -- GitLab From 8dc3b3c453180211f4be5302f957664004e1ec04 Mon Sep 17 00:00:00 2001 From: apantykhin Date: Mon, 16 Apr 2018 20:40:51 +0400 Subject: [PATCH 003/911] add checking for input values in GANHead constructor --- .../gan/python/estimator/python/head_impl.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/gan/python/estimator/python/head_impl.py b/tensorflow/contrib/gan/python/estimator/python/head_impl.py index a21358c50b..652ffee30a 100644 --- a/tensorflow/contrib/gan/python/estimator/python/head_impl.py +++ b/tensorflow/contrib/gan/python/estimator/python/head_impl.py @@ -25,6 +25,7 @@ from tensorflow.contrib.gan.python import train as tfgan_train from tensorflow.python.estimator import model_fn as model_fn_lib from tensorflow.python.estimator.canned import head from tensorflow.python.framework import ops +from tensorflow.python.training import optimizer __all__ = [ 'GANHead', @@ -90,9 +91,24 @@ class GANHead(head._Head): # pylint: disable=protected-access name: name of the head. If provided, summary and metrics keys will be suffixed by `"/" + name`. """ + + if not callable(generator_loss_fn): + raise TypeError('generator_loss_fn must be callable.') + if not callable(discriminator_loss_fn): + raise TypeError('discriminator_loss_fn must be callable.') + if not isinstance(generator_optimizer, optimizer.Optimizer): + raise TypeError('generator_optimizer must be Optimizer.') + if not isinstance(discriminator_optimizer, optimizer.Optimizer): + raise TypeError('discriminator_optimizer must be Optimizer.') + if not use_loss_summaries in [True, False, None]: + raise ValueError('use_loss_summaries must be True, False or None.') + if get_hooks_fn is not None and not callable(get_hooks_fn): + raise TypeError('get_hooks_fn must be callable.') + if name is not None and not isinstance(name, str): + raise TypeError('name must be string.') + if get_hooks_fn is None: get_hooks_fn = tfgan_train.get_sequential_train_hooks() - # TODO(joelshor): Validate inputs. if use_loss_summaries in [True, False]: generator_loss_fn = functools.partial( -- GitLab From 6583c9a693b122a49f17e7ec99463c6c3b7dbe98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Tue, 24 Apr 2018 20:22:24 +0800 Subject: [PATCH 004/911] CLN: move _safe_embedding_lookup_sparse to embedding_ops and make it public --- tensorflow/python/BUILD | 2 + .../python/feature_column/feature_column.py | 161 +----------------- tensorflow/python/ops/embedding_ops.py | 157 +++++++++++++++++ tensorflow/python/ops/nn.py | 1 + 4 files changed, 163 insertions(+), 158 deletions(-) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index bb32f4bbe0..6e2e546984 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -1845,6 +1845,8 @@ py_library( ":math_ops", ":platform", ":resource_variable_ops", + ":sparse_ops", + ":tensor_shape", ":variables", ], ) diff --git a/tensorflow/python/feature_column/feature_column.py b/tensorflow/python/feature_column/feature_column.py index c16c3cda48..f48634d0c7 100644 --- a/tensorflow/python/feature_column/feature_column.py +++ b/tensorflow/python/feature_column/feature_column.py @@ -2058,7 +2058,7 @@ def _create_categorical_column_weighted_sum(column, initializer=init_ops.zeros_initializer(), trainable=trainable, collections=weight_collections) - return _safe_embedding_lookup_sparse( + return embedding_ops.safe_embedding_lookup_sparse( weight, id_tensor, sparse_weights=weight_tensor, @@ -2479,7 +2479,7 @@ class _EmbeddingColumn( }) # Return embedding lookup result. - return _safe_embedding_lookup_sparse( + return embedding_ops.safe_embedding_lookup_sparse( embedding_weights=embedding_weights, sparse_ids=sparse_ids, sparse_weights=sparse_weights, @@ -2612,7 +2612,7 @@ class _SharedEmbeddingColumn( }) # Return embedding lookup result. - return _safe_embedding_lookup_sparse( + return embedding_ops.safe_embedding_lookup_sparse( embedding_weights=embedding_weights, sparse_ids=sparse_ids, sparse_weights=sparse_weights, @@ -3065,161 +3065,6 @@ def _collect_leaf_level_keys(cross): return leaf_level_keys -# TODO(zakaria): Move this to embedding_ops and make it public. -def _safe_embedding_lookup_sparse(embedding_weights, - sparse_ids, - sparse_weights=None, - combiner='mean', - default_id=None, - name=None, - partition_strategy='div', - max_norm=None): - """Lookup embedding results, accounting for invalid IDs and empty features. - - The partitioned embedding in `embedding_weights` must all be the same shape - except for the first dimension. The first dimension is allowed to vary as the - vocabulary size is not necessarily a multiple of `P`. `embedding_weights` - may be a `PartitionedVariable` as returned by using `tf.get_variable()` with a - partitioner. - - Invalid IDs (< 0) are pruned from input IDs and weights, as well as any IDs - with non-positive weight. For an entry with no features, the embedding vector - for `default_id` is returned, or the 0-vector if `default_id` is not supplied. - - The ids and weights may be multi-dimensional. Embeddings are always aggregated - along the last dimension. - - Args: - embedding_weights: A list of `P` float `Tensor`s or values representing - partitioned embedding `Tensor`s. Alternatively, a `PartitionedVariable` - created by partitioning along dimension 0. The total unpartitioned - shape should be `[e_0, e_1, ..., e_m]`, where `e_0` represents the - vocab size and `e_1, ..., e_m` are the embedding dimensions. - sparse_ids: `SparseTensor` of shape `[d_0, d_1, ..., d_n]` containing the - ids. `d_0` is typically batch size. - sparse_weights: `SparseTensor` of same shape as `sparse_ids`, containing - float weights corresponding to `sparse_ids`, or `None` if all weights - are be assumed to be 1.0. - combiner: A string specifying how to combine embedding results for each - entry. Currently "mean", "sqrtn" and "sum" are supported, with "mean" - the default. - default_id: The id to use for an entry with no features. - name: A name for this operation (optional). - partition_strategy: A string specifying the partitioning strategy. - Currently `"div"` and `"mod"` are supported. Default is `"div"`. - max_norm: If not `None`, all embeddings are l2-normalized to max_norm before - combining. - - - Returns: - Dense `Tensor` of shape `[d_0, d_1, ..., d_{n-1}, e_1, ..., e_m]`. - - Raises: - ValueError: if `embedding_weights` is empty. - """ - if embedding_weights is None: - raise ValueError('Missing embedding_weights %s.' % embedding_weights) - if isinstance(embedding_weights, variables.PartitionedVariable): - embedding_weights = list(embedding_weights) # get underlying Variables. - if not isinstance(embedding_weights, list): - embedding_weights = [embedding_weights] - if len(embedding_weights) < 1: - raise ValueError('Missing embedding_weights %s.' % embedding_weights) - - dtype = sparse_weights.dtype if sparse_weights is not None else None - embedding_weights = [ - ops.convert_to_tensor(w, dtype=dtype) for w in embedding_weights - ] - - with ops.name_scope(name, 'embedding_lookup', - embedding_weights + [sparse_ids, - sparse_weights]) as scope: - # Reshape higher-rank sparse ids and weights to linear segment ids. - original_shape = sparse_ids.dense_shape - original_rank_dim = sparse_ids.dense_shape.get_shape()[0] - original_rank = ( - array_ops.size(original_shape) - if original_rank_dim.value is None - else original_rank_dim.value) - sparse_ids = sparse_ops.sparse_reshape(sparse_ids, [ - math_ops.reduce_prod( - array_ops.slice(original_shape, [0], [original_rank - 1])), - array_ops.gather(original_shape, original_rank - 1)]) - if sparse_weights is not None: - sparse_weights = sparse_tensor_lib.SparseTensor( - sparse_ids.indices, - sparse_weights.values, sparse_ids.dense_shape) - - # Prune invalid ids and weights. - sparse_ids, sparse_weights = _prune_invalid_ids(sparse_ids, sparse_weights) - if combiner != 'sum': - sparse_ids, sparse_weights = _prune_invalid_weights( - sparse_ids, sparse_weights) - - # Fill in dummy values for empty features, if necessary. - sparse_ids, is_row_empty = sparse_ops.sparse_fill_empty_rows(sparse_ids, - default_id or - 0) - if sparse_weights is not None: - sparse_weights, _ = sparse_ops.sparse_fill_empty_rows(sparse_weights, 1.0) - - result = embedding_ops.embedding_lookup_sparse( - embedding_weights, - sparse_ids, - sparse_weights, - combiner=combiner, - partition_strategy=partition_strategy, - name=None if default_id is None else scope, - max_norm=max_norm) - - if default_id is None: - # Broadcast is_row_empty to the same shape as embedding_lookup_result, - # for use in Select. - is_row_empty = array_ops.tile( - array_ops.reshape(is_row_empty, [-1, 1]), - array_ops.stack([1, array_ops.shape(result)[1]])) - - result = array_ops.where(is_row_empty, - array_ops.zeros_like(result), - result, - name=scope) - - # Reshape back from linear ids back into higher-dimensional dense result. - final_result = array_ops.reshape( - result, - array_ops.concat([ - array_ops.slice( - math_ops.cast(original_shape, dtypes.int32), [0], - [original_rank - 1]), - array_ops.slice(array_ops.shape(result), [1], [-1]) - ], 0)) - final_result.set_shape(tensor_shape.unknown_shape( - (original_rank_dim - 1).value).concatenate(result.get_shape()[1:])) - return final_result - - -def _prune_invalid_ids(sparse_ids, sparse_weights): - """Prune invalid IDs (< 0) from the input ids and weights.""" - is_id_valid = math_ops.greater_equal(sparse_ids.values, 0) - if sparse_weights is not None: - is_id_valid = math_ops.logical_and( - is_id_valid, - array_ops.ones_like(sparse_weights.values, dtype=dtypes.bool)) - sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_id_valid) - if sparse_weights is not None: - sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_id_valid) - return sparse_ids, sparse_weights - - -def _prune_invalid_weights(sparse_ids, sparse_weights): - """Prune invalid weights (< 0) from the input ids and weights.""" - if sparse_weights is not None: - is_weights_valid = math_ops.greater(sparse_weights.values, 0) - sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_weights_valid) - sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_weights_valid) - return sparse_ids, sparse_weights - - class _IndicatorColumn(_DenseColumn, _SequenceDenseColumn, collections.namedtuple('_IndicatorColumn', ['categorical_column'])): diff --git a/tensorflow/python/ops/embedding_ops.py b/tensorflow/python/ops/embedding_ops.py index 9e46739bc1..a8cfeca119 100644 --- a/tensorflow/python/ops/embedding_ops.py +++ b/tensorflow/python/ops/embedding_ops.py @@ -23,6 +23,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import array_ops from tensorflow.python.ops import clip_ops # Imports gradient definitions. @@ -30,6 +31,7 @@ from tensorflow.python.ops import data_flow_grad # pylint: disable=unused-impor from tensorflow.python.ops import data_flow_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import sparse_ops from tensorflow.python.ops import variables from tensorflow.python.platform import tf_logging as logging from tensorflow.python.util.tf_export import tf_export @@ -480,3 +482,158 @@ def embedding_lookup_sparse(params, assert False, "Unrecognized combiner" return embeddings + + +@tf_export("nn.safe_embedding_lookup_sparse") +def safe_embedding_lookup_sparse(embedding_weights, + sparse_ids, + sparse_weights=None, + combiner='mean', + default_id=None, + name=None, + partition_strategy='div', + max_norm=None): + """Lookup embedding results, accounting for invalid IDs and empty features. + + The partitioned embedding in `embedding_weights` must all be the same shape + except for the first dimension. The first dimension is allowed to vary as the + vocabulary size is not necessarily a multiple of `P`. `embedding_weights` + may be a `PartitionedVariable` as returned by using `tf.get_variable()` with a + partitioner. + + Invalid IDs (< 0) are pruned from input IDs and weights, as well as any IDs + with non-positive weight. For an entry with no features, the embedding vector + for `default_id` is returned, or the 0-vector if `default_id` is not supplied. + + The ids and weights may be multi-dimensional. Embeddings are always aggregated + along the last dimension. + + Args: + embedding_weights: A list of `P` float `Tensor`s or values representing + partitioned embedding `Tensor`s. Alternatively, a `PartitionedVariable` + created by partitioning along dimension 0. The total unpartitioned + shape should be `[e_0, e_1, ..., e_m]`, where `e_0` represents the + vocab size and `e_1, ..., e_m` are the embedding dimensions. + sparse_ids: `SparseTensor` of shape `[d_0, d_1, ..., d_n]` containing the + ids. `d_0` is typically batch size. + sparse_weights: `SparseTensor` of same shape as `sparse_ids`, containing + float weights corresponding to `sparse_ids`, or `None` if all weights + are be assumed to be 1.0. + combiner: A string specifying how to combine embedding results for each + entry. Currently "mean", "sqrtn" and "sum" are supported, with "mean" + the default. + default_id: The id to use for an entry with no features. + name: A name for this operation (optional). + partition_strategy: A string specifying the partitioning strategy. + Currently `"div"` and `"mod"` are supported. Default is `"div"`. + max_norm: If not `None`, all embeddings are l2-normalized to max_norm before + combining. + + + Returns: + Dense `Tensor` of shape `[d_0, d_1, ..., d_{n-1}, e_1, ..., e_m]`. + + Raises: + ValueError: if `embedding_weights` is empty. + """ + if embedding_weights is None: + raise ValueError('Missing embedding_weights %s.' % embedding_weights) + if isinstance(embedding_weights, variables.PartitionedVariable): + embedding_weights = list(embedding_weights) # get underlying Variables. + if not isinstance(embedding_weights, list): + embedding_weights = [embedding_weights] + if len(embedding_weights) < 1: + raise ValueError('Missing embedding_weights %s.' % embedding_weights) + + dtype = sparse_weights.dtype if sparse_weights is not None else None + embedding_weights = [ + ops.convert_to_tensor(w, dtype=dtype) for w in embedding_weights + ] + + with ops.name_scope(name, 'embedding_lookup', + embedding_weights + [sparse_ids, + sparse_weights]) as scope: + # Reshape higher-rank sparse ids and weights to linear segment ids. + original_shape = sparse_ids.dense_shape + original_rank_dim = sparse_ids.dense_shape.get_shape()[0] + original_rank = ( + array_ops.size(original_shape) + if original_rank_dim.value is None + else original_rank_dim.value) + sparse_ids = sparse_ops.sparse_reshape(sparse_ids, [ + math_ops.reduce_prod( + array_ops.slice(original_shape, [0], [original_rank - 1])), + array_ops.gather(original_shape, original_rank - 1)]) + if sparse_weights is not None: + sparse_weights = sparse_tensor.SparseTensor( + sparse_ids.indices, + sparse_weights.values, sparse_ids.dense_shape) + + # Prune invalid ids and weights. + sparse_ids, sparse_weights = _prune_invalid_ids(sparse_ids, sparse_weights) + if combiner != 'sum': + sparse_ids, sparse_weights = _prune_invalid_weights( + sparse_ids, sparse_weights) + + # Fill in dummy values for empty features, if necessary. + sparse_ids, is_row_empty = sparse_ops.sparse_fill_empty_rows(sparse_ids, + default_id or + 0) + if sparse_weights is not None: + sparse_weights, _ = sparse_ops.sparse_fill_empty_rows(sparse_weights, 1.0) + + result = embedding_lookup_sparse( + embedding_weights, + sparse_ids, + sparse_weights, + combiner=combiner, + partition_strategy=partition_strategy, + name=None if default_id is None else scope, + max_norm=max_norm) + + if default_id is None: + # Broadcast is_row_empty to the same shape as embedding_lookup_result, + # for use in Select. + is_row_empty = array_ops.tile( + array_ops.reshape(is_row_empty, [-1, 1]), + array_ops.stack([1, array_ops.shape(result)[1]])) + + result = array_ops.where(is_row_empty, + array_ops.zeros_like(result), + result, + name=scope) + + # Reshape back from linear ids back into higher-dimensional dense result. + final_result = array_ops.reshape( + result, + array_ops.concat([ + array_ops.slice( + math_ops.cast(original_shape, dtypes.int32), [0], + [original_rank - 1]), + array_ops.slice(array_ops.shape(result), [1], [-1]) + ], 0)) + final_result.set_shape(tensor_shape.unknown_shape( + (original_rank_dim - 1).value).concatenate(result.get_shape()[1:])) + return final_result + + +def _prune_invalid_ids(sparse_ids, sparse_weights): + """Prune invalid IDs (< 0) from the input ids and weights.""" + is_id_valid = math_ops.greater_equal(sparse_ids.values, 0) + if sparse_weights is not None: + is_id_valid = math_ops.logical_and( + is_id_valid, + array_ops.ones_like(sparse_weights.values, dtype=dtypes.bool)) + sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_id_valid) + if sparse_weights is not None: + sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_id_valid) + return sparse_ids, sparse_weights + + +def _prune_invalid_weights(sparse_ids, sparse_weights): + """Prune invalid weights (< 0) from the input ids and weights.""" + if sparse_weights is not None: + is_weights_valid = math_ops.greater(sparse_weights.values, 0) + sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_weights_valid) + sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_weights_valid) + return sparse_ids, sparse_weights diff --git a/tensorflow/python/ops/nn.py b/tensorflow/python/ops/nn.py index 1d0d9a52a1..fb896bf042 100644 --- a/tensorflow/python/ops/nn.py +++ b/tensorflow/python/ops/nn.py @@ -79,6 +79,7 @@ See the @{$python/nn} guide. @@weighted_cross_entropy_with_logits @@embedding_lookup @@embedding_lookup_sparse +@@safe_embedding_lookup_sparse @@dynamic_rnn @@bidirectional_dynamic_rnn @@raw_rnn -- GitLab From 608508c35a4b87a17b9f07364e6fbeae2fa948c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Tue, 24 Apr 2018 20:32:38 +0800 Subject: [PATCH 005/911] CLN: move the corresponding test case --- .../python/layers/embedding_ops_test.py | 1 - tensorflow/python/kernel_tests/BUILD | 1 + .../python/kernel_tests/embedding_ops_test.py | 218 ++++++++++++++++++ 3 files changed, 219 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py index bf25144982..87f00f94a6 100644 --- a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py +++ b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py @@ -21,7 +21,6 @@ from __future__ import print_function import itertools import math -import sys import numpy as np diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index ba8f1fd3ca..2fbdde849b 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -2717,6 +2717,7 @@ cuda_py_test( "//tensorflow/python:embedding_ops", "//tensorflow/python:framework", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:init_ops", "//tensorflow/python:linalg_ops", "//tensorflow/python:math_ops", "//tensorflow/python:partitioned_variables", diff --git a/tensorflow/python/kernel_tests/embedding_ops_test.py b/tensorflow/python/kernel_tests/embedding_ops_test.py index e53ca1dcaa..55d75cb474 100644 --- a/tensorflow/python/kernel_tests/embedding_ops_test.py +++ b/tensorflow/python/kernel_tests/embedding_ops_test.py @@ -19,6 +19,7 @@ from __future__ import division from __future__ import print_function import itertools +import math import numpy as np from six.moves import xrange # pylint: disable=redefined-builtin @@ -31,6 +32,7 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import data_flow_ops from tensorflow.python.ops import embedding_ops from tensorflow.python.ops import gradient_checker +from tensorflow.python.ops import init_ops from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import partitioned_variables @@ -736,6 +738,222 @@ class EmbeddingLookupSparseTest(test.TestCase): x, sp_ids, sp_weights, combiner="mean") +class SafeEmbeddingLookupSparseTest(test.TestCase): + + def _random_weights(self, vocab_size=4, embed_dim=4, num_shards=1): + assert vocab_size > 0 + assert embed_dim > 0 + assert num_shards > 0 + assert num_shards <= vocab_size + + embedding_weights = partitioned_variables.create_partitioned_variables( + shape=[vocab_size, embed_dim], + slicing=[num_shards, 1], + initializer=init_ops.truncated_normal_initializer( + mean=0.0, stddev=1.0 / math.sqrt(vocab_size), dtype=dtypes.float32)) + for w in embedding_weights: + w.initializer.run() + embedding_weights = [w.eval() for w in embedding_weights] + return embedding_weights + + def _ids_and_weights_2d(self): + # Each row demonstrates a test case: + # Row 0: multiple valid ids, 1 invalid id, weighted mean + # Row 1: all ids are invalid (leaving no valid ids after pruning) + # Row 2: no ids to begin with + # Row 3: single id + # Row 4: all ids have <=0 weight + indices = [[0, 0], [0, 1], [0, 2], [1, 0], [3, 0], [4, 0], [4, 1]] + ids = [0, 1, -1, -1, 2, 0, 1] + weights = [1.0, 2.0, 1.0, 1.0, 3.0, 0.0, -0.5] + shape = [5, 4] + + sparse_ids = sparse_tensor.SparseTensor( + constant_op.constant(indices, dtypes.int64), + constant_op.constant(ids, dtypes.int64), + constant_op.constant(shape, dtypes.int64)) + + sparse_weights = sparse_tensor.SparseTensor( + constant_op.constant(indices, dtypes.int64), + constant_op.constant(weights, dtypes.float32), + constant_op.constant(shape, dtypes.int64)) + + return sparse_ids, sparse_weights + + def _ids_and_weights_3d(self): + # Each (2-D) index demonstrates a test case: + # Index 0, 0: multiple valid ids, 1 invalid id, weighted mean + # Index 0, 1: all ids are invalid (leaving no valid ids after pruning) + # Index 0, 2: no ids to begin with + # Index 1, 0: single id + # Index 1, 1: all ids have <=0 weight + # Index 1, 2: no ids to begin with + indices = [[0, 0, 0], [0, 0, 1], [0, 0, 2], [0, 1, 0], [1, 0, 0], [1, 1, 0], + [1, 1, 1]] + ids = [0, 1, -1, -1, 2, 0, 1] + weights = [1.0, 2.0, 1.0, 1.0, 3.0, 0.0, -0.5] + shape = [2, 3, 4] + + sparse_ids = sparse_tensor.SparseTensor( + constant_op.constant(indices, dtypes.int64), + constant_op.constant(ids, dtypes.int64), + constant_op.constant(shape, dtypes.int64)) + + sparse_weights = sparse_tensor.SparseTensor( + constant_op.constant(indices, dtypes.int64), + constant_op.constant(weights, dtypes.float32), + constant_op.constant(shape, dtypes.int64)) + + return sparse_ids, sparse_weights + + def test_safe_embedding_lookup_sparse_return_zero_vector(self): + with self.test_session(): + embedding_weights = self._random_weights() + sparse_ids, sparse_weights = self._ids_and_weights_2d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, sparse_weights).eval()) + + self.assertAllClose( + embedding_lookup_result, + [(1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / + 3.0, [0] * 4, [0] * 4, embedding_weights[0][2], [0] * 4]) + + def test_safe_embedding_lookup_sparse_return_special_vector(self): + with self.test_session(): + embedding_weights = self._random_weights() + sparse_ids, sparse_weights = self._ids_and_weights_2d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, sparse_weights, default_id=3).eval()) + + self.assertAllClose( + embedding_lookup_result, + [(1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / + 3.0, embedding_weights[0][3], embedding_weights[0][3], + embedding_weights[0][2], embedding_weights[0][3]]) + + def test_safe_embedding_lookup_sparse_no_weights(self): + with self.test_session(): + embedding_weights = self._random_weights() + sparse_ids, _ = self._ids_and_weights_2d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, None).eval()) + + self.assertAllClose( + embedding_lookup_result, + [(embedding_weights[0][0] + embedding_weights[0][1]) / 2.0, [0] * 4, + [0] * 4, embedding_weights[0][2], ( + embedding_weights[0][0] + embedding_weights[0][1]) / 2.0]) + + def test_safe_embedding_lookup_sparse_partitioned(self): + with self.test_session(): + embedding_weights = self._random_weights(num_shards=3) + sparse_ids, _ = self._ids_and_weights_2d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, None).eval()) + + embedding_weights = list(itertools.chain(*embedding_weights)) + self.assertAllClose(embedding_lookup_result, + [(embedding_weights[0] + embedding_weights[1]) / 2.0, + [0] * 4, [0] * 4, embedding_weights[2], + (embedding_weights[0] + embedding_weights[1]) / 2.0]) + + def test_safe_embedding_lookup_sparse_partitioned_inconsistent_weights(self): + with self.test_session(): + embedding_weights = self._random_weights(num_shards=3) + sparse_ids, sparse_weights = self._ids_and_weights_2d() + + embedding_weights[1] = embedding_weights[1].astype(np.float64) + self.assertRaises(TypeError, embedding_ops.safe_embedding_lookup_sparse, + embedding_weights, sparse_ids) + embedding_weights = [ + constant_op.constant(w, dtype=dtypes.float64) + for w in embedding_weights + ] + self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, + embedding_weights, sparse_ids, sparse_weights) + + def test_safe_embedding_lookup_sparse_3d_return_zero_vector(self): + with self.test_session(): + embedding_weights = self._random_weights() + sparse_ids, sparse_weights = self._ids_and_weights_3d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, sparse_weights).eval()) + + self.assertAllClose(embedding_lookup_result, [[ + (1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / 3.0, + [0] * 4, [0] * 4 + ], [embedding_weights[0][2], [0] * 4, [0] * 4]]) + + def test_safe_embedding_lookup_sparse_3d_return_special_vector(self): + with self.test_session(): + embedding_weights = self._random_weights() + sparse_ids, sparse_weights = self._ids_and_weights_3d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, sparse_weights, default_id=3).eval()) + + self.assertAllClose( + embedding_lookup_result, + [[(1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / + 3.0, embedding_weights[0][3], embedding_weights[0][3]], [ + embedding_weights[0][2], embedding_weights[0][3], + embedding_weights[0][3] + ]]) + + def test_safe_embedding_lookup_sparse_3d_no_weights(self): + with self.test_session(): + embedding_weights = self._random_weights() + sparse_ids, _ = self._ids_and_weights_3d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, None).eval()) + + self.assertAllClose(embedding_lookup_result, [[( + embedding_weights[0][0] + embedding_weights[0][1]) / 2.0, [0] * 4, [ + 0 + ] * 4], [ + embedding_weights[0][2], + (embedding_weights[0][0] + embedding_weights[0][1]) / 2.0, [0] * 4 + ]]) + + def test_safe_embedding_lookup_sparse_3d_partitioned(self): + with self.test_session(): + embedding_weights = self._random_weights(num_shards=3) + sparse_ids, _ = self._ids_and_weights_3d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, None).eval()) + + embedding_weights = list(itertools.chain(*embedding_weights)) + self.assertAllClose(embedding_lookup_result, [[ + (embedding_weights[0] + embedding_weights[1]) / 2.0, [0] * 4, [0] * 4 + ], [ + embedding_weights[2], + (embedding_weights[0] + embedding_weights[1]) / 2.0, [0] * 4 + ]]) + + def test_safe_embedding_lookup_sparse_3d_partitioned_inconsistent_weights( + self): + with self.test_session(): + embedding_weights = self._random_weights(num_shards=3) + sparse_ids, sparse_weights = self._ids_and_weights_3d() + + embedding_weights[1] = embedding_weights[1].astype(np.float64) + self.assertRaises(TypeError, embedding_ops.safe_embedding_lookup_sparse, + embedding_weights, sparse_ids) + embedding_weights = [ + constant_op.constant(w, dtype=dtypes.float64) + for w in embedding_weights + ] + self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, + embedding_weights, sparse_ids, sparse_weights) + + class DynamicStitchOpTest(test.TestCase): def testCint32Cpu(self): -- GitLab From 067c85fb66345e61aee9428cd645cca786ed2bf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Tue, 24 Apr 2018 20:52:33 +0800 Subject: [PATCH 006/911] CLN: delete duplicate codes --- .../layers/python/layers/embedding_ops.py | 117 ++---------------- .../python/layers/embedding_ops_test.py | 4 +- 2 files changed, 11 insertions(+), 110 deletions(-) diff --git a/tensorflow/contrib/layers/python/layers/embedding_ops.py b/tensorflow/contrib/layers/python/layers/embedding_ops.py index 49c3faf3b7..4353bf9c28 100644 --- a/tensorflow/contrib/layers/python/layers/embedding_ops.py +++ b/tensorflow/contrib/layers/python/layers/embedding_ops.py @@ -19,14 +19,12 @@ from __future__ import print_function from six.moves import xrange # pylint: disable=redefined-builtin -from tensorflow.contrib.framework.python.framework import tensor_util as contrib_tensor_util from tensorflow.contrib.layers.python.ops import sparse_feature_cross_op from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import array_ops from tensorflow.python.ops import clip_ops from tensorflow.python.ops import control_flow_ops @@ -100,112 +98,15 @@ def safe_embedding_lookup_sparse(embedding_weights, logging.warn("The default value of combiner will change from \"mean\" " "to \"sqrtn\" after 2016/11/01.") combiner = "mean" - if embedding_weights is None: - raise ValueError("Missing embedding_weights %s." % embedding_weights) - if isinstance(embedding_weights, variables.PartitionedVariable): - embedding_weights = list(embedding_weights) # get underlying Variables. - if not isinstance(embedding_weights, list): - embedding_weights = [embedding_weights] - if len(embedding_weights) < 1: - raise ValueError("Missing embedding_weights %s." % embedding_weights) - - dtype = sparse_weights.dtype if sparse_weights is not None else None - if isinstance(embedding_weights, variables.PartitionedVariable): - embedding_weights = list(embedding_weights) - embedding_weights = [ - ops.convert_to_tensor(w, dtype=dtype) for w in embedding_weights - ] - - contrib_tensor_util.assert_same_float_dtype(embedding_weights + - [sparse_weights]) - - with ops.name_scope(name, "embedding_lookup", - embedding_weights + [sparse_ids, - sparse_weights]) as scope: - # Reshape higher-rank sparse ids and weights to linear segment ids. - original_shape = sparse_ids.dense_shape - original_rank_dim = sparse_ids.dense_shape.get_shape()[0] - original_rank = ( - array_ops.size(original_shape) - if original_rank_dim.value is None - else original_rank_dim.value) - sparse_ids = sparse_ops.sparse_reshape(sparse_ids, [ - math_ops.reduce_prod( - array_ops.slice(original_shape, [0], [original_rank - 1])), - array_ops.gather(original_shape, original_rank - 1)]) - if sparse_weights is not None: - sparse_weights = sparse_tensor.SparseTensor( - sparse_ids.indices, - sparse_weights.values, sparse_ids.dense_shape) - - # Prune invalid ids and weights. - sparse_ids, sparse_weights = _prune_invalid_ids(sparse_ids, sparse_weights) - if combiner != "sum": - sparse_ids, sparse_weights = _prune_invalid_weights( - sparse_ids, sparse_weights) - - # Fill in dummy values for empty features, if necessary. - sparse_ids, is_row_empty = sparse_ops.sparse_fill_empty_rows(sparse_ids, - default_id or - 0) - if sparse_weights is not None: - sparse_weights, _ = sparse_ops.sparse_fill_empty_rows(sparse_weights, 1.0) - - result = embedding_ops.embedding_lookup_sparse( - embedding_weights, - sparse_ids, - sparse_weights, - combiner=combiner, - partition_strategy=partition_strategy, - name=None if default_id is None else scope, - max_norm=max_norm) - - if default_id is None: - # Broadcast is_row_empty to the same shape as embedding_lookup_result, - # for use in Select. - is_row_empty = array_ops.tile( - array_ops.reshape(is_row_empty, [-1, 1]), - array_ops.stack([1, array_ops.shape(result)[1]])) - - result = array_ops.where(is_row_empty, - array_ops.zeros_like(result), - result, - name=scope) - - # Reshape back from linear ids back into higher-dimensional dense result. - final_result = array_ops.reshape( - result, - array_ops.concat([ - array_ops.slice( - math_ops.cast(original_shape, dtypes.int32), [0], - [original_rank - 1]), - array_ops.slice(array_ops.shape(result), [1], [-1]) - ], 0)) - final_result.set_shape(tensor_shape.unknown_shape( - (original_rank_dim - 1).value).concatenate(result.get_shape()[1:])) - return final_result - - -def _prune_invalid_ids(sparse_ids, sparse_weights): - """Prune invalid IDs (< 0) from the input ids and weights.""" - is_id_valid = math_ops.greater_equal(sparse_ids.values, 0) - if sparse_weights is not None: - is_id_valid = math_ops.logical_and( - is_id_valid, - array_ops.ones_like(sparse_weights.values, dtype=dtypes.bool)) - sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_id_valid) - if sparse_weights is not None: - sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_id_valid) - return sparse_ids, sparse_weights - - -def _prune_invalid_weights(sparse_ids, sparse_weights): - """Prune invalid weights (< 0) from the input ids and weights.""" - if sparse_weights is not None: - is_weights_valid = math_ops.greater(sparse_weights.values, 0) - sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_weights_valid) - sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_weights_valid) - return sparse_ids, sparse_weights + return embedding_ops.safe_embedding_lookup_sparse( + embedding_weights=embedding_weights, + sparse_ids=sparse_ids, + sparse_weights=sparse_weights, + combiner=combiner, + default_id=default_id, + name=name, + partition_strategy=partition_strategy, + max_norm=max_norm) def scattered_embedding_lookup(params, diff --git a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py index 87f00f94a6..4d9849b4b1 100644 --- a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py +++ b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py @@ -168,7 +168,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase): sparse_ids, sparse_weights = self._ids_and_weights_2d() embedding_weights[1] = embedding_weights[1].astype(np.float64) - self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, + self.assertRaises(TypeError, embedding_ops.safe_embedding_lookup_sparse, embedding_weights, sparse_ids) embedding_weights = [ constant_op.constant(w, dtype=dtypes.float64) @@ -245,7 +245,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase): sparse_ids, sparse_weights = self._ids_and_weights_3d() embedding_weights[1] = embedding_weights[1].astype(np.float64) - self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, + self.assertRaises(TypeError, embedding_ops.safe_embedding_lookup_sparse, embedding_weights, sparse_ids) embedding_weights = [ constant_op.constant(w, dtype=dtypes.float64) -- GitLab From 91ad552a52242b3d382eee6a3382c79be36b7df7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Tue, 24 Apr 2018 20:57:47 +0800 Subject: [PATCH 007/911] CLN: delete test cases in contrib --- .../python/layers/embedding_ops_test.py | 217 ------------------ 1 file changed, 217 deletions(-) diff --git a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py index 4d9849b4b1..f7b7ade39d 100644 --- a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py +++ b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py @@ -20,7 +20,6 @@ from __future__ import division from __future__ import print_function import itertools -import math import numpy as np @@ -39,222 +38,6 @@ from tensorflow.python.platform import test from tensorflow.python.util import compat -class SafeEmbeddingLookupSparseTest(test.TestCase): - - def _random_weights(self, vocab_size=4, embed_dim=4, num_shards=1): - assert vocab_size > 0 - assert embed_dim > 0 - assert num_shards > 0 - assert num_shards <= vocab_size - - embedding_weights = partitioned_variables.create_partitioned_variables( - shape=[vocab_size, embed_dim], - slicing=[num_shards, 1], - initializer=init_ops.truncated_normal_initializer( - mean=0.0, stddev=1.0 / math.sqrt(vocab_size), dtype=dtypes.float32)) - for w in embedding_weights: - w.initializer.run() - embedding_weights = [w.eval() for w in embedding_weights] - return embedding_weights - - def _ids_and_weights_2d(self): - # Each row demonstrates a test case: - # Row 0: multiple valid ids, 1 invalid id, weighted mean - # Row 1: all ids are invalid (leaving no valid ids after pruning) - # Row 2: no ids to begin with - # Row 3: single id - # Row 4: all ids have <=0 weight - indices = [[0, 0], [0, 1], [0, 2], [1, 0], [3, 0], [4, 0], [4, 1]] - ids = [0, 1, -1, -1, 2, 0, 1] - weights = [1.0, 2.0, 1.0, 1.0, 3.0, 0.0, -0.5] - shape = [5, 4] - - sparse_ids = sparse_tensor_lib.SparseTensor( - constant_op.constant(indices, dtypes.int64), - constant_op.constant(ids, dtypes.int64), - constant_op.constant(shape, dtypes.int64)) - - sparse_weights = sparse_tensor_lib.SparseTensor( - constant_op.constant(indices, dtypes.int64), - constant_op.constant(weights, dtypes.float32), - constant_op.constant(shape, dtypes.int64)) - - return sparse_ids, sparse_weights - - def _ids_and_weights_3d(self): - # Each (2-D) index demonstrates a test case: - # Index 0, 0: multiple valid ids, 1 invalid id, weighted mean - # Index 0, 1: all ids are invalid (leaving no valid ids after pruning) - # Index 0, 2: no ids to begin with - # Index 1, 0: single id - # Index 1, 1: all ids have <=0 weight - # Index 1, 2: no ids to begin with - indices = [[0, 0, 0], [0, 0, 1], [0, 0, 2], [0, 1, 0], [1, 0, 0], [1, 1, 0], - [1, 1, 1]] - ids = [0, 1, -1, -1, 2, 0, 1] - weights = [1.0, 2.0, 1.0, 1.0, 3.0, 0.0, -0.5] - shape = [2, 3, 4] - - sparse_ids = sparse_tensor_lib.SparseTensor( - constant_op.constant(indices, dtypes.int64), - constant_op.constant(ids, dtypes.int64), - constant_op.constant(shape, dtypes.int64)) - - sparse_weights = sparse_tensor_lib.SparseTensor( - constant_op.constant(indices, dtypes.int64), - constant_op.constant(weights, dtypes.float32), - constant_op.constant(shape, dtypes.int64)) - - return sparse_ids, sparse_weights - - def test_safe_embedding_lookup_sparse_return_zero_vector(self): - with self.test_session(): - embedding_weights = self._random_weights() - sparse_ids, sparse_weights = self._ids_and_weights_2d() - - embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( - embedding_weights, sparse_ids, sparse_weights).eval()) - - self.assertAllClose( - embedding_lookup_result, - [(1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / - 3.0, [0] * 4, [0] * 4, embedding_weights[0][2], [0] * 4]) - - def test_safe_embedding_lookup_sparse_return_special_vector(self): - with self.test_session(): - embedding_weights = self._random_weights() - sparse_ids, sparse_weights = self._ids_and_weights_2d() - - embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( - embedding_weights, sparse_ids, sparse_weights, default_id=3).eval()) - - self.assertAllClose( - embedding_lookup_result, - [(1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / - 3.0, embedding_weights[0][3], embedding_weights[0][3], - embedding_weights[0][2], embedding_weights[0][3]]) - - def test_safe_embedding_lookup_sparse_no_weights(self): - with self.test_session(): - embedding_weights = self._random_weights() - sparse_ids, _ = self._ids_and_weights_2d() - - embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( - embedding_weights, sparse_ids, None).eval()) - - self.assertAllClose( - embedding_lookup_result, - [(embedding_weights[0][0] + embedding_weights[0][1]) / 2.0, [0] * 4, - [0] * 4, embedding_weights[0][2], ( - embedding_weights[0][0] + embedding_weights[0][1]) / 2.0]) - - def test_safe_embedding_lookup_sparse_partitioned(self): - with self.test_session(): - embedding_weights = self._random_weights(num_shards=3) - sparse_ids, _ = self._ids_and_weights_2d() - - embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( - embedding_weights, sparse_ids, None).eval()) - - embedding_weights = list(itertools.chain(*embedding_weights)) - self.assertAllClose(embedding_lookup_result, - [(embedding_weights[0] + embedding_weights[1]) / 2.0, - [0] * 4, [0] * 4, embedding_weights[2], - (embedding_weights[0] + embedding_weights[1]) / 2.0]) - - def test_safe_embedding_lookup_sparse_partitioned_inconsistent_weights(self): - with self.test_session(): - embedding_weights = self._random_weights(num_shards=3) - sparse_ids, sparse_weights = self._ids_and_weights_2d() - - embedding_weights[1] = embedding_weights[1].astype(np.float64) - self.assertRaises(TypeError, embedding_ops.safe_embedding_lookup_sparse, - embedding_weights, sparse_ids) - embedding_weights = [ - constant_op.constant(w, dtype=dtypes.float64) - for w in embedding_weights - ] - self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, - embedding_weights, sparse_ids, sparse_weights) - - def test_safe_embedding_lookup_sparse_3d_return_zero_vector(self): - with self.test_session(): - embedding_weights = self._random_weights() - sparse_ids, sparse_weights = self._ids_and_weights_3d() - - embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( - embedding_weights, sparse_ids, sparse_weights).eval()) - - self.assertAllClose(embedding_lookup_result, [[ - (1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / 3.0, - [0] * 4, [0] * 4 - ], [embedding_weights[0][2], [0] * 4, [0] * 4]]) - - def test_safe_embedding_lookup_sparse_3d_return_special_vector(self): - with self.test_session(): - embedding_weights = self._random_weights() - sparse_ids, sparse_weights = self._ids_and_weights_3d() - - embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( - embedding_weights, sparse_ids, sparse_weights, default_id=3).eval()) - - self.assertAllClose( - embedding_lookup_result, - [[(1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / - 3.0, embedding_weights[0][3], embedding_weights[0][3]], [ - embedding_weights[0][2], embedding_weights[0][3], - embedding_weights[0][3] - ]]) - - def test_safe_embedding_lookup_sparse_3d_no_weights(self): - with self.test_session(): - embedding_weights = self._random_weights() - sparse_ids, _ = self._ids_and_weights_3d() - - embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( - embedding_weights, sparse_ids, None).eval()) - - self.assertAllClose(embedding_lookup_result, [[( - embedding_weights[0][0] + embedding_weights[0][1]) / 2.0, [0] * 4, [ - 0 - ] * 4], [ - embedding_weights[0][2], - (embedding_weights[0][0] + embedding_weights[0][1]) / 2.0, [0] * 4 - ]]) - - def test_safe_embedding_lookup_sparse_3d_partitioned(self): - with self.test_session(): - embedding_weights = self._random_weights(num_shards=3) - sparse_ids, _ = self._ids_and_weights_3d() - - embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( - embedding_weights, sparse_ids, None).eval()) - - embedding_weights = list(itertools.chain(*embedding_weights)) - self.assertAllClose(embedding_lookup_result, [[ - (embedding_weights[0] + embedding_weights[1]) / 2.0, [0] * 4, [0] * 4 - ], [ - embedding_weights[2], - (embedding_weights[0] + embedding_weights[1]) / 2.0, [0] * 4 - ]]) - - def test_safe_embedding_lookup_sparse_3d_partitioned_inconsistent_weights( - self): - with self.test_session(): - embedding_weights = self._random_weights(num_shards=3) - sparse_ids, sparse_weights = self._ids_and_weights_3d() - - embedding_weights[1] = embedding_weights[1].astype(np.float64) - self.assertRaises(TypeError, embedding_ops.safe_embedding_lookup_sparse, - embedding_weights, sparse_ids) - embedding_weights = [ - constant_op.constant(w, dtype=dtypes.float64) - for w in embedding_weights - ] - self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, - embedding_weights, sparse_ids, sparse_weights) - - class ScatteredEmbeddingLookupTest(test.TestCase): def setUp(self): -- GitLab From 24a6350ad173865c16351825f251f2fde97b7d9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Thu, 26 Apr 2018 07:01:06 +0800 Subject: [PATCH 008/911] Revert "CLN: delete test cases in contrib" This reverts commit 91ad552a52242b3d382eee6a3382c79be36b7df7. --- .../python/layers/embedding_ops_test.py | 217 ++++++++++++++++++ 1 file changed, 217 insertions(+) diff --git a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py index f7b7ade39d..4d9849b4b1 100644 --- a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py +++ b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py @@ -20,6 +20,7 @@ from __future__ import division from __future__ import print_function import itertools +import math import numpy as np @@ -38,6 +39,222 @@ from tensorflow.python.platform import test from tensorflow.python.util import compat +class SafeEmbeddingLookupSparseTest(test.TestCase): + + def _random_weights(self, vocab_size=4, embed_dim=4, num_shards=1): + assert vocab_size > 0 + assert embed_dim > 0 + assert num_shards > 0 + assert num_shards <= vocab_size + + embedding_weights = partitioned_variables.create_partitioned_variables( + shape=[vocab_size, embed_dim], + slicing=[num_shards, 1], + initializer=init_ops.truncated_normal_initializer( + mean=0.0, stddev=1.0 / math.sqrt(vocab_size), dtype=dtypes.float32)) + for w in embedding_weights: + w.initializer.run() + embedding_weights = [w.eval() for w in embedding_weights] + return embedding_weights + + def _ids_and_weights_2d(self): + # Each row demonstrates a test case: + # Row 0: multiple valid ids, 1 invalid id, weighted mean + # Row 1: all ids are invalid (leaving no valid ids after pruning) + # Row 2: no ids to begin with + # Row 3: single id + # Row 4: all ids have <=0 weight + indices = [[0, 0], [0, 1], [0, 2], [1, 0], [3, 0], [4, 0], [4, 1]] + ids = [0, 1, -1, -1, 2, 0, 1] + weights = [1.0, 2.0, 1.0, 1.0, 3.0, 0.0, -0.5] + shape = [5, 4] + + sparse_ids = sparse_tensor_lib.SparseTensor( + constant_op.constant(indices, dtypes.int64), + constant_op.constant(ids, dtypes.int64), + constant_op.constant(shape, dtypes.int64)) + + sparse_weights = sparse_tensor_lib.SparseTensor( + constant_op.constant(indices, dtypes.int64), + constant_op.constant(weights, dtypes.float32), + constant_op.constant(shape, dtypes.int64)) + + return sparse_ids, sparse_weights + + def _ids_and_weights_3d(self): + # Each (2-D) index demonstrates a test case: + # Index 0, 0: multiple valid ids, 1 invalid id, weighted mean + # Index 0, 1: all ids are invalid (leaving no valid ids after pruning) + # Index 0, 2: no ids to begin with + # Index 1, 0: single id + # Index 1, 1: all ids have <=0 weight + # Index 1, 2: no ids to begin with + indices = [[0, 0, 0], [0, 0, 1], [0, 0, 2], [0, 1, 0], [1, 0, 0], [1, 1, 0], + [1, 1, 1]] + ids = [0, 1, -1, -1, 2, 0, 1] + weights = [1.0, 2.0, 1.0, 1.0, 3.0, 0.0, -0.5] + shape = [2, 3, 4] + + sparse_ids = sparse_tensor_lib.SparseTensor( + constant_op.constant(indices, dtypes.int64), + constant_op.constant(ids, dtypes.int64), + constant_op.constant(shape, dtypes.int64)) + + sparse_weights = sparse_tensor_lib.SparseTensor( + constant_op.constant(indices, dtypes.int64), + constant_op.constant(weights, dtypes.float32), + constant_op.constant(shape, dtypes.int64)) + + return sparse_ids, sparse_weights + + def test_safe_embedding_lookup_sparse_return_zero_vector(self): + with self.test_session(): + embedding_weights = self._random_weights() + sparse_ids, sparse_weights = self._ids_and_weights_2d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, sparse_weights).eval()) + + self.assertAllClose( + embedding_lookup_result, + [(1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / + 3.0, [0] * 4, [0] * 4, embedding_weights[0][2], [0] * 4]) + + def test_safe_embedding_lookup_sparse_return_special_vector(self): + with self.test_session(): + embedding_weights = self._random_weights() + sparse_ids, sparse_weights = self._ids_and_weights_2d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, sparse_weights, default_id=3).eval()) + + self.assertAllClose( + embedding_lookup_result, + [(1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / + 3.0, embedding_weights[0][3], embedding_weights[0][3], + embedding_weights[0][2], embedding_weights[0][3]]) + + def test_safe_embedding_lookup_sparse_no_weights(self): + with self.test_session(): + embedding_weights = self._random_weights() + sparse_ids, _ = self._ids_and_weights_2d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, None).eval()) + + self.assertAllClose( + embedding_lookup_result, + [(embedding_weights[0][0] + embedding_weights[0][1]) / 2.0, [0] * 4, + [0] * 4, embedding_weights[0][2], ( + embedding_weights[0][0] + embedding_weights[0][1]) / 2.0]) + + def test_safe_embedding_lookup_sparse_partitioned(self): + with self.test_session(): + embedding_weights = self._random_weights(num_shards=3) + sparse_ids, _ = self._ids_and_weights_2d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, None).eval()) + + embedding_weights = list(itertools.chain(*embedding_weights)) + self.assertAllClose(embedding_lookup_result, + [(embedding_weights[0] + embedding_weights[1]) / 2.0, + [0] * 4, [0] * 4, embedding_weights[2], + (embedding_weights[0] + embedding_weights[1]) / 2.0]) + + def test_safe_embedding_lookup_sparse_partitioned_inconsistent_weights(self): + with self.test_session(): + embedding_weights = self._random_weights(num_shards=3) + sparse_ids, sparse_weights = self._ids_and_weights_2d() + + embedding_weights[1] = embedding_weights[1].astype(np.float64) + self.assertRaises(TypeError, embedding_ops.safe_embedding_lookup_sparse, + embedding_weights, sparse_ids) + embedding_weights = [ + constant_op.constant(w, dtype=dtypes.float64) + for w in embedding_weights + ] + self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, + embedding_weights, sparse_ids, sparse_weights) + + def test_safe_embedding_lookup_sparse_3d_return_zero_vector(self): + with self.test_session(): + embedding_weights = self._random_weights() + sparse_ids, sparse_weights = self._ids_and_weights_3d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, sparse_weights).eval()) + + self.assertAllClose(embedding_lookup_result, [[ + (1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / 3.0, + [0] * 4, [0] * 4 + ], [embedding_weights[0][2], [0] * 4, [0] * 4]]) + + def test_safe_embedding_lookup_sparse_3d_return_special_vector(self): + with self.test_session(): + embedding_weights = self._random_weights() + sparse_ids, sparse_weights = self._ids_and_weights_3d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, sparse_weights, default_id=3).eval()) + + self.assertAllClose( + embedding_lookup_result, + [[(1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / + 3.0, embedding_weights[0][3], embedding_weights[0][3]], [ + embedding_weights[0][2], embedding_weights[0][3], + embedding_weights[0][3] + ]]) + + def test_safe_embedding_lookup_sparse_3d_no_weights(self): + with self.test_session(): + embedding_weights = self._random_weights() + sparse_ids, _ = self._ids_and_weights_3d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, None).eval()) + + self.assertAllClose(embedding_lookup_result, [[( + embedding_weights[0][0] + embedding_weights[0][1]) / 2.0, [0] * 4, [ + 0 + ] * 4], [ + embedding_weights[0][2], + (embedding_weights[0][0] + embedding_weights[0][1]) / 2.0, [0] * 4 + ]]) + + def test_safe_embedding_lookup_sparse_3d_partitioned(self): + with self.test_session(): + embedding_weights = self._random_weights(num_shards=3) + sparse_ids, _ = self._ids_and_weights_3d() + + embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse( + embedding_weights, sparse_ids, None).eval()) + + embedding_weights = list(itertools.chain(*embedding_weights)) + self.assertAllClose(embedding_lookup_result, [[ + (embedding_weights[0] + embedding_weights[1]) / 2.0, [0] * 4, [0] * 4 + ], [ + embedding_weights[2], + (embedding_weights[0] + embedding_weights[1]) / 2.0, [0] * 4 + ]]) + + def test_safe_embedding_lookup_sparse_3d_partitioned_inconsistent_weights( + self): + with self.test_session(): + embedding_weights = self._random_weights(num_shards=3) + sparse_ids, sparse_weights = self._ids_and_weights_3d() + + embedding_weights[1] = embedding_weights[1].astype(np.float64) + self.assertRaises(TypeError, embedding_ops.safe_embedding_lookup_sparse, + embedding_weights, sparse_ids) + embedding_weights = [ + constant_op.constant(w, dtype=dtypes.float64) + for w in embedding_weights + ] + self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, + embedding_weights, sparse_ids, sparse_weights) + + class ScatteredEmbeddingLookupTest(test.TestCase): def setUp(self): -- GitLab From 1c1b4d47707a439c157b5dcf3755e391730a328c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Thu, 26 Apr 2018 07:01:21 +0800 Subject: [PATCH 009/911] Revert "CLN: delete duplicate codes" This reverts commit 067c85fb66345e61aee9428cd645cca786ed2bf4. --- .../layers/python/layers/embedding_ops.py | 117 ++++++++++++++++-- .../python/layers/embedding_ops_test.py | 4 +- 2 files changed, 110 insertions(+), 11 deletions(-) diff --git a/tensorflow/contrib/layers/python/layers/embedding_ops.py b/tensorflow/contrib/layers/python/layers/embedding_ops.py index 4353bf9c28..49c3faf3b7 100644 --- a/tensorflow/contrib/layers/python/layers/embedding_ops.py +++ b/tensorflow/contrib/layers/python/layers/embedding_ops.py @@ -19,12 +19,14 @@ from __future__ import print_function from six.moves import xrange # pylint: disable=redefined-builtin +from tensorflow.contrib.framework.python.framework import tensor_util as contrib_tensor_util from tensorflow.contrib.layers.python.ops import sparse_feature_cross_op from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import array_ops from tensorflow.python.ops import clip_ops from tensorflow.python.ops import control_flow_ops @@ -98,15 +100,112 @@ def safe_embedding_lookup_sparse(embedding_weights, logging.warn("The default value of combiner will change from \"mean\" " "to \"sqrtn\" after 2016/11/01.") combiner = "mean" - return embedding_ops.safe_embedding_lookup_sparse( - embedding_weights=embedding_weights, - sparse_ids=sparse_ids, - sparse_weights=sparse_weights, - combiner=combiner, - default_id=default_id, - name=name, - partition_strategy=partition_strategy, - max_norm=max_norm) + if embedding_weights is None: + raise ValueError("Missing embedding_weights %s." % embedding_weights) + if isinstance(embedding_weights, variables.PartitionedVariable): + embedding_weights = list(embedding_weights) # get underlying Variables. + if not isinstance(embedding_weights, list): + embedding_weights = [embedding_weights] + if len(embedding_weights) < 1: + raise ValueError("Missing embedding_weights %s." % embedding_weights) + + dtype = sparse_weights.dtype if sparse_weights is not None else None + if isinstance(embedding_weights, variables.PartitionedVariable): + embedding_weights = list(embedding_weights) + embedding_weights = [ + ops.convert_to_tensor(w, dtype=dtype) for w in embedding_weights + ] + + contrib_tensor_util.assert_same_float_dtype(embedding_weights + + [sparse_weights]) + + with ops.name_scope(name, "embedding_lookup", + embedding_weights + [sparse_ids, + sparse_weights]) as scope: + # Reshape higher-rank sparse ids and weights to linear segment ids. + original_shape = sparse_ids.dense_shape + original_rank_dim = sparse_ids.dense_shape.get_shape()[0] + original_rank = ( + array_ops.size(original_shape) + if original_rank_dim.value is None + else original_rank_dim.value) + sparse_ids = sparse_ops.sparse_reshape(sparse_ids, [ + math_ops.reduce_prod( + array_ops.slice(original_shape, [0], [original_rank - 1])), + array_ops.gather(original_shape, original_rank - 1)]) + if sparse_weights is not None: + sparse_weights = sparse_tensor.SparseTensor( + sparse_ids.indices, + sparse_weights.values, sparse_ids.dense_shape) + + # Prune invalid ids and weights. + sparse_ids, sparse_weights = _prune_invalid_ids(sparse_ids, sparse_weights) + if combiner != "sum": + sparse_ids, sparse_weights = _prune_invalid_weights( + sparse_ids, sparse_weights) + + # Fill in dummy values for empty features, if necessary. + sparse_ids, is_row_empty = sparse_ops.sparse_fill_empty_rows(sparse_ids, + default_id or + 0) + if sparse_weights is not None: + sparse_weights, _ = sparse_ops.sparse_fill_empty_rows(sparse_weights, 1.0) + + result = embedding_ops.embedding_lookup_sparse( + embedding_weights, + sparse_ids, + sparse_weights, + combiner=combiner, + partition_strategy=partition_strategy, + name=None if default_id is None else scope, + max_norm=max_norm) + + if default_id is None: + # Broadcast is_row_empty to the same shape as embedding_lookup_result, + # for use in Select. + is_row_empty = array_ops.tile( + array_ops.reshape(is_row_empty, [-1, 1]), + array_ops.stack([1, array_ops.shape(result)[1]])) + + result = array_ops.where(is_row_empty, + array_ops.zeros_like(result), + result, + name=scope) + + # Reshape back from linear ids back into higher-dimensional dense result. + final_result = array_ops.reshape( + result, + array_ops.concat([ + array_ops.slice( + math_ops.cast(original_shape, dtypes.int32), [0], + [original_rank - 1]), + array_ops.slice(array_ops.shape(result), [1], [-1]) + ], 0)) + final_result.set_shape(tensor_shape.unknown_shape( + (original_rank_dim - 1).value).concatenate(result.get_shape()[1:])) + return final_result + + +def _prune_invalid_ids(sparse_ids, sparse_weights): + """Prune invalid IDs (< 0) from the input ids and weights.""" + is_id_valid = math_ops.greater_equal(sparse_ids.values, 0) + if sparse_weights is not None: + is_id_valid = math_ops.logical_and( + is_id_valid, + array_ops.ones_like(sparse_weights.values, dtype=dtypes.bool)) + sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_id_valid) + if sparse_weights is not None: + sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_id_valid) + return sparse_ids, sparse_weights + + +def _prune_invalid_weights(sparse_ids, sparse_weights): + """Prune invalid weights (< 0) from the input ids and weights.""" + if sparse_weights is not None: + is_weights_valid = math_ops.greater(sparse_weights.values, 0) + sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_weights_valid) + sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_weights_valid) + return sparse_ids, sparse_weights def scattered_embedding_lookup(params, diff --git a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py index 4d9849b4b1..87f00f94a6 100644 --- a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py +++ b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py @@ -168,7 +168,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase): sparse_ids, sparse_weights = self._ids_and_weights_2d() embedding_weights[1] = embedding_weights[1].astype(np.float64) - self.assertRaises(TypeError, embedding_ops.safe_embedding_lookup_sparse, + self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, embedding_weights, sparse_ids) embedding_weights = [ constant_op.constant(w, dtype=dtypes.float64) @@ -245,7 +245,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase): sparse_ids, sparse_weights = self._ids_and_weights_3d() embedding_weights[1] = embedding_weights[1].astype(np.float64) - self.assertRaises(TypeError, embedding_ops.safe_embedding_lookup_sparse, + self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, embedding_weights, sparse_ids) embedding_weights = [ constant_op.constant(w, dtype=dtypes.float64) -- GitLab From f2a0bc58db70cc792649672b81317288c4151ebb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Thu, 26 Apr 2018 20:52:03 +0800 Subject: [PATCH 010/911] BLD: update golden --- tensorflow/tools/api/golden/tensorflow.nn.pbtxt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tensorflow/tools/api/golden/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.pbtxt index 455590d866..d9e5b0d0fc 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.pbtxt @@ -260,6 +260,10 @@ tf_module { name: "relu_layer" argspec: "args=[\'x\', \'weights\', \'biases\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "safe_embedding_lookup_sparse" + argspec: "args=[\'embedding_weights\', \'sparse_ids\', \'sparse_weights\', \'combiner\', \'default_id\', \'name\', \'partition_strategy\', \'max_norm\'], varargs=None, keywords=None, defaults=[\'None\', \'mean\', \'None\', \'None\', \'div\', \'None\'], " + } member_method { name: "sampled_softmax_loss" argspec: "args=[\'weights\', \'biases\', \'labels\', \'inputs\', \'num_sampled\', \'num_classes\', \'num_true\', \'sampled_values\', \'remove_accidental_hits\', \'partition_strategy\', \'name\', \'seed\'], varargs=None, keywords=None, defaults=[\'1\', \'None\', \'True\', \'mod\', \'sampled_softmax_loss\', \'None\'], " -- GitLab From 05e4d01dd5db5125969b29405bbf9c4eea4a0cd3 Mon Sep 17 00:00:00 2001 From: nrstott Date: Fri, 18 May 2018 11:15:21 -0400 Subject: [PATCH 011/911] accept pd.DataFrame as y for pandas_input_fn --- .../python/estimator/inputs/pandas_io.py | 15 ++++++++--- .../python/estimator/inputs/pandas_io_test.py | 25 +++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/estimator/inputs/pandas_io.py b/tensorflow/python/estimator/inputs/pandas_io.py index bd06843021..abf3f33519 100644 --- a/tensorflow/python/estimator/inputs/pandas_io.py +++ b/tensorflow/python/estimator/inputs/pandas_io.py @@ -50,7 +50,7 @@ def pandas_input_fn(x, Args: x: pandas `DataFrame` object. - y: pandas `Series` object. `None` if absent. + y: pandas `Series` object or `DataFrame`. `None` if absent. batch_size: int, size of batches to return. num_epochs: int, number of epochs to iterate over data. If not `None`, read attempts that would exceed this value will raise `OutOfRangeError`. @@ -87,7 +87,13 @@ def pandas_input_fn(x, if not np.array_equal(x.index, y.index): raise ValueError('Index for x and y are mismatched.\nIndex for x: %s\n' 'Index for y: %s\n' % (x.index, y.index)) - x[target_column] = y + if isinstance(y, pd.DataFrame): + target_column = list(y) + print(target_column) + x[target_column] = y + print(x) + else: + x[target_column] = y # TODO(mdan): These are memory copies. We probably don't need 4x slack space. # The sizes below are consistent with what I've seen elsewhere. @@ -117,7 +123,10 @@ def pandas_input_fn(x, features = features[1:] features = dict(zip(list(x.columns), features)) if y is not None: - target = features.pop(target_column) + if isinstance(target_column, list): + target = {column: features.pop(column) for column in target_column} + else: + target = features.pop(target_column) return features, target return features return input_fn diff --git a/tensorflow/python/estimator/inputs/pandas_io_test.py b/tensorflow/python/estimator/inputs/pandas_io_test.py index e5912a3b28..f4970f07b3 100644 --- a/tensorflow/python/estimator/inputs/pandas_io_test.py +++ b/tensorflow/python/estimator/inputs/pandas_io_test.py @@ -47,6 +47,16 @@ class PandasIoTest(test.TestCase): y = pd.Series(np.arange(-32, -28), index=index) return x, y + def makeTestDataFrameWithYAsDataFrame(self): + index = np.arange(100, 104) + a = np.arange(4) + b = np.arange(32, 36) + a_label = np.arange(10, 14) + b_label = np.arange(50, 54) + x = pd.DataFrame({'a': a, 'b': b}, index=index) + y = pd.DataFrame({'a_target': a_label, 'b_target': b_label}, index=index) + return x, y + def callInputFnOnce(self, input_fn, session): results = input_fn() coord = coordinator.Coordinator() @@ -89,6 +99,21 @@ class PandasIoTest(test.TestCase): self.assertAllEqual(features['b'], [32, 33]) self.assertAllEqual(target, [-32, -31]) + def testPandasInputFnWhenYIsDataFrame_ProducesExpectedOutput(self): + if not HAS_PANDAS: + return + with self.test_session() as session: + x, y = self.makeTestDataFrameWithYAsDataFrame() + input_fn = pandas_io.pandas_input_fn( + x, y, batch_size=2, shuffle=False, num_epochs=1) + + features, targets = self.callInputFnOnce(input_fn, session) + + self.assertAllEqual(features['a'], [0, 1]) + self.assertAllEqual(features['b'], [32, 33]) + self.assertAllEqual(targets['a_target'], [10, 11]) + self.assertAllEqual(targets['b_target'], [50, 51]) + def testPandasInputFn_ProducesOutputsForLargeBatchAndMultipleEpochs(self): if not HAS_PANDAS: return -- GitLab From 45fb10adbde00a82af4576e0de01a5012b0b1ad8 Mon Sep 17 00:00:00 2001 From: nrstott Date: Fri, 18 May 2018 12:01:03 -0400 Subject: [PATCH 012/911] handle overlapping columns in pandas_input_fn when y is df --- .../python/estimator/inputs/pandas_io.py | 25 ++++++++++++++++--- .../python/estimator/inputs/pandas_io_test.py | 16 ++++++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/estimator/inputs/pandas_io.py b/tensorflow/python/estimator/inputs/pandas_io.py index abf3f33519..6918683ce7 100644 --- a/tensorflow/python/estimator/inputs/pandas_io.py +++ b/tensorflow/python/estimator/inputs/pandas_io.py @@ -35,6 +35,22 @@ except ImportError: HAS_PANDAS = False +def _get_unique_target_key(features, target_column_name): + """Returns a key that does not exist in the input DataFrame `features`. + + Args: + features: DataFrame + target_column_name: Name of the target column as a `str` + + Returns: + A unique key that can be used to insert the target into + features. + """ + while target_column_name in features: + target_column_name += '_n' + return target_column_name + + @tf_export('estimator.inputs.pandas_input_fn') def pandas_input_fn(x, y=None, @@ -88,10 +104,9 @@ def pandas_input_fn(x, raise ValueError('Index for x and y are mismatched.\nIndex for x: %s\n' 'Index for y: %s\n' % (x.index, y.index)) if isinstance(y, pd.DataFrame): - target_column = list(y) - print(target_column) + y_columns = [(column, _get_unique_target_key(x, column)) for column in list(y)] + target_column = [v for _, v in y_columns] x[target_column] = y - print(x) else: x[target_column] = y @@ -124,7 +139,9 @@ def pandas_input_fn(x, features = dict(zip(list(x.columns), features)) if y is not None: if isinstance(target_column, list): - target = {column: features.pop(column) for column in target_column} + keys = [k for k, _ in y_columns] + values = [features.pop(column) for column in target_column] + target = {k: v for k, v in zip(keys, values)} else: target = features.pop(target_column) return features, target diff --git a/tensorflow/python/estimator/inputs/pandas_io_test.py b/tensorflow/python/estimator/inputs/pandas_io_test.py index f4970f07b3..f8546abb8a 100644 --- a/tensorflow/python/estimator/inputs/pandas_io_test.py +++ b/tensorflow/python/estimator/inputs/pandas_io_test.py @@ -114,6 +114,22 @@ class PandasIoTest(test.TestCase): self.assertAllEqual(targets['a_target'], [10, 11]) self.assertAllEqual(targets['b_target'], [50, 51]) + def testPandasInputFnWhenYIsDataFrame_HandlesOverlappingColumnNames(self): + if not HAS_PANDAS: + return + with self.test_session() as session: + x, y = self.makeTestDataFrameWithYAsDataFrame() + y = y.rename(columns={'a_target': 'a', 'b_target': 'b'}) + input_fn = pandas_io.pandas_input_fn( + x, y, batch_size=2, shuffle=False, num_epochs=1) + + features, targets = self.callInputFnOnce(input_fn, session) + + self.assertAllEqual(features['a'], [0, 1]) + self.assertAllEqual(features['b'], [32, 33]) + self.assertAllEqual(targets['a'], [10, 11]) + self.assertAllEqual(targets['b'], [50, 51]) + def testPandasInputFn_ProducesOutputsForLargeBatchAndMultipleEpochs(self): if not HAS_PANDAS: return -- GitLab From a611e3afdf1dad0ed485af6cf4b8aa94de744511 Mon Sep 17 00:00:00 2001 From: Guozhong Zhuang Date: Fri, 18 May 2018 11:32:50 -0700 Subject: [PATCH 013/911] enhancement with conv2d bwd primitive reuse and related conv fwd refactoring --- .../core/kernels/mkl_conv_grad_filter_ops.cc | 658 ++++++++++++++---- .../core/kernels/mkl_conv_grad_input_ops.cc | 474 +++++++++++-- tensorflow/core/kernels/mkl_conv_ops.cc | 280 +++++--- tensorflow/core/kernels/mkl_conv_ops.h | 222 +----- tensorflow/core/util/mkl_util.h | 32 +- 5 files changed, 1109 insertions(+), 557 deletions(-) diff --git a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc b/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc index e0706568b1..d12bccc02a 100644 --- a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc +++ b/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc @@ -52,9 +52,310 @@ using mkldnn::stream; #endif namespace tensorflow { - typedef Eigen::ThreadPoolDevice CPUDevice; +#ifndef INTEL_MKL_ML + +struct MklConvBwdFilterParams { + memory::dims src_dims; + memory::dims diff_filter_dims; + memory::dims diff_bias_dims; + memory::dims diff_dst_dims; + memory::dims strides; + memory::dims dilations; + memory::dims padding_left; + memory::dims padding_right; + padding_kind padding; + + MklConvBwdFilterParams(memory::dims src_dims, + memory::dims diff_filter_dims, memory::dims diff_bias_dims, + memory::dims diff_dst_dims, memory::dims strides, + memory::dims dilations, memory::dims padding_left, + memory::dims padding_right, padding_kind padding) : + src_dims(src_dims), diff_filter_dims(diff_filter_dims), + diff_bias_dims(diff_bias_dims), diff_dst_dims(diff_dst_dims), + strides(strides), dilations(dilations), + padding_left(padding_left), padding_right(padding_right), + padding(padding) { + } +}; + +template +class MklConv2DBwdFilterPrimitive : public MklPrimitive { + public: + explicit MklConv2DBwdFilterPrimitive( + const MklConvBwdFilterParams& convBwdFilterDims) { + context_.bwd_filter_stream.reset(new stream(stream::kind::eager)); + // create conv primitive + if (context_.conv_bwd_filter == nullptr) { + Setup(convBwdFilterDims); + } + } + + ~MklConv2DBwdFilterPrimitive() {} + + // Convolution backward weights with bias + // src_data: input data buffer of src + // diff_filter_data: output data buffer of diff_filter + // diff_bias_data: output data buffer of diff_bias + // diff_dst_data: input data buffer of diff_dst + void Execute(const T* src_data, const T* diff_filter_data, + const T* diff_bias_data, const T* diff_dst_data) { + context_.src_mem->set_data_handle( + static_cast(const_cast(src_data))); + context_.diff_filter_mem->set_data_handle( + static_cast(const_cast(diff_filter_data))); + context_.diff_bias_mem->set_data_handle( + static_cast(const_cast(diff_bias_data))); + context_.diff_dst_mem->set_data_handle( + static_cast(const_cast(diff_dst_data))); + + context_.bwd_filter_stream->submit(context_.bwd_filter_primitives); + + context_.src_mem->set_data_handle(DummyData); + context_.diff_filter_mem->set_data_handle(DummyData); + context_.diff_bias_mem->set_data_handle(DummyData); + context_.diff_dst_mem->set_data_handle(DummyData); + return; + } + + // Convolution backward weights without bias + // src_data: input data buffer of src + // diff_filter_data: output data buffer of diff_filter + // diff_dst_data: input data buffer of diff_dst + void Execute(const T* src_data, + const T* diff_filter_data, const T* diff_dst_data) { + context_.src_mem->set_data_handle( + static_cast(const_cast(src_data))); + context_.diff_filter_mem->set_data_handle( + static_cast(const_cast(diff_filter_data))); + context_.diff_dst_mem->set_data_handle( + static_cast(const_cast(diff_dst_data))); + + context_.bwd_filter_stream->submit(context_.bwd_filter_primitives); + + context_.src_mem->set_data_handle(DummyData); + context_.diff_filter_mem->set_data_handle(DummyData); + context_.diff_dst_mem->set_data_handle(DummyData); + return; + } + + memory::format GetSrcMemoryFormat() const { + return context_.src_fmt; + } + + memory::format GetDiffDstMemoryFormat() const { + return context_.diff_dst_fmt; + } + + memory::format GetDiffFilterMemoryFormat() const { + return context_.diff_filter_fmt; + } + + // convolution primitive + std::shared_ptr + GetPrimitiveDesc() const { + return context_.bwd_filter_pd; + } + + private: + // Primitive reuse context for Conv2D bwd filter op + struct ConvBwdFilterContext { + // expected memory format for this primitive instance + memory::format src_fmt; + memory::format diff_dst_fmt; + memory::format diff_filter_fmt; + + // convolution bwd input primitive + std::shared_ptr + bwd_filter_pd; + std::shared_ptr conv_bwd_filter; + + // MKLDNN memory + std::shared_ptr src_mem; + std::shared_ptr diff_filter_mem; + std::shared_ptr diff_bias_mem; + std::shared_ptr diff_dst_mem; + + // desc & prmitive desc + std::shared_ptr bwd_filter_desc; + std::shared_ptr fwd_desc; + std::shared_ptr fwd_pd; + + // memory desc: forward & backward can share same memory desc + std::shared_ptr src_md; + std::shared_ptr diff_filter_md; + std::shared_ptr diff_bias_md; + std::shared_ptr diff_dst_md; + + // MKL pipeline + std::shared_ptr bwd_filter_stream; + std::vector bwd_filter_primitives; + + ConvBwdFilterContext() : + src_fmt(memory::format::any), + diff_dst_fmt(memory::format::any), + diff_filter_fmt(memory::format::any), + src_mem(nullptr), diff_filter_mem(nullptr), + diff_bias_mem(nullptr), diff_dst_mem(nullptr), + bwd_filter_desc(nullptr), fwd_desc(nullptr), fwd_pd(nullptr), + src_md(nullptr), diff_filter_md(nullptr), + diff_bias_md(nullptr), diff_dst_md(nullptr), + bwd_filter_stream(nullptr) { + } + } context_; + + engine cpu_engine_ = engine(engine::cpu, 0); + + // Setup Conv2d backward filter (weights) primitives. + void Setup(const MklConvBwdFilterParams& convBwdFilterDims) { + // create memory descriptors for convolution data w/ no specified format + context_.src_md.reset(new memory::desc({convBwdFilterDims.src_dims}, + MklDnnType(), memory::format::any)); + + context_.diff_dst_md.reset(new memory::desc( + {convBwdFilterDims.diff_dst_dims}, + MklDnnType(), memory::format::any)); + + context_.diff_filter_md.reset(new memory::desc( + {convBwdFilterDims.diff_filter_dims}, + MklDnnType(), memory::format::any)); + + if (!convBwdFilterDims.diff_bias_dims.empty()) + context_.diff_bias_md.reset(new memory::desc( + {convBwdFilterDims.diff_bias_dims}, + MklDnnType(), memory::format::x)); + + // create a convolution + if (!convBwdFilterDims.diff_bias_dims.empty()) { + context_.bwd_filter_desc.reset(new convolution_backward_weights::desc( + convolution_direct, *context_.src_md, *context_.diff_filter_md, + *context_.diff_bias_md, *context_.diff_dst_md, + convBwdFilterDims.strides, convBwdFilterDims.dilations, + convBwdFilterDims.padding_left, convBwdFilterDims.padding_right, + convBwdFilterDims.padding)); + } else { + context_.bwd_filter_desc.reset( + new convolution_backward_weights::desc( + convolution_direct, *context_.src_md, *context_.diff_filter_md, + *context_.diff_dst_md, convBwdFilterDims.strides, + convBwdFilterDims.dilations, convBwdFilterDims.padding_left, + convBwdFilterDims.padding_right, convBwdFilterDims.padding)); + } + + // create fwd primitive_desc + context_.fwd_desc.reset(new convolution_forward::desc( + prop_kind::forward, convolution_direct, + *context_.src_md, *context_.diff_filter_md, *context_.diff_dst_md, + convBwdFilterDims.strides, + convBwdFilterDims.dilations, convBwdFilterDims.padding_left, + convBwdFilterDims.padding_right, convBwdFilterDims.padding)); + context_.fwd_pd.reset(new convolution_forward::primitive_desc( + *context_.fwd_desc, cpu_engine_)); + + // create backward conv primitive_desc + context_.bwd_filter_pd.reset( + new convolution_backward_weights::primitive_desc( + *context_.bwd_filter_desc, cpu_engine_, *context_.fwd_pd)); + + // store the expected memory format + auto bwd_filter_pd = context_.bwd_filter_pd.get(); + context_.src_fmt = static_cast( + bwd_filter_pd->src_primitive_desc().desc().data.format); + context_.diff_filter_fmt = static_cast( + bwd_filter_pd->diff_weights_primitive_desc().desc().data.format); + context_.diff_dst_fmt = static_cast( + bwd_filter_pd->diff_dst_primitive_desc().desc().data.format); + + // create memory primitive based on dummy data + context_.src_mem.reset(new memory( + bwd_filter_pd->src_primitive_desc(), DummyData)); + context_.diff_filter_mem.reset(new memory( + bwd_filter_pd->diff_weights_primitive_desc(), DummyData)); + context_.diff_dst_mem.reset(new memory( + bwd_filter_pd->diff_dst_primitive_desc(), DummyData)); + + // create convolution primitive and add it to net + if (!convBwdFilterDims.diff_bias_dims.empty()) { + context_.diff_bias_mem.reset(new memory( + {{{convBwdFilterDims.diff_bias_dims}, MklDnnType(), + memory::format::x}, cpu_engine_}, DummyData)); + context_.conv_bwd_filter.reset(new convolution_backward_weights( + *context_.bwd_filter_pd, *context_.src_mem, *context_.diff_dst_mem, + *context_.diff_filter_mem, *context_.diff_bias_mem)); + } else { + context_.conv_bwd_filter.reset(new convolution_backward_weights( + *context_.bwd_filter_pd, *context_.src_mem, + *context_.diff_dst_mem, *context_.diff_filter_mem)); + } + + context_.bwd_filter_primitives.push_back(*context_.conv_bwd_filter); + return; + } +}; + +template +class MklConv2DBwdFilterPrimitiveFactory : public MklPrimitiveFactory { + public: + static MklConv2DBwdFilterPrimitive* Get( + const MklConvBwdFilterParams& convBwdFilterDims) { + MklConv2DBwdFilterPrimitive* conv2d_bwd_filter = nullptr; + + // look into the pool for reusable primitive + conv2d_bwd_filter = dynamic_cast*> ( + MklConv2DBwdFilterPrimitiveFactory::GetInstance().GetConv2dBwdFilter( + convBwdFilterDims)); + + if (conv2d_bwd_filter == nullptr) { + conv2d_bwd_filter = new MklConv2DBwdFilterPrimitive( + convBwdFilterDims); + MklConv2DBwdFilterPrimitiveFactory::GetInstance().SetConv2dBwdFilter( + convBwdFilterDims, conv2d_bwd_filter); + } + return conv2d_bwd_filter; + } + + + private: + MklConv2DBwdFilterPrimitiveFactory() {} + ~MklConv2DBwdFilterPrimitiveFactory() {} + + static MklConv2DBwdFilterPrimitiveFactory& GetInstance() { + static MklConv2DBwdFilterPrimitiveFactory instance_; + return instance_; + } + + static std::string CreateKey( + const MklConvBwdFilterParams& convBwdFilterDims) { + std::string prefix = "conv2d_bwd_filter"; + FactoryKeyCreator key_creator; + key_creator.AddAsKey(prefix); + key_creator.AddAsKey(convBwdFilterDims.src_dims); + key_creator.AddAsKey(convBwdFilterDims.diff_filter_dims); + key_creator.AddAsKey(convBwdFilterDims.diff_bias_dims); + key_creator.AddAsKey(convBwdFilterDims.diff_dst_dims); + key_creator.AddAsKey(convBwdFilterDims.strides); + key_creator.AddAsKey(convBwdFilterDims.dilations); + key_creator.AddAsKey(convBwdFilterDims.padding_left); + key_creator.AddAsKey(convBwdFilterDims.padding_right); + return key_creator.GetKey(); + } + + MklPrimitive* GetConv2dBwdFilter( + const MklConvBwdFilterParams& convBwdFilterDims) { + std::string key = CreateKey(convBwdFilterDims); + return this->GetOp(key); + } + + void SetConv2dBwdFilter( + const MklConvBwdFilterParams& convBwdFilterDims, MklPrimitive* op) { + std::string key = CreateKey(convBwdFilterDims); + this->SetOp(key, op); + } +}; + +#endif + #ifdef INTEL_MKL_ML template @@ -440,11 +741,213 @@ class MklConv2DCustomBackpropFilterOp : public MklConv2DBackpropCommonOp { public: explicit MklConv2DCustomBackpropFilterOp(OpKernelConstruction* context) - : MklConv2DBackpropCommonOp(context) {} + : MklConv2DBackpropCommonOp(context) { + } + ~MklConv2DCustomBackpropFilterOp() {} + void Compute(OpKernelContext* context) { + try { + MklDnnData src(&cpu_engine_); + MklDnnData diff_dst(&cpu_engine_); + MklDnnData diff_filter(&cpu_engine_); // output + + // Input tensors + const int kInputIdx = 0, kFilterIdx = 1, kOutbpropIdx = 2; + const Tensor& src_tensor = MklGetInput(context, kInputIdx); + const Tensor& filter_tensor = MklGetInput(context, kFilterIdx); + const Tensor& diff_dst_tensor = MklGetInput(context, kOutbpropIdx); + + MklDnnShape src_mkl_shape, filter_mkl_shape, diff_dst_mkl_shape; + GetMklShape(context, kInputIdx, &src_mkl_shape); + GetMklShape(context, kFilterIdx, &filter_mkl_shape); + GetMklShape(context, kOutbpropIdx, &diff_dst_mkl_shape); + // Allow operator-specific sanity checking of shapes. + ValidateMklShapes(src_mkl_shape, filter_mkl_shape, diff_dst_mkl_shape); + + // Allow operator-specific generation of shapes. + // E.g., Conv2DBackpropFilter gets filter as filter_sizes. It is a + // tensor containing shape of filter. So filter.shape() is not + // a correct way to get filter shape. These operator-specific calls + // allow this class to handle this case. + TensorShape src_tf_shape = MakeInputTfShape(context, src_tensor); + TensorShape filter_tf_shape = MakeFilterTfShape(context, filter_tensor); + TensorShape diff_dst_tf_shape = GetTfShape(context, kOutbpropIdx); + + // Corner cases: output with 0 elements and 0 batch size. + Tensor* diff_filter_tensor = nullptr; + if (src_tf_shape.num_elements() == 0 || + filter_tf_shape.num_elements() == 0 || + diff_dst_tf_shape.num_elements() == 0) { + MklDnnShape diff_filter_mkl_shape; + diff_filter_mkl_shape.SetMklTensor(false); + TensorShape diff_filter_tf_shape = GetOutputTfShape( + src_tf_shape, filter_tf_shape, diff_dst_tf_shape); + const int kOutputIdx = 0; + AllocateOutputSetMklShape(context, kOutputIdx, &diff_filter_tensor, + diff_filter_tf_shape, diff_filter_mkl_shape); + CHECK_NOTNULL(diff_filter_tensor); + + // if output tensor has more than 0 elements, we need to 0 them out. + auto diff_filter_data = diff_filter_tensor->flat().data(); + for (size_t i = 0; i < diff_filter_tf_shape.num_elements(); ++i) { + diff_filter_data[i] = 0; + } + return; + } + + // By default, all dims are in MKL order. Only dims in TF order + // are those with prefix tf_order. + memory::dims diff_dst_dims, fwd_src_dims, fwd_filter_dims; + memory::dims padding_left, padding_right, dilations, + strides, fwd_dst_dims; + memory::dims fwd_dst_dims_tf_order; + + // Get forward convolution parameters. + MklDnnConvUtil conv_utl(context, this->strides_, this->padding_, + this->data_format_, this->dilations_); + conv_utl.GetConvFwdSizesInMklOrder( + src_tf_shape, filter_tf_shape, &fwd_src_dims, &fwd_filter_dims, + &strides, &dilations, &fwd_dst_dims_tf_order, + &fwd_dst_dims, &padding_left, &padding_right); + if (!context->status().ok()) return; + + auto tf_fmt = TFDataFormatToMklDnnDataFormat(this->data_format_); + auto fwd_src_md = + src_mkl_shape.IsMklTensor() + ? src_mkl_shape.GetMklLayout() + : memory::desc(fwd_src_dims, MklDnnType(), tf_fmt); + + conv_utl.GetInputSizeInMklOrder(diff_dst_tf_shape, &diff_dst_dims); + if (!context->status().ok()) return; + + auto diff_dst_md = diff_dst_mkl_shape.IsMklTensor() + ? diff_dst_mkl_shape.GetMklLayout() + : memory::desc(diff_dst_dims, + MklDnnType(), tf_fmt); + + memory::dims diff_bias_dims = {}; + int64 depth = 0; + if (biasEnabled) { + TensorShape obp_tf_shape = GetTfShape(context, 2); + depth = (this->data_format_ == FORMAT_NCHW) + ? obp_tf_shape.dim_size(1) + : obp_tf_shape.dim_size(3); + diff_bias_dims = {static_cast(depth)}; + } + + dilations[kDilationH] -= 1; + dilations[kDilationW] -= 1; + + MklConv2DBwdFilterPrimitive *conv2d_bwd_filter = nullptr; + MklConvBwdFilterParams convBwdFilterDims(fwd_src_dims, fwd_filter_dims, + diff_bias_dims, diff_dst_dims, strides, dilations, padding_left, + padding_right, TFPaddingToMklDnnPadding(this->padding_)); + conv2d_bwd_filter = MklConv2DBwdFilterPrimitiveFactory::Get( + convBwdFilterDims); + auto bwd_filter_pd = conv2d_bwd_filter->GetPrimitiveDesc(); + + // allocate output tensors: diff_fitler and diff_bias (w bias) + auto bwd_output_dims = GetOutputDims(fwd_src_dims, fwd_filter_dims); + + // diff_filter + MklDnnShape diff_filter_mkl_shape; + diff_filter_mkl_shape.SetMklTensor(false); + // output_dims_mkl_order is in OIHW format. + TensorShape diff_filter_tf_shape( + {bwd_output_dims[MklDnnDims::Dim_H], + bwd_output_dims[MklDnnDims::Dim_W], + bwd_output_dims[MklDnnDims::Dim_I], + bwd_output_dims[MklDnnDims::Dim_O]}); + AllocateOutputSetMklShape(context, 0, &diff_filter_tensor, + diff_filter_tf_shape, diff_filter_mkl_shape); + + Tensor* diff_bias_tensor = nullptr; + if (biasEnabled) { + TensorShape diff_bias_shape({depth}); + AllocateBiasGradTensor(context, diff_bias_shape, &diff_bias_tensor); + } + + // check if src and diff_dst need reorder + std::vector net; + T *src_data = nullptr; + if (fwd_src_md.data.format != conv2d_bwd_filter->GetSrcMemoryFormat()) { + src.SetUsrMem(fwd_src_md, &src_tensor); + src.CheckReorderToOpMem( + bwd_filter_pd->src_primitive_desc(), &net); + src_data = static_cast(src.GetOpMem().get_data_handle()); + } else { + src_data = static_cast(const_cast( + src_tensor.flat().data())); + } + + T *diff_dst_data = nullptr; + if (diff_dst_md.data.format != + conv2d_bwd_filter->GetDiffDstMemoryFormat()) { + diff_dst.SetUsrMem(diff_dst_md, &diff_dst_tensor); + diff_dst.CheckReorderToOpMem( + bwd_filter_pd->diff_dst_primitive_desc(), &net); + diff_dst_data = static_cast( + diff_dst.GetOpMem().get_data_handle()); + } else { + diff_dst_data = static_cast(const_cast( + diff_dst_tensor.flat().data())); + } + stream(stream::kind::eager).submit(net).wait(); + + // For backward filter, convert diff_filter back to Tensorflow layout + // Here we prepare to reorder op memory back to user memory + bool diff_filter_reorder_required = false; + T *diff_filter_data = nullptr; + if (GetOutputFormat(tf_fmt) != + conv2d_bwd_filter->GetDiffFilterMemoryFormat()) { + // Allocate diff filter tensor as Tensorflow layout + diff_filter.SetUsrMem(bwd_output_dims, GetOutputFormat(tf_fmt), + diff_filter_tensor); + diff_filter_reorder_required = true; + diff_filter.PrepareReorderToUserMemIfReq( + bwd_filter_pd->diff_weights_primitive_desc()); + diff_filter_data = static_cast( + diff_filter.GetOpMem().get_data_handle()); + } else { + diff_filter_data = static_cast(const_cast( + diff_filter_tensor->flat().data())); + } + + // Execute convolution filter bwd + if (biasEnabled) { + T* diff_bias_data = static_cast(const_cast( + diff_bias_tensor->flat().data())); + conv2d_bwd_filter->Execute(src_data, diff_filter_data, + diff_bias_data, diff_dst_data); + } else { + conv2d_bwd_filter->Execute(src_data, diff_filter_data, diff_dst_data); + } + + // Reorder diff_filter back to Tensorflow layout if necessary + if (diff_filter_reorder_required) { + std::vector net; + diff_filter.InsertReorderToUserMem(&net); + stream(stream::kind::eager).submit(net).wait(); + } + } catch (mkldnn::error& e) { + string error_msg = "Status: " + std::to_string(e.status) + + ", message: " + string(e.message) + ", in file " + + string(__FILE__) + ":" + std::to_string(__LINE__); + OP_REQUIRES_OK( + context, + errors::Aborted("Operation received an exception:", error_msg)); + } + } + private: + const int kInputIndex_Filter = 1; + const int kInputIndex_InputSizes = 0; const int kDilationH = 0, kDilationW = 1; + engine cpu_engine_ = engine(engine::cpu, 0); + + // Validate input shapes. + // Function asserts that input shapes are valid. void ValidateMklShapes(const MklDnnShape& input_mkl_shape, const MklDnnShape& filter_mkl_shape, const MklDnnShape& obp_mkl_shape) { @@ -452,141 +955,44 @@ class MklConv2DCustomBackpropFilterOp << "Conv2DBackpropFilter: filter should not be in MKL Layout"; } - size_t GetInputTensorIndexWithSizes() { return 1; /* filter index */ } - + // Get TensorFlow shape of input tensor. TensorShape MakeInputTfShape(OpKernelContext* context, const Tensor& input_tensor) { size_t input_idx = 0; return GetTfShape(context, input_idx); } + // Get TensorFlow shape of filter tensor. TensorShape MakeFilterTfShape(OpKernelContext* context, const Tensor& filter_tensor) { TensorShape filter_tf_shape; CHECK_EQ(TensorShapeUtils::IsVector(filter_tensor.shape()), true); CHECK_EQ(TensorShapeUtils::MakeShape(filter_tensor.vec(), - &filter_tf_shape) - .ok(), - true); + &filter_tf_shape).ok(), true); return filter_tf_shape; } + // Get Tensorflow shape of output tensor (diff_filter), + // which is same as shape of filter. TensorShape GetOutputTfShape(const TensorShape& input_shape, const TensorShape& filter_shape, const TensorShape& outbprop_shape) { - // Shape of output of Conv2DBackpropFilter is same as shape of filter. return filter_shape; } + // Get the shape of output (diff_filter) in MKL-DNN order. + // Computes shape of output from input shape (fwd_input_dims) + // and filter shape (fwd_filter_dims). const memory::dims& GetOutputDims(const memory::dims& fwd_input_dims, const memory::dims& fwd_filter_dims) { - // Shape of output of Conv2DBackpropFilter is same as shape of filter. return fwd_filter_dims; } + // Output layout is Tensorflow's filter layout (HWIO). memory::format GetOutputFormat(const memory::format data_format) { - // Output layout is Tensorflow's filter layout (HWIO). return memory::format::hwio; } - void CreatePrimitive(OpKernelContext* context, const engine& cpu_engine, - const convolution_forward::primitive_desc& conv_fwd_pd, - MklDnnData* input, MklDnnData* filter, - MklDnnData* outbackprop, MklDnnData* output, - Tensor** output_tensor, - const memory::dims& strides, - const memory::dims& dilations, - const memory::dims& padding_l, - const memory::dims& padding_r, padding_kind padding, - const memory::dims& bwd_output_dims, - memory::format bwd_output_format) { - CHECK_NOTNULL(context); - CHECK_NOTNULL(input); - CHECK_NOTNULL(filter); - CHECK_NOTNULL(outbackprop); - CHECK_NOTNULL(output); - CHECK_NOTNULL(output_tensor); - - MklDnnData* bias_grad = nullptr; - int depth = 0; - if (biasEnabled) { - // Data structure for bias_grad - bias_grad = new MklDnnData(&cpu_engine); - TensorShape obp_tf_shape = GetTfShape(context, 2); - depth = (MklConv2DBackpropCommonOp::GetTFDataFormat() == - FORMAT_NCHW) - ? obp_tf_shape.dim_size(1) - : obp_tf_shape.dim_size(3); - memory::dims bias_grad_dims = {depth}; - bias_grad->SetOpMemDesc(bias_grad_dims, memory::format::x); - } - - if (biasEnabled && (bias_grad != nullptr)) { - // Create convolution backward weights with bias primitive. - // Use dilated convolution in case dilate rates are greater than zero. - auto bwd_desc = (dilations[kDilationH] > 0 || dilations[kDilationW] > 0) ? - convolution_backward_weights::desc(convolution_direct, - input->GetOpMemDesc(), output->GetOpMemDesc(), - bias_grad->GetOpMemDesc(), - outbackprop->GetOpMemDesc(), strides, - dilations, padding_l, padding_r, padding) : - convolution_backward_weights::desc(convolution_direct, - input->GetOpMemDesc(), output->GetOpMemDesc(), - bias_grad->GetOpMemDesc(), - outbackprop->GetOpMemDesc(), - strides, padding_l, padding_r, padding); - auto bwd_pd = convolution_backward_weights::primitive_desc(bwd_desc, - cpu_engine, - conv_fwd_pd); - - // Allocate output tensor. - AllocateOutputTensor(context, bwd_pd, bwd_output_dims, - bwd_output_format, output_tensor); - - CHECK_NOTNULL(*output_tensor); - // Set buffer handle using allocated output tensor. - output->SetUsrMemDataHandle(*output_tensor); - - // Allocate bias_grad tensor - TensorShape bias_grad_shape({depth}); - Tensor* bias_grad_tensor = nullptr; - AllocateBiasGradTensor(context, bias_grad_shape, &bias_grad_tensor); - memory::dims bias_grad_dims = {depth}; - // Since Bias is 1D, we use format::x from MKLDNN to represent it. - auto bias_grad_md = - memory::desc({bias_grad_dims}, MklDnnType(), memory::format::x); - bias_grad->SetUsrMem(bias_grad_md, bias_grad_tensor); - bias_grad->SetUsrMemDataHandle(bias_grad_tensor); - - PrepareAndExecutePrimitive(bwd_pd, input, outbackprop, output, - bias_grad); - } else { - // Create convolution backward weights primitive. - // Use dilated convolution in case dilate rates are greater than zero. - auto bwd_desc = (dilations[kDilationH] > 0 || dilations[kDilationW] > 0) ? - convolution_backward_weights::desc(convolution_direct, - input->GetOpMemDesc(), output->GetOpMemDesc(), - outbackprop->GetOpMemDesc(), strides, - dilations, padding_l, padding_r, padding) : - convolution_backward_weights::desc(convolution_direct, - input->GetOpMemDesc(), output->GetOpMemDesc(), - outbackprop->GetOpMemDesc(), - strides, padding_l, padding_r, padding); - auto bwd_pd = convolution_backward_weights::primitive_desc(bwd_desc, - cpu_engine, - conv_fwd_pd); - - // Allocate output tensor. - AllocateOutputTensor(context, bwd_pd, bwd_output_dims, - bwd_output_format, output_tensor); - - CHECK_NOTNULL(*output_tensor); - // Set buffer handle using allocated output tensor. - output->SetUsrMemDataHandle(*output_tensor); - PrepareAndExecutePrimitive(bwd_pd, input, outbackprop, output); - } - } - // Allocate output tensor. void AllocateOutputTensor( OpKernelContext* context, @@ -621,40 +1027,8 @@ class MklConv2DCustomBackpropFilterOp MklDnnShape bias_grad_mkl_shape; bias_grad_mkl_shape.SetMklTensor(false); - AllocateOutputSetMklShape(context, 1, bias_grad_tensor, bias_grad_shape, - bias_grad_mkl_shape); - } - - // Prepare and execute net - checks for input and output reorders. - void PrepareAndExecutePrimitive( - const convolution_backward_weights::primitive_desc& conv_pd, - MklDnnData* input, MklDnnData* obp, MklDnnData* output, - MklDnnData* bias_grad = nullptr) { - // Create reorders between user layout and MKL layout if it is needed and - // add it to the net before convolution. - std::vector net; - input->CheckReorderToOpMem(conv_pd.src_primitive_desc(), &net); - obp->CheckReorderToOpMem(conv_pd.diff_dst_primitive_desc(), &net); - - // For BackpropFilter, we convert the output tensor back in Tensorflow - // layout. - bool output_reorder_required = output->PrepareReorderToUserMemIfReq( - conv_pd.diff_weights_primitive_desc()); - - if (biasEnabled && (bias_grad != nullptr)) { - net.push_back(convolution_backward_weights( - conv_pd, input->GetOpMem(), obp->GetOpMem(), output->GetOpMem(), - bias_grad->GetOpMem())); - } else { - net.push_back(convolution_backward_weights( - conv_pd, input->GetOpMem(), obp->GetOpMem(), output->GetOpMem())); - } - - if (output_reorder_required) { - output->InsertReorderToUserMem(&net); - } - - stream(stream::kind::eager).submit(net).wait(); + AllocateOutputSetMklShape(context, 1, bias_grad_tensor, + bias_grad_shape, bias_grad_mkl_shape); } }; diff --git a/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc b/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc index d203c04934..e4b8564589 100644 --- a/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc +++ b/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc @@ -53,9 +53,244 @@ using mkldnn::stream; #endif namespace tensorflow { - typedef Eigen::ThreadPoolDevice CPUDevice; +#ifndef INTEL_MKL_ML + +/// utility classes enabling primitive reuse for backward conv2d ops. +struct MklConvBwdInputParams { + memory::dims diff_src_dims; + memory::dims filter_dims; + memory::dims diff_dst_dims; + memory::dims strides; + memory::dims dilations; + memory::dims padding_left; + memory::dims padding_right; + padding_kind padding; + + MklConvBwdInputParams(memory::dims diff_src_dims, + memory::dims filter_dims, memory::dims diff_dst_dims, + memory::dims strides, memory::dims dilations, + memory::dims padding_left, memory::dims padding_right, + padding_kind padding) : + diff_src_dims(diff_src_dims), filter_dims(filter_dims), + diff_dst_dims(diff_dst_dims), strides(strides), + dilations(dilations), padding_left(padding_left), + padding_right(padding_right), padding(padding) { + } +}; + +template +class MklConv2DBwdInputPrimitive : public MklPrimitive { + public: + explicit MklConv2DBwdInputPrimitive( + const MklConvBwdInputParams& convBwdInputDims) { + context_.bwd_input_stream.reset(new stream(stream::kind::eager)); + + // create conv primitive + if (context_.conv_bwd_input == nullptr) { + Setup(convBwdInputDims); + } + } + ~MklConv2DBwdInputPrimitive() {} + + // Convolution backward filter (weights) + // diff_src_data: output data buffer of diff_src + // filter_data: input data buffer of filter (weights) + // diff_dst_data: input data buffer of dst + // Bias does not matter here + void Execute(const T* diff_src_data, + const T* filter_data, const T* diff_dst_data) { + context_.diff_src_mem->set_data_handle( + static_cast(const_cast(diff_src_data))); + context_.filter_mem->set_data_handle( + static_cast(const_cast(filter_data))); + context_.diff_dst_mem->set_data_handle( + static_cast(const_cast(diff_dst_data))); + + context_.bwd_input_stream->submit(context_.bwd_input_primitives); + + // set back data handle + context_.diff_src_mem->set_data_handle(DummyData); + context_.filter_mem->set_data_handle(DummyData); + context_.diff_dst_mem->set_data_handle(DummyData); + return; + } + + memory::format GetFilterMemoryFormat() const { + return context_.filter_fmt; + } + + memory::format GetDiffDstMemoryFormat() const { + return context_.diff_dst_fmt; + } + + std::shared_ptr + GetPrimitiveDesc() const { + return context_.bwd_input_pd; + } + + private: + // Primitive reuse context for Conv2D Bwd Input op + struct ConvBwdInputContext { + // expected memory format for this primitive instance + memory::format filter_fmt; + memory::format diff_dst_fmt; + + // MKLDNN memory + std::shared_ptr diff_src_mem; + std::shared_ptr filter_mem; + std::shared_ptr diff_dst_mem; + + // convolution primitive + std::shared_ptr + bwd_input_pd; + std::shared_ptr conv_bwd_input; + + // desc & prmitive desc + std::shared_ptr bwd_input_desc; + std::shared_ptr fwd_desc; + std::shared_ptr fwd_pd; + + // memory desc: forward & backward can share same memory::desc + std::shared_ptr diff_src_md; + std::shared_ptr filter_md; + std::shared_ptr diff_dst_md; + + // MKL pipeline + std::shared_ptr bwd_input_stream; + std::vector bwd_input_primitives; + + ConvBwdInputContext() : + filter_fmt(memory::format::any), diff_dst_fmt(memory::format::any), + diff_src_mem(nullptr), filter_mem(nullptr), diff_dst_mem(nullptr), + bwd_input_pd(nullptr), conv_bwd_input(nullptr), + bwd_input_desc(nullptr), fwd_desc(nullptr), fwd_pd(nullptr), + diff_src_md(nullptr), filter_md(nullptr), diff_dst_md(nullptr), + bwd_input_stream(nullptr) { + } + } context_; + + engine cpu_engine_ = engine(engine::cpu, 0); + + void Setup(const MklConvBwdInputParams& convBwdInputDims) { + // create memory descriptors for convolution data w/ no specified format + context_.diff_src_md.reset(new memory::desc( + {convBwdInputDims.diff_src_dims}, + MklDnnType(), memory::format::any)); + context_.filter_md.reset(new memory::desc( + {convBwdInputDims.filter_dims}, + MklDnnType(), memory::format::any)); + context_.diff_dst_md.reset(new memory::desc( + {convBwdInputDims.diff_dst_dims}, + MklDnnType(), memory::format::any)); + + // create convolution primitives + context_.bwd_input_desc.reset(new convolution_backward_data::desc( + convolution_direct, *context_.diff_src_md, *context_.filter_md, + *context_.diff_dst_md, convBwdInputDims.strides, + convBwdInputDims.dilations, convBwdInputDims.padding_left, + convBwdInputDims.padding_right, convBwdInputDims.padding)); + + context_.fwd_desc.reset(new convolution_forward::desc(prop_kind::forward, + convolution_direct, *context_.diff_src_md, *context_.filter_md, + *context_.diff_dst_md, convBwdInputDims.strides, + convBwdInputDims.dilations, convBwdInputDims.padding_left, + convBwdInputDims.padding_right, convBwdInputDims.padding)); + + context_.fwd_pd.reset(new convolution_forward::primitive_desc( + *context_.fwd_desc, cpu_engine_)); + + // create backward conv prim desc + context_.bwd_input_pd.reset( + new convolution_backward_data::primitive_desc( + *context_.bwd_input_desc, cpu_engine_, *context_.fwd_pd)); + + // create memory primitive based on dummy data + context_.diff_src_mem.reset(new memory( + context_.bwd_input_pd.get()->diff_src_primitive_desc(), DummyData)); + context_.filter_mem.reset(new memory( + context_.bwd_input_pd.get()->weights_primitive_desc(), DummyData)); + context_.diff_dst_mem.reset(new memory( + context_.bwd_input_pd.get()->diff_dst_primitive_desc(), DummyData)); + + // store the expected memory format + context_.filter_fmt = static_cast( + context_.bwd_input_pd.get()->weights_primitive_desc().desc().data.format); + context_.diff_dst_fmt = static_cast( + context_.bwd_input_pd.get()->diff_dst_primitive_desc().desc().data.format); + + // create convolution primitive and add it to net + context_.conv_bwd_input.reset(new convolution_backward_data( + *context_.bwd_input_pd, *context_.diff_dst_mem, + *context_.filter_mem, *context_.diff_src_mem)); + + context_.bwd_input_primitives.push_back(*context_.conv_bwd_input); + return; + } +}; + +template +class MklConv2DBwdInputPrimitiveFactory : public MklPrimitiveFactory { + private: + MklConv2DBwdInputPrimitiveFactory() {} + ~MklConv2DBwdInputPrimitiveFactory() {} + + public: + static MklConv2DBwdInputPrimitive* Get( + const MklConvBwdInputParams& convBwdInputDims) { + MklConv2DBwdInputPrimitive* conv2d_bwd_input = nullptr; + + // look into the pool for reusable primitive + conv2d_bwd_input = dynamic_cast*> ( + MklConv2DBwdInputPrimitiveFactory::GetInstance().GetConv2dBwdInput( + convBwdInputDims)); + + if (conv2d_bwd_input == nullptr) { + conv2d_bwd_input = new MklConv2DBwdInputPrimitive( + convBwdInputDims); + MklConv2DBwdInputPrimitiveFactory::GetInstance().SetConv2dBwdInput( + convBwdInputDims, conv2d_bwd_input); + } + return conv2d_bwd_input; + } + + private: + static MklConv2DBwdInputPrimitiveFactory& GetInstance() { + static MklConv2DBwdInputPrimitiveFactory instance_; + return instance_; + } + + static std::string CreateKey( + const MklConvBwdInputParams& convBwdInputDims) { + std::string prefix = "conv2d_bwd_input"; + FactoryKeyCreator key_creator; + key_creator.AddAsKey(prefix); + key_creator.AddAsKey(convBwdInputDims.diff_src_dims); + key_creator.AddAsKey(convBwdInputDims.filter_dims); + key_creator.AddAsKey(convBwdInputDims.diff_dst_dims); + key_creator.AddAsKey(convBwdInputDims.strides); + key_creator.AddAsKey(convBwdInputDims.dilations); + key_creator.AddAsKey(convBwdInputDims.padding_left); + key_creator.AddAsKey(convBwdInputDims.padding_right); + return key_creator.GetKey(); + } + + MklPrimitive* GetConv2dBwdInput( + const MklConvBwdInputParams& convBwdInputDims) { + std::string key = CreateKey(convBwdInputDims); + return this->GetOp(key); + } + + void SetConv2dBwdInput( + const MklConvBwdInputParams& convBwdInputDims, MklPrimitive *op) { + std::string key = CreateKey(convBwdInputDims); + this->SetOp(key, op); + } +}; + +#endif + #ifdef INTEL_MKL_ML template @@ -363,13 +598,173 @@ class MklConv2DCustomBackpropInputOp : public MklConv2DBackpropCommonOp { public: explicit MklConv2DCustomBackpropInputOp(OpKernelConstruction* context) - : MklConv2DBackpropCommonOp(context) {} + : MklConv2DBackpropCommonOp(context) { + } + ~MklConv2DCustomBackpropInputOp() {} + void Compute(OpKernelContext* context) { + try { + MklDnnData filter(&cpu_engine); + MklDnnData diff_dst(&cpu_engine); + + // Input tensors + const int kInputIdx = 0, kFilterIdx = 1, kOutbpropIdx = 2; + const Tensor& src_tensor = MklGetInput(context, kInputIdx); + const Tensor& filter_tensor = MklGetInput(context, kFilterIdx); + const Tensor& diff_dst_tensor = MklGetInput(context, kOutbpropIdx); + + MklDnnShape src_mkl_shape, filter_mkl_shape, diff_dst_mkl_shape; + GetMklShape(context, kInputIdx, &src_mkl_shape); + GetMklShape(context, kFilterIdx, &filter_mkl_shape); + GetMklShape(context, kOutbpropIdx, &diff_dst_mkl_shape); + // Allow operator-specific sanity checking of shapes. + ValidateMklShapes(src_mkl_shape, filter_mkl_shape, + diff_dst_mkl_shape); + + // Allow operator-specific generation of shapes. + // E.g., Conv2DBackpropFilter gets filter as filter_sizes. It is a + // tensor containing shape of filter. So filter.shape() is not + // a correct way to get filter shape. These operator-specific calls + // allow this class to handle this case. + TensorShape src_tf_shape = MakeInputTfShape(context, src_tensor); + TensorShape filter_tf_shape = MakeFilterTfShape(context, filter_tensor); + TensorShape diff_dst_tf_shape = GetTfShape(context, kOutbpropIdx); + + // Corner cases: output with 0 elements and 0 batch size. + Tensor* diff_src_tensor = nullptr; + if (src_tf_shape.num_elements() == 0 || + filter_tf_shape.num_elements() == 0 || + diff_dst_tf_shape.num_elements() == 0) { + MklDnnShape diff_src_mkl_shape; + diff_src_mkl_shape.SetMklTensor(false); + TensorShape diff_src_tf_shape = GetOutputTfShape( + src_tf_shape, filter_tf_shape, diff_dst_tf_shape); + const int kOutputIdx = 0; + AllocateOutputSetMklShape(context, kOutputIdx, &diff_src_tensor, + diff_src_tf_shape, diff_src_mkl_shape); + CHECK_NOTNULL(diff_src_tensor); + + // if output tensor has more than 0 elements, we need to 0 them out. + auto diff_src_data = diff_src_tensor->flat().data(); + for (size_t i = 0; i < diff_src_tf_shape.num_elements(); ++i) { + diff_src_data[i] = 0; + } + return; + } + // By default, all dims are in MKL order. Only dims in TF order + // are those with postfix tf_order. + memory::dims diff_dst_dims, fwd_src_dims, fwd_filter_dims; + memory::dims padding_left, padding_right, dilations, strides; + memory::dims fwd_output_dims, fwd_output_dims_tf_order; + + // Get forward convolution parameters. + MklDnnConvUtil conv_utl(context, this->strides_, this->padding_, + this->data_format_, this->dilations_); + conv_utl.GetConvFwdSizesInMklOrder( + src_tf_shape, filter_tf_shape, &fwd_src_dims, &fwd_filter_dims, + &strides, &dilations, &fwd_output_dims_tf_order, &fwd_output_dims, + &padding_left, &padding_right); + if (!context->status().ok()) return; + + // Create Convolution forward descriptor since Convolution backward + // API needs it. For that, we first need to create input, filter + // and output memory descriptors. + auto tf_fmt = TFDataFormatToMklDnnDataFormat(this->data_format_); + + // If filter is in MKL layout, then simply grab filter layout; + // otherwise, construct filter in TF layout. + // For TF layout, filter is in HWIO format. + auto fwd_filter_md = filter_mkl_shape.IsMklTensor() + ? filter_mkl_shape.GetMklLayout() + : memory::desc(fwd_filter_dims, MklDnnType(), + memory::format::hwio); + + conv_utl.GetInputSizeInMklOrder(diff_dst_tf_shape, &diff_dst_dims); + if (!context->status().ok()) return; + auto diff_dst_md = diff_dst_mkl_shape.IsMklTensor() + ? diff_dst_mkl_shape.GetMklLayout() + : memory::desc(diff_dst_dims, + MklDnnType(), tf_fmt); + + dilations[kDilationH] -= 1; + dilations[kDilationW] -= 1; + + MklConv2DBwdInputPrimitive *conv2d_bwd_input = nullptr; + conv_utl.GetInputSizeInMklOrder(diff_dst_tf_shape, &diff_dst_dims); + MklConvBwdInputParams convBwdInputDims(fwd_src_dims, fwd_filter_dims, + diff_dst_dims, strides, dilations, padding_left, padding_right, + TFPaddingToMklDnnPadding(this->padding_)); + conv2d_bwd_input = MklConv2DBwdInputPrimitiveFactory::Get( + convBwdInputDims); + auto bwd_input_pd = conv2d_bwd_input->GetPrimitiveDesc(); + + // allocate output tensor + auto diff_src_pd = bwd_input_pd->diff_src_primitive_desc(); + auto bwd_diff_src_dims = GetOutputDims(fwd_src_dims, fwd_filter_dims); + auto bwd_diff_src_format = GetOutputFormat(tf_fmt); + MklDnnShape diff_src_mkl_shape; + diff_src_mkl_shape.SetMklTensor(true); + diff_src_mkl_shape.SetMklLayout(&diff_src_pd); + diff_src_mkl_shape.SetElemType(MklDnnType()); + diff_src_mkl_shape.SetTfLayout(bwd_diff_src_dims.size(), + bwd_diff_src_dims, bwd_diff_src_format); + TensorShape diff_src_tf_shape; + diff_src_tf_shape.AddDim(diff_src_pd.get_size() / sizeof(T)); + AllocateOutputSetMklShape(context, 0, &diff_src_tensor, + diff_src_tf_shape, diff_src_mkl_shape); + + T *diff_src_data = static_cast(const_cast( + diff_src_tensor->flat().data())); + + // check if filter and diff_dst need reorder + std::vector net; + T* filter_data = nullptr; + if (fwd_filter_md.data.format != + conv2d_bwd_input->GetFilterMemoryFormat()) { + filter.SetUsrMem(fwd_filter_md, &filter_tensor); + filter.CheckReorderToOpMem( + bwd_input_pd->weights_primitive_desc(), + &net); + filter_data = static_cast(filter.GetOpMem().get_data_handle()); + } else { + filter_data = static_cast(const_cast( + filter_tensor.flat().data())); + } + + T* diff_dst_data = nullptr; + if (diff_dst_md.data.format != + conv2d_bwd_input->GetDiffDstMemoryFormat()) { + diff_dst.SetUsrMem(diff_dst_md, &diff_dst_tensor); + diff_dst.CheckReorderToOpMem( + bwd_input_pd->diff_dst_primitive_desc(), &net); + diff_dst_data = static_cast( + diff_dst.GetOpMem().get_data_handle()); + } else { + diff_dst_data = static_cast(const_cast( + diff_dst_tensor.flat().data())); + } + stream(stream::kind::eager).submit(net).wait(); + + // execute convolution input bwd + conv2d_bwd_input->Execute(diff_src_data, filter_data, diff_dst_data); + } catch (mkldnn::error& e) { + string error_msg = "Status: " + std::to_string(e.status) + + ", message: " + string(e.message) + ", in file " + + string(__FILE__) + ":" + std::to_string(__LINE__); + OP_REQUIRES_OK( + context, + errors::Aborted("Operation received an exception:", error_msg)); + } + } + private: - const int kInputIndex_Filter = 1, kInputIndex_InputSizes = 0, - kInputIndex_OutBackProp = 2; + const int kInputIndex_Filter = 1, kInputIndex_InputSizes = 0; const int kDilationH = 0, kDilationW = 1; + engine cpu_engine = engine(engine::cpu, 0); + + // Validate input shapes. + // Function asserts that input shapes are valid. void ValidateMklShapes(const MklDnnShape& input_mkl_shape, const MklDnnShape& filter_mkl_shape, const MklDnnShape& obp_mkl_shape) { @@ -380,8 +775,7 @@ class MklConv2DCustomBackpropInputOp << "Conv2DBackpropInput: input should not be in MKL Layout"; } - size_t GetInputTensorIndexWithSizes() { return kInputIndex_InputSizes; } - + // Get TensorFlow shape of input tensor. TensorShape MakeInputTfShape(OpKernelContext* context, const Tensor& input_tensor) { TensorShape input_tf_shape; @@ -393,72 +787,32 @@ class MklConv2DCustomBackpropInputOp return input_tf_shape; } + // Get TensorFlow shape of filter tensor. TensorShape MakeFilterTfShape(OpKernelContext* context, const Tensor& filter_tensor) { return GetTfShape(context, kInputIndex_Filter); } + // Get the Tensorflow shape of Output (diff_src), + // which is same as shape of Conv2D 'input'. TensorShape GetOutputTfShape(const TensorShape& input_shape, const TensorShape& filter_shape, const TensorShape& outbprop_shape) { - // Output Shape of Conv2DBackpropInput is same as shape of Conv2D 'input'. return input_shape; } + // Get the Tensorflow shape of Output (diff_src), + // which is same as shape of Conv2D 'input'. const memory::dims& GetOutputDims(const memory::dims& fwd_input_dims, const memory::dims& fwd_filter_dims) { - // Output Shape of Conv2DBackpropInput is same as shape of Conv2D 'input'. return fwd_input_dims; } + // Output layout is Tensorflow's layout in data format order. memory::format GetOutputFormat(const memory::format data_format) { - // Output layout is Tensorflow's layout in data format order. return data_format; } - void CreatePrimitive(OpKernelContext* context, const engine& cpu_engine, - const convolution_forward::primitive_desc& conv_fwd_pd, - MklDnnData* input, MklDnnData* filter, - MklDnnData* outbackprop, MklDnnData* output, - Tensor** output_tensor, - const memory::dims& strides, - const memory::dims& dilations, - const memory::dims& padding_l, - const memory::dims& padding_r, padding_kind padding, - const memory::dims& bwd_output_dims, - memory::format bwd_output_format) { - CHECK_NOTNULL(context); - CHECK_NOTNULL(input); - CHECK_NOTNULL(filter); - CHECK_NOTNULL(outbackprop); - CHECK_NOTNULL(output); - CHECK_NOTNULL(output_tensor); - - // Create convolution backward data primitive. - // Use dilated convolution in case dilate rates are greater than zero. - auto bwd_desc = (dilations[kDilationH] > 0 || dilations[kDilationW] > 0) ? - convolution_backward_data::desc(convolution_direct, - output->GetOpMemDesc(), filter->GetOpMemDesc(), - outbackprop->GetOpMemDesc(), strides, - dilations, padding_l, padding_r, padding): - convolution_backward_data::desc(convolution_direct, - output->GetOpMemDesc(), filter->GetOpMemDesc(), - outbackprop->GetOpMemDesc(), - strides, padding_l, padding_r, padding); - - auto bwd_pd = convolution_backward_data::primitive_desc( - bwd_desc, cpu_engine, conv_fwd_pd); - - // Allocate output tensor in TensorFlow and MKL layout. - AllocateOutputTensor(context, bwd_pd, bwd_output_dims, bwd_output_format, - output_tensor); - CHECK_NOTNULL(*output_tensor); - // Set buffer handle using allocated output tensor. - output->SetUsrMemDataHandle(*output_tensor); - - PrepareAndExecutePrimitive(bwd_pd, filter, outbackprop, output); - } - // Allocate output tensor. void AllocateOutputTensor( OpKernelContext* context, @@ -485,22 +839,6 @@ class MklConv2DCustomBackpropInputOp AllocateOutputSetMklShape(context, 0, output_tensor, output_tf_shape, output_mkl_shape); } - - // Prepare and execute net - checks for input and output reorders. - void PrepareAndExecutePrimitive( - const convolution_backward_data::primitive_desc& conv_pd, - MklDnnData* filter, MklDnnData* obp, MklDnnData* output) { - // Create reorders between user layout and MKL layout if it is needed and - // add it to the net before convolution. - std::vector net; - filter->CheckReorderToOpMem(conv_pd.weights_primitive_desc(), &net); - obp->CheckReorderToOpMem(conv_pd.diff_dst_primitive_desc(), &net); - - net.push_back(convolution_backward_data( - conv_pd, obp->GetOpMem(), filter->GetOpMem(), output->GetOpMem())); - - stream(stream::kind::eager).submit(net).wait(); - } }; #endif // INTEL_MKL_ML diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc index f2b14f1278..c032add82e 100644 --- a/tensorflow/core/kernels/mkl_conv_ops.cc +++ b/tensorflow/core/kernels/mkl_conv_ops.cc @@ -59,7 +59,8 @@ namespace tensorflow { #ifndef INTEL_MKL_ML -struct ConvFwdDimensions { +// This structure aggregates multiple inputs to Conv2DFwd* methods. +struct MklConvFwdParams { memory::dims src_dims; memory::dims filter_dims; memory::dims bias_dims; @@ -69,7 +70,7 @@ struct ConvFwdDimensions { memory::dims padding_left; memory::dims padding_right; - ConvFwdDimensions(memory::dims src_dims, + MklConvFwdParams(memory::dims src_dims, memory::dims filter_dims, memory::dims bias_dims, memory::dims dst_dims, memory::dims strides, memory::dims dilations, memory::dims padding_left, @@ -82,35 +83,40 @@ struct ConvFwdDimensions { }; template -class Conv2DFwd : public DnnOp { +class MklConv2DFwdPrimitive: public MklPrimitive { public: - explicit Conv2DFwd(const ConvFwdDimensions& convFwdDims) { - fwd_stream_.reset(new stream(stream::kind::eager)); + explicit MklConv2DFwdPrimitive(const MklConvFwdParams& convFwdDims) { + context_.fwd_stream.reset(new stream(stream::kind::eager)); // create conv primitive - if (conv_fwd_ == nullptr) { + if (context_.conv_fwd == nullptr) { Setup(convFwdDims); } } - ~Conv2DFwd() {} + ~MklConv2DFwdPrimitive() {} // Convolution forward execute with bias // src_data: input data buffer of src // filter_data: input data buffer of filter (weights) // bias_data: input data buffer of bias // dst_data: output data buffer of dst - void Execute(T* src_data, T* filter_data, T* bias_data, T* dst_data) { - src_mem_->set_data_handle(static_cast(src_data)); - filter_mem_->set_data_handle(static_cast(filter_data)); - bias_mem_->set_data_handle(static_cast(bias_data)); - dst_mem_->set_data_handle(static_cast(dst_data)); - fwd_stream_->submit(fwd_primitives_); + void Execute(const T* src_data, const T* filter_data, + const T* bias_data, const T* dst_data) { + context_.src_mem->set_data_handle( + static_cast(const_cast(src_data))); + context_.filter_mem->set_data_handle( + static_cast(const_cast(filter_data))); + context_.bias_mem->set_data_handle( + static_cast(const_cast(bias_data))); + context_.dst_mem->set_data_handle( + static_cast(const_cast(dst_data))); + context_.fwd_stream->submit(context_.fwd_primitives); // after exec, set data handle back - src_mem_->set_data_handle(DummyData); - filter_mem_->set_data_handle(DummyData); - bias_mem_->set_data_handle(DummyData); - dst_mem_->set_data_handle(DummyData); + context_.src_mem->set_data_handle(DummyData); + context_.filter_mem->set_data_handle(DummyData); + context_.bias_mem->set_data_handle(DummyData); + context_.dst_mem->set_data_handle(DummyData); return; } @@ -119,139 +125,174 @@ class Conv2DFwd : public DnnOp { // src_data: input data buffer of src // filter_data: input data buffer of filter (weights) // dst_data: output data buffer of dst - void Execute(T* src_data, T* filter_data, T* dst_data) { - src_mem_->set_data_handle(static_cast(src_data)); - filter_mem_->set_data_handle(static_cast(filter_data)); - dst_mem_->set_data_handle(static_cast(dst_data)); - fwd_stream_->submit(fwd_primitives_); - - // after exec, set data handle back - src_mem_->set_data_handle(DummyData); - filter_mem_->set_data_handle(DummyData); - dst_mem_->set_data_handle(DummyData); + void Execute(const T* src_data, const T* filter_data, + const T* dst_data) { + context_.src_mem->set_data_handle( + static_cast(const_cast(src_data))); + context_.filter_mem->set_data_handle( + static_cast(const_cast(filter_data))); + context_.dst_mem->set_data_handle( + static_cast(const_cast(dst_data))); + context_.fwd_stream->submit(context_.fwd_primitives); + + // after execution, set data handle back + context_.src_mem->set_data_handle(DummyData); + context_.filter_mem->set_data_handle(DummyData); + context_.dst_mem->set_data_handle(DummyData); return; } - // expected memory format for this primitive instance - memory::format src_fmt_; - memory::format filter_fmt_; + memory::format GetSrcMemoryFormat() const { + return context_.src_fmt; + } + + memory::format GetFilterMemoryFormat() const { + return context_.filter_fmt; + } - // convolution primitive - std::shared_ptr fwd_pd_; - std::shared_ptr conv_fwd_; + std::shared_ptr + GetPrimitiveDesc() const { + return context_.fwd_pd; + } private: - void Setup(const ConvFwdDimensions& convFwdDims) { + // Primitive reuse context for Conv2D Fwd op + struct ConvFwdContext { + // expected memory format for this primitive instance + memory::format src_fmt; + memory::format filter_fmt; + + // MKLDNN memory + std::shared_ptr src_mem; + std::shared_ptr filter_mem; + std::shared_ptr bias_mem; + std::shared_ptr dst_mem; + + // desc & prmitive desc + std::shared_ptr fwd_desc; + + // memory desc + std::shared_ptr src_md; + std::shared_ptr filter_md; + std::shared_ptr bias_md; + std::shared_ptr dst_md; + + // convolution primitive + std::shared_ptr fwd_pd; + std::shared_ptr conv_fwd; + + std::shared_ptr fwd_stream; + std::vector fwd_primitives; + + ConvFwdContext() : + src_fmt(memory::format::any), filter_fmt(memory::format::any), + src_mem(nullptr), filter_mem(nullptr), bias_mem(nullptr), + dst_mem(nullptr), fwd_desc(nullptr), + src_md(nullptr), filter_md(nullptr), bias_md(nullptr), + fwd_pd(nullptr), conv_fwd(nullptr), fwd_stream(nullptr) { + } + } context_; + + engine cpu_engine_ = engine(engine::cpu, 0); + + void Setup(const MklConvFwdParams& convFwdDims) { // create memory descriptors for convolution data w/ no specified format - src_md_.reset(new memory::desc({convFwdDims.src_dims}, + context_.src_md.reset(new memory::desc({convFwdDims.src_dims}, MklDnnType(), memory::format::any)); - filter_md_.reset(new memory::desc({convFwdDims.filter_dims}, + context_.filter_md.reset(new memory::desc({convFwdDims.filter_dims}, MklDnnType(), memory::format::any)); - dst_md_.reset(new memory::desc({convFwdDims.dst_dims}, + context_.dst_md.reset(new memory::desc({convFwdDims.dst_dims}, MklDnnType(), memory::format::any)); if (!convFwdDims.bias_dims.empty()) - bias_md_.reset(new memory::desc({convFwdDims.bias_dims}, + context_.bias_md.reset(new memory::desc({convFwdDims.bias_dims}, MklDnnType(), memory::format::any)); // create a convolution if (!convFwdDims.bias_dims.empty()) { - fwd_desc_.reset(new convolution_forward::desc(prop_kind::forward, - convolution_direct, *src_md_, *filter_md_, *bias_md_, *dst_md_, + context_.fwd_desc.reset(new convolution_forward::desc(prop_kind::forward, + convolution_direct, *context_.src_md, *context_.filter_md, + *context_.bias_md, *context_.dst_md, convFwdDims.strides, convFwdDims.dilations, convFwdDims.padding_left, convFwdDims.padding_right, padding_kind::zero)); } else { - fwd_desc_.reset(new convolution_forward::desc(prop_kind::forward, - convolution_direct, *src_md_, *filter_md_, *dst_md_, - convFwdDims.strides, convFwdDims.dilations, convFwdDims.padding_left, - convFwdDims.padding_right, padding_kind::zero)); + context_.fwd_desc.reset(new convolution_forward::desc(prop_kind::forward, + convolution_direct, *context_.src_md, *context_.filter_md, + *context_.dst_md, convFwdDims.strides, convFwdDims.dilations, + convFwdDims.padding_left, convFwdDims.padding_right, + padding_kind::zero)); } - fwd_pd_.reset(new convolution_forward::primitive_desc( - *fwd_desc_, cpu_engine_)); + context_.fwd_pd.reset(new convolution_forward::primitive_desc( + *context_.fwd_desc, cpu_engine_)); // store the expected memory format - src_fmt_ = static_cast( - fwd_pd_.get()->src_primitive_desc().desc().data.format); + context_.src_fmt = static_cast( + context_.fwd_pd.get()->src_primitive_desc().desc().data.format); - filter_fmt_ = static_cast( - fwd_pd_.get()->weights_primitive_desc().desc().data.format); + context_.filter_fmt = static_cast( + context_.fwd_pd.get()->weights_primitive_desc().desc().data.format); // create memory primitive based on dummy data - src_mem_.reset(new memory(fwd_pd_.get()->src_primitive_desc(), DummyData)); - filter_mem_.reset(new memory(fwd_pd_.get()->weights_primitive_desc(), - DummyData)); - dst_mem_.reset(new memory(fwd_pd_.get()->dst_primitive_desc(), DummyData)); + context_.src_mem.reset(new memory( + context_.fwd_pd.get()->src_primitive_desc(), DummyData)); + context_.filter_mem.reset(new memory( + context_.fwd_pd.get()->weights_primitive_desc(), DummyData)); + context_.dst_mem.reset(new memory( + context_.fwd_pd.get()->dst_primitive_desc(), DummyData)); // create convolution primitive and add it to net if (!convFwdDims.bias_dims.empty()) { - bias_mem_.reset(new memory({{{convFwdDims.bias_dims}, MklDnnType(), - memory::format::x}, cpu_engine_}, DummyData)); - conv_fwd_.reset(new convolution_forward(*fwd_pd_, *src_mem_, - *filter_mem_, *bias_mem_, *dst_mem_)); + context_.bias_mem.reset(new memory({{{convFwdDims.bias_dims}, + MklDnnType(), memory::format::x}, cpu_engine_}, DummyData)); + context_.conv_fwd.reset(new convolution_forward( + *context_.fwd_pd, *context_.src_mem, *context_.filter_mem, + *context_.bias_mem, *context_.dst_mem)); } else { - conv_fwd_.reset(new convolution_forward(*fwd_pd_, *src_mem_, - *filter_mem_, *dst_mem_)); + context_.conv_fwd.reset(new convolution_forward( + *context_.fwd_pd, *context_.src_mem, + *context_.filter_mem, *context_.dst_mem)); } - fwd_primitives_.push_back(*conv_fwd_); + context_.fwd_primitives.push_back(*context_.conv_fwd); return; } - - // MKLDNN memory - std::shared_ptr src_mem_; - std::shared_ptr filter_mem_; - std::shared_ptr bias_mem_; - std::shared_ptr dst_mem_; - - std::shared_ptr fwd_stream_; - std::vector fwd_primitives_; - - // desc & prmitive desc - std::shared_ptr fwd_desc_; - - // memory desc - std::shared_ptr src_md_; - std::shared_ptr filter_md_; - std::shared_ptr bias_md_; - std::shared_ptr dst_md_; - - engine cpu_engine_ = engine(engine::cpu, 0); }; template -class Conv2DFwdFactory : public DnnOpFactory { +class MklConv2DFwdPrimitiveFactory : public MklPrimitiveFactory { public: - static Conv2DFwd* Get(const ConvFwdDimensions& convFwdDims) { - Conv2DFwd* conv2d_fwd = nullptr; + static MklConv2DFwdPrimitive* Get(const MklConvFwdParams& convFwdDims) { + MklConv2DFwdPrimitive* conv2d_fwd = nullptr; // try to find a suitable one in pool - conv2d_fwd = dynamic_cast*> ( - Conv2DFwdFactory::GetInstance().GetConv2DFwd(convFwdDims)); + conv2d_fwd = dynamic_cast*> ( + MklConv2DFwdPrimitiveFactory::GetInstance().GetConv2DFwd( + convFwdDims)); if (conv2d_fwd == nullptr) { - conv2d_fwd = new Conv2DFwd(convFwdDims); - Conv2DFwdFactory::GetInstance().SetConv2DFwd( + conv2d_fwd = new MklConv2DFwdPrimitive(convFwdDims); + MklConv2DFwdPrimitiveFactory::GetInstance().SetConv2DFwd( convFwdDims, conv2d_fwd); } return conv2d_fwd; } private: - Conv2DFwdFactory() {} - ~Conv2DFwdFactory() {} + MklConv2DFwdPrimitiveFactory() {} + ~MklConv2DFwdPrimitiveFactory() {} static const int kDilationH = 0, kDilationW = 1; - static Conv2DFwdFactory& GetInstance() { - static Conv2DFwdFactory instance_; + static MklConv2DFwdPrimitiveFactory& GetInstance() { + static MklConv2DFwdPrimitiveFactory instance_; return instance_; } - static std::string CreateKey(const ConvFwdDimensions& convFwdDims) { + static std::string CreateKey(const MklConvFwdParams& convFwdDims) { std::string prefix = "conv2d_fwd_"; FactoryKeyCreator key_creator; key_creator.AddAsKey(prefix); @@ -266,12 +307,12 @@ class Conv2DFwdFactory : public DnnOpFactory { return key_creator.GetKey(); } - DnnOp* GetConv2DFwd(const ConvFwdDimensions& convFwdDims) { + MklPrimitive* GetConv2DFwd(const MklConvFwdParams& convFwdDims) { std::string key = CreateKey(convFwdDims); return this->GetOp(key); } - void SetConv2DFwd(const ConvFwdDimensions& convFwdDims, DnnOp *op) { + void SetConv2DFwd(const MklConvFwdParams& convFwdDims, MklPrimitive *op) { std::string key = CreateKey(convFwdDims); this->SetOp(key, op); } @@ -762,7 +803,6 @@ class MklConv2DOp : public OpKernel { MklDnnData src(&cpu_engine); MklDnnData filter(&cpu_engine); - MklDnnData dst(&cpu_engine); // output memory::dims src_dims, filter_dims, padding_left, padding_right, dilations, strides; @@ -812,7 +852,6 @@ class MklConv2DOp : public OpKernel { auto src_md = src_mkl_shape.IsMklTensor() ? src_mkl_shape.GetMklLayout() : memory::desc(src_dims, MklDnnType(), tf_fmt); - src.SetUsrMem(src_md, &src_tensor); // Although filter shape (filter_dims) required is in MKL-DNN order, // the layout is Tensorflow's layout (HWIO). @@ -820,29 +859,28 @@ class MklConv2DOp : public OpKernel { ? filter_mkl_shape.GetMklLayout() : memory::desc(filter_dims, MklDnnType(), memory::format::hwio); - filter.SetUsrMem(filter_md, &filter_tensor); // MKLDNN dilation starts from 0. dilations[kDilationH] -= 1; dilations[kDilationW] -= 1; // get a conv2d fwd from primitive pool - Conv2DFwd *conv2d_fwd = nullptr; + MklConv2DFwdPrimitive *conv2d_fwd = nullptr; if (biasEnabled) { memory::dims bias_dims = {}; conv_utl.GetBiasSizeInMklOrder(kInputIndex_Bias, &bias_dims); - ConvFwdDimensions convFwdDims(src_dims, filter_dims, bias_dims, + MklConvFwdParams convFwdDims(src_dims, filter_dims, bias_dims, dst_dims_mkl_order, strides, dilations, padding_left, padding_right); - conv2d_fwd = Conv2DFwdFactory::Get(convFwdDims); + conv2d_fwd = MklConv2DFwdPrimitiveFactory::Get(convFwdDims); } else { - ConvFwdDimensions convFwdDims(src_dims, filter_dims, NONE_DIMS, + MklConvFwdParams convFwdDims(src_dims, filter_dims, NONE_DIMS, dst_dims_mkl_order, strides, dilations, padding_left, padding_right); - conv2d_fwd = Conv2DFwdFactory::Get(convFwdDims); + conv2d_fwd = MklConv2DFwdPrimitiveFactory::Get(convFwdDims); } // allocate output tensors output_tensor and filter_out_tensor std::shared_ptr - conv_fwd_pd = conv2d_fwd->fwd_pd_; + conv_fwd_pd = conv2d_fwd->GetPrimitiveDesc(); AllocateOutputTensor(context, *conv_fwd_pd, dst_dims_mkl_order, tf_fmt, &dst_tensor); Tensor* filter_out_tensor = nullptr; @@ -854,20 +892,30 @@ class MklConv2DOp : public OpKernel { // check whether src/filter need reorder std::vector net; - if (src_md.data.format != conv2d_fwd->src_fmt_) - src.CheckReorderToOpMem( - conv_fwd_pd.get()->src_primitive_desc(), &net); - - if (filter_md.data.format != conv2d_fwd->filter_fmt_) - filter.CheckReorderToOpMem( - conv_fwd_pd.get()->weights_primitive_desc(), - filter.GetTensorBuffer(filter_out_tensor), &net); + T *src_data = nullptr; + if (src_md.data.format != conv2d_fwd->GetSrcMemoryFormat()) { + src.SetUsrMem(src_md, &src_tensor); + src.CheckReorderToOpMem( + conv_fwd_pd.get()->src_primitive_desc(), &net); + src_data = static_cast(src.GetOpMem().get_data_handle()); + } else { + src_data = static_cast(const_cast( + src_tensor.flat().data())); + } + T *filter_data = nullptr; + if (filter_md.data.format != conv2d_fwd->GetFilterMemoryFormat()) { + filter.SetUsrMem(filter_md, &filter_tensor); + filter.CheckReorderToOpMem( + conv_fwd_pd.get()->weights_primitive_desc(), + filter.GetTensorBuffer(filter_out_tensor), &net); + filter_data = static_cast(filter.GetOpMem().get_data_handle()); + } else { + filter_data = static_cast(const_cast( + filter_tensor.flat().data())); + } + stream(stream::kind::eager).submit(net).wait(); - T* src_data = static_cast( - src.GetOpMem().get_data_handle()); - T* filter_data = static_cast( - filter.GetOpMem().get_data_handle()); // execute convolution if (biasEnabled) { diff --git a/tensorflow/core/kernels/mkl_conv_ops.h b/tensorflow/core/kernels/mkl_conv_ops.h index 8333a09316..5e1a5001dc 100644 --- a/tensorflow/core/kernels/mkl_conv_ops.h +++ b/tensorflow/core/kernels/mkl_conv_ops.h @@ -19,6 +19,7 @@ limitations under the License. #include #include #include +#include #include "tensorflow/core/framework/numeric_op.h" #include "tensorflow/core/framework/op_kernel.h" @@ -349,6 +350,7 @@ class MklDnnConvUtil { } }; + ///////////////////////////////////////////////////////////////////// /// Common class that implements Conv2DBackpropFilter and Input ///////////////////////////////////////////////////////////////////// @@ -388,227 +390,17 @@ class MklConv2DBackpropCommonOp : public OpKernel { OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_)); } - void Compute(OpKernelContext* context) override { - try { - auto cpu_engine = engine(engine::cpu, 0); - - // Prepare common tensors for Conv2DBackpropInput and - // Conv2DBackpropFilter. - MklDnnData input(&cpu_engine); - MklDnnData filter(&cpu_engine); - MklDnnData outbackprop(&cpu_engine); - MklDnnData output(&cpu_engine); - - // Input tensors - const int kInputIdx = 0, kFilterIdx = 1, kOutbpropIdx = 2; - const Tensor& input_tensor = MklGetInput(context, kInputIdx); - const Tensor& filter_tensor = MklGetInput(context, kFilterIdx); - const Tensor& outbprop_tensor = MklGetInput(context, kOutbpropIdx); - - MklDnnShape input_mkl_shape, filter_mkl_shape, outbprop_mkl_shape; - GetMklShape(context, kInputIdx, &input_mkl_shape); - GetMklShape(context, kFilterIdx, &filter_mkl_shape); - GetMklShape(context, kOutbpropIdx, &outbprop_mkl_shape); - // Allow operator-specific sanity checking of shapes. - ValidateMklShapes(input_mkl_shape, filter_mkl_shape, outbprop_mkl_shape); - - // Allow operator-specific generation of shapes. - // E.g., Conv2DBackpropFilter gets filter as filter_sizes. It is a - // tensor containing shape of filter. So filter.shape() is not - // a correct way to get filter shape. These operator-specific calls - // allow this class to handle this case. - TensorShape input_tf_shape = MakeInputTfShape(context, input_tensor); - TensorShape filter_tf_shape = MakeFilterTfShape(context, filter_tensor); - TensorShape outbprop_tf_shape = GetTfShape(context, kOutbpropIdx); - - // Corner cases: output with 0 elements and 0 batch size. - Tensor* output_tensor = nullptr; - if (input_tf_shape.num_elements() == 0 || - filter_tf_shape.num_elements() == 0 || - outbprop_tf_shape.num_elements() == 0) { - MklDnnShape output_mkl_shape; - output_mkl_shape.SetMklTensor(false); - TensorShape output_tf_shape = GetOutputTfShape( - input_tf_shape, filter_tf_shape, outbprop_tf_shape); - const int kOutputIdx = 0; - AllocateOutputSetMklShape(context, kOutputIdx, &output_tensor, - output_tf_shape, output_mkl_shape); - CHECK_NOTNULL(output_tensor); - - // if output tensor has more than 0 elements, we need to 0 them out. - for (size_t i = 0; i < output_tf_shape.num_elements(); ++i) { - output_tensor->flat().data()[i] = 0; - } - - return; - } - - // By default, all dims are in MKL order. Only dims in TF order - // are those with prefix tf_order. - memory::dims outbprop_dims, fwd_input_dims, fwd_filter_dims; - memory::dims padding_l, padding_r, dilations, strides, fwd_output_dims; - memory::dims fwd_output_dims_tf_order; - - // Get forward convolution parameters. - MklDnnConvUtil conv_utl(context, strides_, padding_, data_format_, - dilations_); - conv_utl.GetConvFwdSizesInMklOrder( - input_tf_shape, filter_tf_shape, &fwd_input_dims, &fwd_filter_dims, - &strides, &dilations, &fwd_output_dims_tf_order, &fwd_output_dims, - &padding_l, &padding_r); - if (!context->status().ok()) return; - - // Create Convolution forward descriptor since Convolution backward - // API needs it. For that, we first need to create input, filter - // and output memory descriptors. - auto tf_fmt = TFDataFormatToMklDnnDataFormat(data_format_); - // If input is in MKL layout, then simply grab input layout; otherwise, - // construct input TF layout. For TF layout, although input shape - // required is in MKL-DNN order, the layout is Tensorflow's layout - // (NHWC or NCHW depending on data format). - auto fwd_input_md = - input_mkl_shape.IsMklTensor() - ? input_mkl_shape.GetMklLayout() - : memory::desc(fwd_input_dims, MklDnnType(), tf_fmt); - // If filter is in MKL layout, then simply grab filter layout; otherwise - // construct filter in TF layout. For TF layout, filter is in HWIO format. - auto fwd_filter_md = filter_mkl_shape.IsMklTensor() - ? filter_mkl_shape.GetMklLayout() - : memory::desc(fwd_filter_dims, MklDnnType(), - memory::format::hwio); - // Tensorflow Output of Conv2D is in data_format order. - auto fwd_out_md = memory::desc(fwd_output_dims, MklDnnType(), tf_fmt); - - const int kDilationH = 0, kDilationW = 1; - dilations[kDilationH] -= 1; - dilations[kDilationW] -= 1; - auto fwd_desc = (dilations[kDilationH] > 0 || dilations[kDilationW] > 0)? - convolution_forward::desc(prop_kind::forward, - convolution_direct, fwd_input_md, - fwd_filter_md, fwd_out_md, - strides, dilations, padding_l, padding_r, - TFPaddingToMklDnnPadding(padding_)) : - convolution_forward::desc(prop_kind::forward, - convolution_direct, fwd_input_md, - fwd_filter_md, fwd_out_md, - strides, padding_l, padding_r, - TFPaddingToMklDnnPadding(padding_)); - auto fwd_pd = convolution_forward::primitive_desc(fwd_desc, cpu_engine); - - // Create memory for user data. Describe how the inputs and outputs of - // Convolution look like. Also specify buffers containing actual input - // and output data. - - // Since this is a common class for both Conv2DBackpropFilter and - // Conv2DBackpropInput, we skip SetUsrMem call for input tensor (for - // Conv2DBackpropInput) and for filter tensor (for - // conv2DBackpropFilter) depending on which tensor is int32 type. - size_t input_with_sizes = GetInputTensorIndexWithSizes(); - if (input_with_sizes != kInputIdx) { - // Shape of Conv2DBackpropFilter's input is same as Conv2D input. - input.SetUsrMem(fwd_input_md, &input_tensor); - } else if (input_with_sizes != kFilterIdx) { - // Shape of Conv2DBackpropInput's filter is same as Conv2D filter. - filter.SetUsrMem(fwd_filter_md, &filter_tensor); - } - - conv_utl.GetInputSizeInMklOrder(outbprop_tf_shape, &outbprop_dims); - if (!context->status().ok()) return; - if (outbprop_mkl_shape.IsMklTensor()) { - // If outbackprop is in Mkl layout, then simply grab it. - auto outbprop_md = outbprop_mkl_shape.GetMklLayout(); - outbackprop.SetUsrMem(outbprop_md, &outbprop_tensor); - } else { - // If outbackprop is in TensorFlow layout, then we need to create memory - // descriptor for it. Outbackprop shape is data format order. - outbackprop.SetUsrMem(outbprop_dims, tf_fmt, &outbprop_tensor); - } - - // Operator specific call to get output shape and data_format. - auto bwd_output_dims = GetOutputDims(fwd_input_dims, fwd_filter_dims); - auto bwd_output_format = GetOutputFormat(tf_fmt); - output.SetUsrMem(bwd_output_dims, bwd_output_format); - - // Create memory descriptors for convolution data w/ no specified format. - input.SetOpMemDesc(fwd_input_dims, memory::format::any); - filter.SetOpMemDesc(fwd_filter_dims, memory::format::any); - outbackprop.SetOpMemDesc(outbprop_dims, memory::format::any); - output.SetOpMemDesc(bwd_output_dims, memory::format::any); - - // Operator-specific call to create and execute primitive. - CreatePrimitive(context, cpu_engine, fwd_pd, &input, &filter, - &outbackprop, &output, &output_tensor, - strides, dilations, padding_l, padding_r, - TFPaddingToMklDnnPadding(padding_), - bwd_output_dims, bwd_output_format); - } catch (mkldnn::error& e) { - string error_msg = "Status: " + std::to_string(e.status) + - ", message: " + string(e.message) + ", in file " + - string(__FILE__) + ":" + std::to_string(__LINE__); - OP_REQUIRES_OK( - context, - errors::Aborted("Operation received an exception:", error_msg)); - } - } - - /// Pure virtual function to allow operator to check for validity of input - /// shapes. Function asserts that input shapes are valid. - virtual void ValidateMklShapes(const MklDnnShape& input_mkl_shape, - const MklDnnShape& filter_mkl_shape, - const MklDnnShape& outbprop_mkl_shape) = 0; - - /// Operator-specific function that returns index of input that is - /// representing input sizes. For Conv2DBackpropFilter it returns 1 since - /// filter for this operator is filter shape. For Conv2DBackpropInput it - /// returns 0 (for input). - virtual size_t GetInputTensorIndexWithSizes() = 0; - - /// Get TensorFlow shape of input tensor. - virtual TensorShape MakeInputTfShape(OpKernelContext* context, - const Tensor& input_tensor) = 0; - - /// Get TensorFlow shape of filter tensor. - virtual TensorShape MakeFilterTfShape(OpKernelContext* context, - const Tensor& filter_tensor) = 0; - - /// Get the TensorFlow shape of output tensor. - virtual TensorShape GetOutputTfShape(const TensorShape& input_shape, - const TensorShape& filter_shape, - const TensorShape& outbprop_shape) = 0; - - /// Get shape of output in MKL-DNN order. Computes shape of output from - /// input shape (fwd_input_dims) and filter shape (fwd_filter_dims). - virtual const memory::dims& GetOutputDims( - const memory::dims& fwd_input_dims, - const memory::dims& fwd_filter_dims) = 0; - - /// Get data_format of output in MKL-DNN order. If output data format is - /// same as input data format, then it simply returns value of data_format - /// parameter as it is. - virtual memory::format GetOutputFormat(const memory::format data_format) = 0; - - /// Create and execute the primitive storing output in the output_tensor. - virtual void CreatePrimitive(OpKernelContext* context, - const engine& cpu_engine, - const convolution_forward::primitive_desc& conv_fwd_pd, - MklDnnData* input, MklDnnData* filter, MklDnnData* outbackprop, - MklDnnData* output, Tensor** output_tensor, const memory::dims& strides, - const memory::dims& dilations, const memory::dims& padding_l, - const memory::dims& padding_r, padding_kind padding, - const memory::dims& bwd_output_dims, - memory::format bwd_output_format) = 0; - - // Get the data_format {NCHW, NHWC} - TensorFormat GetTFDataFormat() { return data_format_; } - - private: + protected: + // data members accessible to derived classes. std::vector dilations_; std::vector strides_; Padding padding_; - TensorFormat data_format_; + TensorFormat data_format_; // NCHW or NHWC }; + #endif // INTEL_MKL_ML + ///////////////////////////////////////////////////////////////////// /// Dummy Mkl op that is just used for operators that are intermediate /// output of node fusion in the graph diff --git a/tensorflow/core/util/mkl_util.h b/tensorflow/core/util/mkl_util.h index 230b4278ca..f19307756f 100644 --- a/tensorflow/core/util/mkl_util.h +++ b/tensorflow/core/util/mkl_util.h @@ -1794,11 +1794,11 @@ class MklDnnData { } }; -/// Base class for operations with reuse of DNN primitives +/// Base class for operations with reuse of primitives /// -class DnnOp { +class MklPrimitive { public: - virtual ~DnnOp() {} + virtual ~MklPrimitive() {} // Dummy data. Its size, hard-coded as 256 here, does // not matter since MKL should never operate on this buffer. @@ -1806,33 +1806,33 @@ class DnnOp { }; const mkldnn::memory::dims NONE_DIMS = {}; -// This constant is used to declare dummy buffer (size), for MKL primitives + template -class DnnOpFactory { +class MklPrimitiveFactory { public: - DnnOpFactory() {} - ~DnnOpFactory() {} + MklPrimitiveFactory() {} + ~MklPrimitiveFactory() {} - DnnOp* GetOp(const std::string& key) { - auto stream_iter = DnnOpFactory::GetHashMap().find(key); - if (stream_iter == DnnOpFactory::GetHashMap().end()) { + MklPrimitive* GetOp(const std::string& key) { + auto stream_iter = MklPrimitiveFactory::GetHashMap().find(key); + if (stream_iter == MklPrimitiveFactory::GetHashMap().end()) { return nullptr; } else { return stream_iter->second; } } - void SetOp(const std::string& key, DnnOp* op) { - auto stream_iter = DnnOpFactory::GetHashMap().find(key); + void SetOp(const std::string& key, MklPrimitive* op) { + auto stream_iter = MklPrimitiveFactory::GetHashMap().find(key); - CHECK(stream_iter == DnnOpFactory::GetHashMap().end()); + CHECK(stream_iter == MklPrimitiveFactory::GetHashMap().end()); - DnnOpFactory::GetHashMap()[key] = op; + MklPrimitiveFactory::GetHashMap()[key] = op; } private: - static inline std::unordered_map &GetHashMap() { - static thread_local std::unordered_map map_; + static inline std::unordered_map &GetHashMap() { + static thread_local std::unordered_map map_; return map_; } }; -- GitLab From e84a1cb522f868257284cd440840bcbd81cbea78 Mon Sep 17 00:00:00 2001 From: Guozhong Zhuang Date: Tue, 22 May 2018 21:52:48 -0700 Subject: [PATCH 014/911] minor code style fix --- tensorflow/core/util/mkl_util.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/util/mkl_util.h b/tensorflow/core/util/mkl_util.h index f19307756f..c4b5e124fb 100644 --- a/tensorflow/core/util/mkl_util.h +++ b/tensorflow/core/util/mkl_util.h @@ -1794,7 +1794,7 @@ class MklDnnData { } }; -/// Base class for operations with reuse of primitives +/// Base class for operations with reuse of primitives /// class MklPrimitive { public: -- GitLab From 4196890b1eba299a72ce79a542d0d3d452f6088f Mon Sep 17 00:00:00 2001 From: Dan Douthit Date: Fri, 1 Jun 2018 15:46:32 -0600 Subject: [PATCH 015/911] Updating loss calculation to use one_hot vectors The in-depth description for the code uses one_hot labels as input to the softmax_cross_entropy() function, therefore the code should use one hot labels as well. --- tensorflow/docs_src/tutorials/layers.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/docs_src/tutorials/layers.md b/tensorflow/docs_src/tutorials/layers.md index 0f17899dae..f1f87dbad7 100644 --- a/tensorflow/docs_src/tutorials/layers.md +++ b/tensorflow/docs_src/tutorials/layers.md @@ -169,7 +169,8 @@ def cnn_model_fn(features, labels, mode): return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions) # Calculate Loss (for both TRAIN and EVAL modes) - loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits) + onehot_labels = tf.one_hot(indices=tf.cast(labels, tf.int32), depth=10) + loss = tf.losses.sparse_softmax_cross_entropy(labels=onehot_labels, logits=logits) # Configure the Training Op (for TRAIN mode) if mode == tf.estimator.ModeKeys.TRAIN: -- GitLab From 327bea444a2f8c58e8844561dd20abb88032cd89 Mon Sep 17 00:00:00 2001 From: Guozhong Zhuang Date: Mon, 4 Jun 2018 09:39:54 -0700 Subject: [PATCH 016/911] revert changes with mkl_conv_ops.cc and mkl_util.h --- tensorflow/core/kernels/mkl_conv_ops.cc | 280 ++++++++++-------------- tensorflow/core/util/mkl_util.h | 32 +-- 2 files changed, 132 insertions(+), 180 deletions(-) diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc index c032add82e..f2b14f1278 100644 --- a/tensorflow/core/kernels/mkl_conv_ops.cc +++ b/tensorflow/core/kernels/mkl_conv_ops.cc @@ -59,8 +59,7 @@ namespace tensorflow { #ifndef INTEL_MKL_ML -// This structure aggregates multiple inputs to Conv2DFwd* methods. -struct MklConvFwdParams { +struct ConvFwdDimensions { memory::dims src_dims; memory::dims filter_dims; memory::dims bias_dims; @@ -70,7 +69,7 @@ struct MklConvFwdParams { memory::dims padding_left; memory::dims padding_right; - MklConvFwdParams(memory::dims src_dims, + ConvFwdDimensions(memory::dims src_dims, memory::dims filter_dims, memory::dims bias_dims, memory::dims dst_dims, memory::dims strides, memory::dims dilations, memory::dims padding_left, @@ -83,40 +82,35 @@ struct MklConvFwdParams { }; template -class MklConv2DFwdPrimitive: public MklPrimitive { +class Conv2DFwd : public DnnOp { public: - explicit MklConv2DFwdPrimitive(const MklConvFwdParams& convFwdDims) { - context_.fwd_stream.reset(new stream(stream::kind::eager)); + explicit Conv2DFwd(const ConvFwdDimensions& convFwdDims) { + fwd_stream_.reset(new stream(stream::kind::eager)); // create conv primitive - if (context_.conv_fwd == nullptr) { + if (conv_fwd_ == nullptr) { Setup(convFwdDims); } } - ~MklConv2DFwdPrimitive() {} + ~Conv2DFwd() {} // Convolution forward execute with bias // src_data: input data buffer of src // filter_data: input data buffer of filter (weights) // bias_data: input data buffer of bias // dst_data: output data buffer of dst - void Execute(const T* src_data, const T* filter_data, - const T* bias_data, const T* dst_data) { - context_.src_mem->set_data_handle( - static_cast(const_cast(src_data))); - context_.filter_mem->set_data_handle( - static_cast(const_cast(filter_data))); - context_.bias_mem->set_data_handle( - static_cast(const_cast(bias_data))); - context_.dst_mem->set_data_handle( - static_cast(const_cast(dst_data))); - context_.fwd_stream->submit(context_.fwd_primitives); + void Execute(T* src_data, T* filter_data, T* bias_data, T* dst_data) { + src_mem_->set_data_handle(static_cast(src_data)); + filter_mem_->set_data_handle(static_cast(filter_data)); + bias_mem_->set_data_handle(static_cast(bias_data)); + dst_mem_->set_data_handle(static_cast(dst_data)); + fwd_stream_->submit(fwd_primitives_); // after exec, set data handle back - context_.src_mem->set_data_handle(DummyData); - context_.filter_mem->set_data_handle(DummyData); - context_.bias_mem->set_data_handle(DummyData); - context_.dst_mem->set_data_handle(DummyData); + src_mem_->set_data_handle(DummyData); + filter_mem_->set_data_handle(DummyData); + bias_mem_->set_data_handle(DummyData); + dst_mem_->set_data_handle(DummyData); return; } @@ -125,174 +119,139 @@ class MklConv2DFwdPrimitive: public MklPrimitive { // src_data: input data buffer of src // filter_data: input data buffer of filter (weights) // dst_data: output data buffer of dst - void Execute(const T* src_data, const T* filter_data, - const T* dst_data) { - context_.src_mem->set_data_handle( - static_cast(const_cast(src_data))); - context_.filter_mem->set_data_handle( - static_cast(const_cast(filter_data))); - context_.dst_mem->set_data_handle( - static_cast(const_cast(dst_data))); - context_.fwd_stream->submit(context_.fwd_primitives); - - // after execution, set data handle back - context_.src_mem->set_data_handle(DummyData); - context_.filter_mem->set_data_handle(DummyData); - context_.dst_mem->set_data_handle(DummyData); + void Execute(T* src_data, T* filter_data, T* dst_data) { + src_mem_->set_data_handle(static_cast(src_data)); + filter_mem_->set_data_handle(static_cast(filter_data)); + dst_mem_->set_data_handle(static_cast(dst_data)); + fwd_stream_->submit(fwd_primitives_); - return; - } + // after exec, set data handle back + src_mem_->set_data_handle(DummyData); + filter_mem_->set_data_handle(DummyData); + dst_mem_->set_data_handle(DummyData); - memory::format GetSrcMemoryFormat() const { - return context_.src_fmt; + return; } - memory::format GetFilterMemoryFormat() const { - return context_.filter_fmt; - } + // expected memory format for this primitive instance + memory::format src_fmt_; + memory::format filter_fmt_; - std::shared_ptr - GetPrimitiveDesc() const { - return context_.fwd_pd; - } + // convolution primitive + std::shared_ptr fwd_pd_; + std::shared_ptr conv_fwd_; private: - // Primitive reuse context for Conv2D Fwd op - struct ConvFwdContext { - // expected memory format for this primitive instance - memory::format src_fmt; - memory::format filter_fmt; - - // MKLDNN memory - std::shared_ptr src_mem; - std::shared_ptr filter_mem; - std::shared_ptr bias_mem; - std::shared_ptr dst_mem; - - // desc & prmitive desc - std::shared_ptr fwd_desc; - - // memory desc - std::shared_ptr src_md; - std::shared_ptr filter_md; - std::shared_ptr bias_md; - std::shared_ptr dst_md; - - // convolution primitive - std::shared_ptr fwd_pd; - std::shared_ptr conv_fwd; - - std::shared_ptr fwd_stream; - std::vector fwd_primitives; - - ConvFwdContext() : - src_fmt(memory::format::any), filter_fmt(memory::format::any), - src_mem(nullptr), filter_mem(nullptr), bias_mem(nullptr), - dst_mem(nullptr), fwd_desc(nullptr), - src_md(nullptr), filter_md(nullptr), bias_md(nullptr), - fwd_pd(nullptr), conv_fwd(nullptr), fwd_stream(nullptr) { - } - } context_; - - engine cpu_engine_ = engine(engine::cpu, 0); - - void Setup(const MklConvFwdParams& convFwdDims) { + void Setup(const ConvFwdDimensions& convFwdDims) { // create memory descriptors for convolution data w/ no specified format - context_.src_md.reset(new memory::desc({convFwdDims.src_dims}, + src_md_.reset(new memory::desc({convFwdDims.src_dims}, MklDnnType(), memory::format::any)); - context_.filter_md.reset(new memory::desc({convFwdDims.filter_dims}, + filter_md_.reset(new memory::desc({convFwdDims.filter_dims}, MklDnnType(), memory::format::any)); - context_.dst_md.reset(new memory::desc({convFwdDims.dst_dims}, + dst_md_.reset(new memory::desc({convFwdDims.dst_dims}, MklDnnType(), memory::format::any)); if (!convFwdDims.bias_dims.empty()) - context_.bias_md.reset(new memory::desc({convFwdDims.bias_dims}, + bias_md_.reset(new memory::desc({convFwdDims.bias_dims}, MklDnnType(), memory::format::any)); // create a convolution if (!convFwdDims.bias_dims.empty()) { - context_.fwd_desc.reset(new convolution_forward::desc(prop_kind::forward, - convolution_direct, *context_.src_md, *context_.filter_md, - *context_.bias_md, *context_.dst_md, + fwd_desc_.reset(new convolution_forward::desc(prop_kind::forward, + convolution_direct, *src_md_, *filter_md_, *bias_md_, *dst_md_, convFwdDims.strides, convFwdDims.dilations, convFwdDims.padding_left, convFwdDims.padding_right, padding_kind::zero)); } else { - context_.fwd_desc.reset(new convolution_forward::desc(prop_kind::forward, - convolution_direct, *context_.src_md, *context_.filter_md, - *context_.dst_md, convFwdDims.strides, convFwdDims.dilations, - convFwdDims.padding_left, convFwdDims.padding_right, - padding_kind::zero)); + fwd_desc_.reset(new convolution_forward::desc(prop_kind::forward, + convolution_direct, *src_md_, *filter_md_, *dst_md_, + convFwdDims.strides, convFwdDims.dilations, convFwdDims.padding_left, + convFwdDims.padding_right, padding_kind::zero)); } - context_.fwd_pd.reset(new convolution_forward::primitive_desc( - *context_.fwd_desc, cpu_engine_)); + fwd_pd_.reset(new convolution_forward::primitive_desc( + *fwd_desc_, cpu_engine_)); // store the expected memory format - context_.src_fmt = static_cast( - context_.fwd_pd.get()->src_primitive_desc().desc().data.format); + src_fmt_ = static_cast( + fwd_pd_.get()->src_primitive_desc().desc().data.format); - context_.filter_fmt = static_cast( - context_.fwd_pd.get()->weights_primitive_desc().desc().data.format); + filter_fmt_ = static_cast( + fwd_pd_.get()->weights_primitive_desc().desc().data.format); // create memory primitive based on dummy data - context_.src_mem.reset(new memory( - context_.fwd_pd.get()->src_primitive_desc(), DummyData)); - context_.filter_mem.reset(new memory( - context_.fwd_pd.get()->weights_primitive_desc(), DummyData)); - context_.dst_mem.reset(new memory( - context_.fwd_pd.get()->dst_primitive_desc(), DummyData)); + src_mem_.reset(new memory(fwd_pd_.get()->src_primitive_desc(), DummyData)); + filter_mem_.reset(new memory(fwd_pd_.get()->weights_primitive_desc(), + DummyData)); + dst_mem_.reset(new memory(fwd_pd_.get()->dst_primitive_desc(), DummyData)); // create convolution primitive and add it to net if (!convFwdDims.bias_dims.empty()) { - context_.bias_mem.reset(new memory({{{convFwdDims.bias_dims}, - MklDnnType(), memory::format::x}, cpu_engine_}, DummyData)); - context_.conv_fwd.reset(new convolution_forward( - *context_.fwd_pd, *context_.src_mem, *context_.filter_mem, - *context_.bias_mem, *context_.dst_mem)); + bias_mem_.reset(new memory({{{convFwdDims.bias_dims}, MklDnnType(), + memory::format::x}, cpu_engine_}, DummyData)); + conv_fwd_.reset(new convolution_forward(*fwd_pd_, *src_mem_, + *filter_mem_, *bias_mem_, *dst_mem_)); } else { - context_.conv_fwd.reset(new convolution_forward( - *context_.fwd_pd, *context_.src_mem, - *context_.filter_mem, *context_.dst_mem)); + conv_fwd_.reset(new convolution_forward(*fwd_pd_, *src_mem_, + *filter_mem_, *dst_mem_)); } - context_.fwd_primitives.push_back(*context_.conv_fwd); + fwd_primitives_.push_back(*conv_fwd_); return; } + + // MKLDNN memory + std::shared_ptr src_mem_; + std::shared_ptr filter_mem_; + std::shared_ptr bias_mem_; + std::shared_ptr dst_mem_; + + std::shared_ptr fwd_stream_; + std::vector fwd_primitives_; + + // desc & prmitive desc + std::shared_ptr fwd_desc_; + + // memory desc + std::shared_ptr src_md_; + std::shared_ptr filter_md_; + std::shared_ptr bias_md_; + std::shared_ptr dst_md_; + + engine cpu_engine_ = engine(engine::cpu, 0); }; template -class MklConv2DFwdPrimitiveFactory : public MklPrimitiveFactory { +class Conv2DFwdFactory : public DnnOpFactory { public: - static MklConv2DFwdPrimitive* Get(const MklConvFwdParams& convFwdDims) { - MklConv2DFwdPrimitive* conv2d_fwd = nullptr; + static Conv2DFwd* Get(const ConvFwdDimensions& convFwdDims) { + Conv2DFwd* conv2d_fwd = nullptr; // try to find a suitable one in pool - conv2d_fwd = dynamic_cast*> ( - MklConv2DFwdPrimitiveFactory::GetInstance().GetConv2DFwd( - convFwdDims)); + conv2d_fwd = dynamic_cast*> ( + Conv2DFwdFactory::GetInstance().GetConv2DFwd(convFwdDims)); if (conv2d_fwd == nullptr) { - conv2d_fwd = new MklConv2DFwdPrimitive(convFwdDims); - MklConv2DFwdPrimitiveFactory::GetInstance().SetConv2DFwd( + conv2d_fwd = new Conv2DFwd(convFwdDims); + Conv2DFwdFactory::GetInstance().SetConv2DFwd( convFwdDims, conv2d_fwd); } return conv2d_fwd; } private: - MklConv2DFwdPrimitiveFactory() {} - ~MklConv2DFwdPrimitiveFactory() {} + Conv2DFwdFactory() {} + ~Conv2DFwdFactory() {} static const int kDilationH = 0, kDilationW = 1; - static MklConv2DFwdPrimitiveFactory& GetInstance() { - static MklConv2DFwdPrimitiveFactory instance_; + static Conv2DFwdFactory& GetInstance() { + static Conv2DFwdFactory instance_; return instance_; } - static std::string CreateKey(const MklConvFwdParams& convFwdDims) { + static std::string CreateKey(const ConvFwdDimensions& convFwdDims) { std::string prefix = "conv2d_fwd_"; FactoryKeyCreator key_creator; key_creator.AddAsKey(prefix); @@ -307,12 +266,12 @@ class MklConv2DFwdPrimitiveFactory : public MklPrimitiveFactory { return key_creator.GetKey(); } - MklPrimitive* GetConv2DFwd(const MklConvFwdParams& convFwdDims) { + DnnOp* GetConv2DFwd(const ConvFwdDimensions& convFwdDims) { std::string key = CreateKey(convFwdDims); return this->GetOp(key); } - void SetConv2DFwd(const MklConvFwdParams& convFwdDims, MklPrimitive *op) { + void SetConv2DFwd(const ConvFwdDimensions& convFwdDims, DnnOp *op) { std::string key = CreateKey(convFwdDims); this->SetOp(key, op); } @@ -803,6 +762,7 @@ class MklConv2DOp : public OpKernel { MklDnnData src(&cpu_engine); MklDnnData filter(&cpu_engine); + MklDnnData dst(&cpu_engine); // output memory::dims src_dims, filter_dims, padding_left, padding_right, dilations, strides; @@ -852,6 +812,7 @@ class MklConv2DOp : public OpKernel { auto src_md = src_mkl_shape.IsMklTensor() ? src_mkl_shape.GetMklLayout() : memory::desc(src_dims, MklDnnType(), tf_fmt); + src.SetUsrMem(src_md, &src_tensor); // Although filter shape (filter_dims) required is in MKL-DNN order, // the layout is Tensorflow's layout (HWIO). @@ -859,28 +820,29 @@ class MklConv2DOp : public OpKernel { ? filter_mkl_shape.GetMklLayout() : memory::desc(filter_dims, MklDnnType(), memory::format::hwio); + filter.SetUsrMem(filter_md, &filter_tensor); // MKLDNN dilation starts from 0. dilations[kDilationH] -= 1; dilations[kDilationW] -= 1; // get a conv2d fwd from primitive pool - MklConv2DFwdPrimitive *conv2d_fwd = nullptr; + Conv2DFwd *conv2d_fwd = nullptr; if (biasEnabled) { memory::dims bias_dims = {}; conv_utl.GetBiasSizeInMklOrder(kInputIndex_Bias, &bias_dims); - MklConvFwdParams convFwdDims(src_dims, filter_dims, bias_dims, + ConvFwdDimensions convFwdDims(src_dims, filter_dims, bias_dims, dst_dims_mkl_order, strides, dilations, padding_left, padding_right); - conv2d_fwd = MklConv2DFwdPrimitiveFactory::Get(convFwdDims); + conv2d_fwd = Conv2DFwdFactory::Get(convFwdDims); } else { - MklConvFwdParams convFwdDims(src_dims, filter_dims, NONE_DIMS, + ConvFwdDimensions convFwdDims(src_dims, filter_dims, NONE_DIMS, dst_dims_mkl_order, strides, dilations, padding_left, padding_right); - conv2d_fwd = MklConv2DFwdPrimitiveFactory::Get(convFwdDims); + conv2d_fwd = Conv2DFwdFactory::Get(convFwdDims); } // allocate output tensors output_tensor and filter_out_tensor std::shared_ptr - conv_fwd_pd = conv2d_fwd->GetPrimitiveDesc(); + conv_fwd_pd = conv2d_fwd->fwd_pd_; AllocateOutputTensor(context, *conv_fwd_pd, dst_dims_mkl_order, tf_fmt, &dst_tensor); Tensor* filter_out_tensor = nullptr; @@ -892,30 +854,20 @@ class MklConv2DOp : public OpKernel { // check whether src/filter need reorder std::vector net; - T *src_data = nullptr; - if (src_md.data.format != conv2d_fwd->GetSrcMemoryFormat()) { - src.SetUsrMem(src_md, &src_tensor); - src.CheckReorderToOpMem( - conv_fwd_pd.get()->src_primitive_desc(), &net); - src_data = static_cast(src.GetOpMem().get_data_handle()); - } else { - src_data = static_cast(const_cast( - src_tensor.flat().data())); - } - T *filter_data = nullptr; - if (filter_md.data.format != conv2d_fwd->GetFilterMemoryFormat()) { - filter.SetUsrMem(filter_md, &filter_tensor); - filter.CheckReorderToOpMem( - conv_fwd_pd.get()->weights_primitive_desc(), - filter.GetTensorBuffer(filter_out_tensor), &net); - filter_data = static_cast(filter.GetOpMem().get_data_handle()); - } else { - filter_data = static_cast(const_cast( - filter_tensor.flat().data())); - } - + if (src_md.data.format != conv2d_fwd->src_fmt_) + src.CheckReorderToOpMem( + conv_fwd_pd.get()->src_primitive_desc(), &net); + + if (filter_md.data.format != conv2d_fwd->filter_fmt_) + filter.CheckReorderToOpMem( + conv_fwd_pd.get()->weights_primitive_desc(), + filter.GetTensorBuffer(filter_out_tensor), &net); stream(stream::kind::eager).submit(net).wait(); + T* src_data = static_cast( + src.GetOpMem().get_data_handle()); + T* filter_data = static_cast( + filter.GetOpMem().get_data_handle()); // execute convolution if (biasEnabled) { diff --git a/tensorflow/core/util/mkl_util.h b/tensorflow/core/util/mkl_util.h index c4b5e124fb..230b4278ca 100644 --- a/tensorflow/core/util/mkl_util.h +++ b/tensorflow/core/util/mkl_util.h @@ -1794,11 +1794,11 @@ class MklDnnData { } }; -/// Base class for operations with reuse of primitives +/// Base class for operations with reuse of DNN primitives /// -class MklPrimitive { +class DnnOp { public: - virtual ~MklPrimitive() {} + virtual ~DnnOp() {} // Dummy data. Its size, hard-coded as 256 here, does // not matter since MKL should never operate on this buffer. @@ -1806,33 +1806,33 @@ class MklPrimitive { }; const mkldnn::memory::dims NONE_DIMS = {}; - +// This constant is used to declare dummy buffer (size), for MKL primitives template -class MklPrimitiveFactory { +class DnnOpFactory { public: - MklPrimitiveFactory() {} - ~MklPrimitiveFactory() {} + DnnOpFactory() {} + ~DnnOpFactory() {} - MklPrimitive* GetOp(const std::string& key) { - auto stream_iter = MklPrimitiveFactory::GetHashMap().find(key); - if (stream_iter == MklPrimitiveFactory::GetHashMap().end()) { + DnnOp* GetOp(const std::string& key) { + auto stream_iter = DnnOpFactory::GetHashMap().find(key); + if (stream_iter == DnnOpFactory::GetHashMap().end()) { return nullptr; } else { return stream_iter->second; } } - void SetOp(const std::string& key, MklPrimitive* op) { - auto stream_iter = MklPrimitiveFactory::GetHashMap().find(key); + void SetOp(const std::string& key, DnnOp* op) { + auto stream_iter = DnnOpFactory::GetHashMap().find(key); - CHECK(stream_iter == MklPrimitiveFactory::GetHashMap().end()); + CHECK(stream_iter == DnnOpFactory::GetHashMap().end()); - MklPrimitiveFactory::GetHashMap()[key] = op; + DnnOpFactory::GetHashMap()[key] = op; } private: - static inline std::unordered_map &GetHashMap() { - static thread_local std::unordered_map map_; + static inline std::unordered_map &GetHashMap() { + static thread_local std::unordered_map map_; return map_; } }; -- GitLab From d7c971c1563cb61f0df6dc6ce3c89641ca0a5d8a Mon Sep 17 00:00:00 2001 From: Dan Douthit Date: Mon, 4 Jun 2018 16:08:38 -0600 Subject: [PATCH 017/911] Updating tutorial to use sparse_softmax_cross_entropy() As per request from @MarkDaoust, I have updated the tutorial section of the notebook to reflect the usage of tf.losses.sparse_softmax_cross_entropy() rather than tf.losses.softmax_cross_entropy() using one-hot vectors. --- tensorflow/docs_src/tutorials/layers.md | 45 +++++-------------------- 1 file changed, 9 insertions(+), 36 deletions(-) diff --git a/tensorflow/docs_src/tutorials/layers.md b/tensorflow/docs_src/tutorials/layers.md index f1f87dbad7..a3b8b7e1fd 100644 --- a/tensorflow/docs_src/tutorials/layers.md +++ b/tensorflow/docs_src/tutorials/layers.md @@ -169,8 +169,7 @@ def cnn_model_fn(features, labels, mode): return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions) # Calculate Loss (for both TRAIN and EVAL modes) - onehot_labels = tf.one_hot(indices=tf.cast(labels, tf.int32), depth=10) - loss = tf.losses.sparse_softmax_cross_entropy(labels=onehot_labels, logits=logits) + loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits) # Configure the Training Op (for TRAIN mode) if mode == tf.estimator.ModeKeys.TRAIN: @@ -471,50 +470,24 @@ as the loss metric. The following code calculates cross entropy when the model runs in either `TRAIN` or `EVAL` mode: ```python -onehot_labels = tf.one_hot(indices=tf.cast(labels, tf.int32), depth=10) -loss = tf.losses.softmax_cross_entropy( - onehot_labels=onehot_labels, logits=logits) +loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits) ``` Let's take a closer look at what's happening above. Our `labels` tensor contains a list of predictions for our examples, e.g. `[1, -9, ...]`. In order to calculate cross-entropy, first we need to convert `labels` +9, ...]`. By using `tf.losses.sparse_softmax_cross_entropy()` we do not need to convert `labels` to the corresponding -[one-hot encoding](https://www.quora.com/What-is-one-hot-encoding-and-when-is-it-used-in-data-science): +[one-hot encoding](https://www.quora.com/What-is-one-hot-encoding-and-when-is-it-used-in-data-science) +that is commonly used in machine learning applications. -```none -[[0, 1, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 1], - ...] -``` - -We use the @{tf.one_hot} function -to perform this conversion. `tf.one_hot()` has two required arguments: - -* `indices`. The locations in the one-hot tensor that will have "on - values"—i.e., the locations of `1` values in the tensor shown above. -* `depth`. The depth of the one-hot tensor—i.e., the number of target classes. - Here, the depth is `10`. - -The following code creates the one-hot tensor for our labels, `onehot_labels`: - -```python -onehot_labels = tf.one_hot(indices=tf.cast(labels, tf.int32), depth=10) -``` - -Because `labels` contains a series of values from 0–9, `indices` is just our -`labels` tensor, with values cast to integers. The `depth` is `10` because we -have 10 possible target classes, one for each digit. - -Next, we compute cross-entropy of `onehot_labels` and the softmax of the -predictions from our logits layer. `tf.losses.softmax_cross_entropy()` takes -`onehot_labels` and `logits` as arguments, performs softmax activation on +Next, we compute cross-entropy of `labels` and the softmax of the +predictions from our logits layer. `tf.losses.sparse_softmax_cross_entropy()` takes +`labels` and `logits` as arguments, performs softmax activation on `logits`, calculates cross-entropy, and returns our `loss` as a scalar `Tensor`: ```python -loss = tf.losses.softmax_cross_entropy( - onehot_labels=onehot_labels, logits=logits) +loss = tf.losses.sparse_softmax_cross_entropy(labels=onehot_labels, logits=logits) ``` ### Configure the Training Op -- GitLab From 558cbd9fc89055f532a9558a276a9e6b438371cf Mon Sep 17 00:00:00 2001 From: apantykhin Date: Wed, 6 Jun 2018 19:55:46 +0400 Subject: [PATCH 018/911] remove optimizer checking. --- tensorflow/contrib/gan/python/estimator/python/head_impl.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tensorflow/contrib/gan/python/estimator/python/head_impl.py b/tensorflow/contrib/gan/python/estimator/python/head_impl.py index 652ffee30a..4750f94d9a 100644 --- a/tensorflow/contrib/gan/python/estimator/python/head_impl.py +++ b/tensorflow/contrib/gan/python/estimator/python/head_impl.py @@ -25,7 +25,6 @@ from tensorflow.contrib.gan.python import train as tfgan_train from tensorflow.python.estimator import model_fn as model_fn_lib from tensorflow.python.estimator.canned import head from tensorflow.python.framework import ops -from tensorflow.python.training import optimizer __all__ = [ 'GANHead', @@ -96,10 +95,6 @@ class GANHead(head._Head): # pylint: disable=protected-access raise TypeError('generator_loss_fn must be callable.') if not callable(discriminator_loss_fn): raise TypeError('discriminator_loss_fn must be callable.') - if not isinstance(generator_optimizer, optimizer.Optimizer): - raise TypeError('generator_optimizer must be Optimizer.') - if not isinstance(discriminator_optimizer, optimizer.Optimizer): - raise TypeError('discriminator_optimizer must be Optimizer.') if not use_loss_summaries in [True, False, None]: raise ValueError('use_loss_summaries must be True, False or None.') if get_hooks_fn is not None and not callable(get_hooks_fn): -- GitLab From 19b77a282b1ade7788ae394f22ac0bd7b0a2ce76 Mon Sep 17 00:00:00 2001 From: nrstott Date: Sun, 10 Jun 2018 20:35:14 -0400 Subject: [PATCH 019/911] document target_column when Y is dataframe --- tensorflow/python/estimator/inputs/pandas_io.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/estimator/inputs/pandas_io.py b/tensorflow/python/estimator/inputs/pandas_io.py index 6918683ce7..91eb98cbf4 100644 --- a/tensorflow/python/estimator/inputs/pandas_io.py +++ b/tensorflow/python/estimator/inputs/pandas_io.py @@ -76,7 +76,8 @@ def pandas_input_fn(x, num_threads: Integer, number of threads used for reading and enqueueing. In order to have predicted and repeatable order of reading and enqueueing, such as in prediction and evaluation mode, `num_threads` should be 1. - target_column: str, name to give the target column `y`. + target_column: str, name to give the target column `y`. This parameter + is not used when `y` is a `DataFrame`. Returns: Function, that has signature of ()->(dict of `features`, `target`) -- GitLab From 932fcbbd3836022a862d2479d716fc9c7563ff47 Mon Sep 17 00:00:00 2001 From: nrstott Date: Mon, 11 Jun 2018 10:11:35 -0400 Subject: [PATCH 020/911] check that target_column is correct type --- tensorflow/python/estimator/inputs/pandas_io.py | 3 +++ .../python/estimator/inputs/pandas_io_test.py | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/tensorflow/python/estimator/inputs/pandas_io.py b/tensorflow/python/estimator/inputs/pandas_io.py index 91eb98cbf4..708d65ff68 100644 --- a/tensorflow/python/estimator/inputs/pandas_io.py +++ b/tensorflow/python/estimator/inputs/pandas_io.py @@ -95,6 +95,9 @@ def pandas_input_fn(x, raise TypeError('shuffle must be explicitly set as boolean; ' 'got {}'.format(shuffle)) + if isinstance(target_column, list): + raise TypeError('target_column must be a string or None') + x = x.copy() if y is not None: if target_column in x: diff --git a/tensorflow/python/estimator/inputs/pandas_io_test.py b/tensorflow/python/estimator/inputs/pandas_io_test.py index f8546abb8a..85ba29fd86 100644 --- a/tensorflow/python/estimator/inputs/pandas_io_test.py +++ b/tensorflow/python/estimator/inputs/pandas_io_test.py @@ -75,6 +75,19 @@ class PandasIoTest(test.TestCase): pandas_io.pandas_input_fn( x, y_noindex, batch_size=2, shuffle=False, num_epochs=1) + def testPandasInputFn_RaisesWhenTargetColumnIsAList(self): + if not HAS_PANDAS: + return + + x, y = self.makeTestDataFrame() + + with self.assertRaisesRegexp(TypeError, + 'target_column must be a string or None'): + pandas_io.pandas_input_fn(x, y, batch_size=2, + shuffle=False, + num_epochs=1, + target_column=['one', 'two']) + def testPandasInputFn_NonBoolShuffle(self): if not HAS_PANDAS: return -- GitLab From 5e81395f7e427615294d99b90641ef4319a0e7ad Mon Sep 17 00:00:00 2001 From: nrstott Date: Mon, 11 Jun 2018 10:20:41 -0400 Subject: [PATCH 021/911] use uuid to generate unique target_column name --- tensorflow/python/estimator/inputs/pandas_io.py | 5 +++-- .../python/estimator/inputs/pandas_io_test.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/estimator/inputs/pandas_io.py b/tensorflow/python/estimator/inputs/pandas_io.py index 708d65ff68..dc46110e87 100644 --- a/tensorflow/python/estimator/inputs/pandas_io.py +++ b/tensorflow/python/estimator/inputs/pandas_io.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import uuid import numpy as np from tensorflow.python.estimator.inputs.queues import feeding_functions @@ -46,8 +47,8 @@ def _get_unique_target_key(features, target_column_name): A unique key that can be used to insert the target into features. """ - while target_column_name in features: - target_column_name += '_n' + if target_column_name in features: + target_column_name += '_' + str(uuid.uuid4()) return target_column_name diff --git a/tensorflow/python/estimator/inputs/pandas_io_test.py b/tensorflow/python/estimator/inputs/pandas_io_test.py index 85ba29fd86..fa75e1ba4a 100644 --- a/tensorflow/python/estimator/inputs/pandas_io_test.py +++ b/tensorflow/python/estimator/inputs/pandas_io_test.py @@ -143,6 +143,22 @@ class PandasIoTest(test.TestCase): self.assertAllEqual(targets['a'], [10, 11]) self.assertAllEqual(targets['b'], [50, 51]) + def testPandasInputFnWhenYIsDataFrame_HandlesOverlappingColumnNamesInTargets(self): + if not HAS_PANDAS: + return + with self.test_session() as session: + x, y = self.makeTestDataFrameWithYAsDataFrame() + y = y.rename(columns={'a_target': 'a', 'b_target': 'a_n'}) + input_fn = pandas_io.pandas_input_fn( + x, y, batch_size=2, shuffle=False, num_epochs=1) + + features, targets = self.callInputFnOnce(input_fn, session) + + self.assertAllEqual(features['a'], [0, 1]) + self.assertAllEqual(features['b'], [32, 33]) + self.assertAllEqual(targets['a'], [10, 11]) + self.assertAllEqual(targets['a_n'], [50, 51]) + def testPandasInputFn_ProducesOutputsForLargeBatchAndMultipleEpochs(self): if not HAS_PANDAS: return -- GitLab From 56150c9829b79c2249a4b90087ce25b1e6624f0b Mon Sep 17 00:00:00 2001 From: Guozhong Zhuang Date: Tue, 12 Jun 2018 15:37:42 -0700 Subject: [PATCH 022/911] code refactoring per Rasmus's suggestions on PR 19754 --- tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc | 11 ++++++----- tensorflow/core/kernels/mkl_conv_grad_input_ops.cc | 10 ++++++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc b/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc index d12bccc02a..b3f1bdcc7d 100644 --- a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc +++ b/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc @@ -84,7 +84,8 @@ template class MklConv2DBwdFilterPrimitive : public MklPrimitive { public: explicit MklConv2DBwdFilterPrimitive( - const MklConvBwdFilterParams& convBwdFilterDims) { + const MklConvBwdFilterParams& convBwdFilterDims) : + cpu_engine_(engine::cpu, 0) { context_.bwd_filter_stream.reset(new stream(stream::kind::eager)); // create conv primitive if (context_.conv_bwd_filter == nullptr) { @@ -203,9 +204,7 @@ class MklConv2DBwdFilterPrimitive : public MklPrimitive { diff_bias_md(nullptr), diff_dst_md(nullptr), bwd_filter_stream(nullptr) { } - } context_; - - engine cpu_engine_ = engine(engine::cpu, 0); + }; // Setup Conv2d backward filter (weights) primitives. void Setup(const MklConvBwdFilterParams& convBwdFilterDims) { @@ -290,8 +289,10 @@ class MklConv2DBwdFilterPrimitive : public MklPrimitive { } context_.bwd_filter_primitives.push_back(*context_.conv_bwd_filter); - return; } + + struct ConvBwdFilterContext context_; + engine cpu_engine_; }; template diff --git a/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc b/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc index e4b8564589..4d9493725d 100644 --- a/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc +++ b/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc @@ -84,7 +84,8 @@ template class MklConv2DBwdInputPrimitive : public MklPrimitive { public: explicit MklConv2DBwdInputPrimitive( - const MklConvBwdInputParams& convBwdInputDims) { + const MklConvBwdInputParams& convBwdInputDims) : + cpu_engine_(engine::cpu, 0) { context_.bwd_input_stream.reset(new stream(stream::kind::eager)); // create conv primitive @@ -169,9 +170,8 @@ class MklConv2DBwdInputPrimitive : public MklPrimitive { diff_src_md(nullptr), filter_md(nullptr), diff_dst_md(nullptr), bwd_input_stream(nullptr) { } - } context_; + }; - engine cpu_engine_ = engine(engine::cpu, 0); void Setup(const MklConvBwdInputParams& convBwdInputDims) { // create memory descriptors for convolution data w/ no specified format @@ -226,8 +226,10 @@ class MklConv2DBwdInputPrimitive : public MklPrimitive { *context_.filter_mem, *context_.diff_src_mem)); context_.bwd_input_primitives.push_back(*context_.conv_bwd_input); - return; } + + struct ConvBwdInputContext context_; + engine cpu_engine_; }; template -- GitLab From b2a12d441d556921ac06d00a356bdc5a34c02f11 Mon Sep 17 00:00:00 2001 From: xxxx001 Date: Mon, 18 Jun 2018 03:04:54 -0700 Subject: [PATCH 023/911] invalid eager env_ or env_->rendezvous_mgr. --- .../core/distributed_runtime/eager/eager_service_impl.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc b/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc index 4bd74b81a7..94f39095e1 100644 --- a/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc +++ b/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc @@ -80,6 +80,9 @@ Status GetNumRetvals(tensorflow::EagerContext* context, const string& op_name, Status EagerServiceImpl::CreateContext(const CreateContextRequest* request, CreateContextResponse* response) { + if (env_ == nullptr || env_->rendezvous_mgr == nullptr) { + return errors::InvalidArgument("invalid eager env_ or env_->rendezvous_mgr."); + } tensorflow::RemoteRendezvous* r = env_->rendezvous_mgr->Find(0); std::vector devices; TF_RETURN_IF_ERROR(tensorflow::DeviceFactory::AddDevices( -- GitLab From ace209ce764d8819b74f80f418ede90389d99b44 Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Mon, 18 Jun 2018 16:33:33 -0700 Subject: [PATCH 024/911] More simplifications to the text. --- tensorflow/docs_src/tutorials/layers.md | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/tensorflow/docs_src/tutorials/layers.md b/tensorflow/docs_src/tutorials/layers.md index a3b8b7e1fd..61b00ad0f2 100644 --- a/tensorflow/docs_src/tutorials/layers.md +++ b/tensorflow/docs_src/tutorials/layers.md @@ -475,20 +475,13 @@ loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits) Let's take a closer look at what's happening above. -Our `labels` tensor contains a list of predictions for our examples, e.g. `[1, -9, ...]`. By using `tf.losses.sparse_softmax_cross_entropy()` we do not need to convert `labels` -to the corresponding -[one-hot encoding](https://www.quora.com/What-is-one-hot-encoding-and-when-is-it-used-in-data-science) -that is commonly used in machine learning applications. +Our `labels` tensor contains a list of prediction indices for our examples, e.g. `[1, +9, ...]`. `logits` contains the linear outputs of our last layer. -Next, we compute cross-entropy of `labels` and the softmax of the -predictions from our logits layer. `tf.losses.sparse_softmax_cross_entropy()` takes -`labels` and `logits` as arguments, performs softmax activation on -`logits`, calculates cross-entropy, and returns our `loss` as a scalar `Tensor`: +`tf.losses.sparse_softmax_cross_entropy`, calculates the softmax crossentropy +(aka: categorical crossentropy, negative log-likelihood) from these two inputs +in an efficient, numerically stable way. -```python -loss = tf.losses.sparse_softmax_cross_entropy(labels=onehot_labels, logits=logits) -``` ### Configure the Training Op -- GitLab From 9e40df6773e1d29d78bfebed7c97e830fc92bd69 Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Mon, 18 Jun 2018 16:40:54 -0700 Subject: [PATCH 025/911] Update api_def_Exp.pbtxt --- tensorflow/core/api_def/base_api/api_def_Exp.pbtxt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/api_def/base_api/api_def_Exp.pbtxt b/tensorflow/core/api_def/base_api/api_def_Exp.pbtxt index 01ac3d433a..dd1e3d5dfc 100644 --- a/tensorflow/core/api_def/base_api/api_def_Exp.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_Exp.pbtxt @@ -1,4 +1,4 @@ op { graph_op_name: "Exp" - summary: "Computes exponential of x element-wise. \\(y = e^x\\)." + summary: "Computes exponential of x element-wise. \\\\(y = e^x\\\\)." } -- GitLab From 2b654c6e9b5c914b3bd0be304ab51c5fc39b4762 Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Mon, 18 Jun 2018 16:41:27 -0700 Subject: [PATCH 026/911] Update api_def_GatherNd.pbtxt --- tensorflow/core/api_def/base_api/api_def_GatherNd.pbtxt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/api_def/base_api/api_def_GatherNd.pbtxt b/tensorflow/core/api_def/base_api/api_def_GatherNd.pbtxt index 342a1f6b05..c156e8854c 100644 --- a/tensorflow/core/api_def/base_api/api_def_GatherNd.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_GatherNd.pbtxt @@ -25,7 +25,7 @@ END (K-1)-dimensional tensor of indices into `params`, where each element defines a slice of `params`: - output[\\(i_0, ..., i_{K-2}\\)] = params[indices[\\(i_0, ..., i_{K-2}\\)]] + output[\\\\(i_0, ..., i_{K-2}\\\\)] = params[indices[\\\\(i_0, ..., i_{K-2}\\\\)]] Whereas in @{tf.gather} `indices` defines slices into the first dimension of `params`, in `tf.gather_nd`, `indices` defines slices into the -- GitLab From 5004bf9e463d1da909a0c62b6fc2f51bb00ed0eb Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Mon, 18 Jun 2018 16:41:49 -0700 Subject: [PATCH 027/911] Update api_def_MatrixExponential.pbtxt --- .../core/api_def/base_api/api_def_MatrixExponential.pbtxt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/api_def/base_api/api_def_MatrixExponential.pbtxt b/tensorflow/core/api_def/base_api/api_def_MatrixExponential.pbtxt index d7b56aec87..cfd8a6d391 100644 --- a/tensorflow/core/api_def/base_api/api_def_MatrixExponential.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_MatrixExponential.pbtxt @@ -18,7 +18,7 @@ END } summary: "Computes the matrix exponential of one or more square matrices:" description: < Date: Mon, 18 Jun 2018 16:42:08 -0700 Subject: [PATCH 028/911] Update api_def_MatrixLogarithm.pbtxt --- tensorflow/core/api_def/base_api/api_def_MatrixLogarithm.pbtxt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/api_def/base_api/api_def_MatrixLogarithm.pbtxt b/tensorflow/core/api_def/base_api/api_def_MatrixLogarithm.pbtxt index 9e80064d15..05bf8ce76a 100644 --- a/tensorflow/core/api_def/base_api/api_def_MatrixLogarithm.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_MatrixLogarithm.pbtxt @@ -20,7 +20,7 @@ END summary: "Computes the matrix logarithm of one or more square matrices:" description: < Date: Mon, 18 Jun 2018 17:37:27 -0700 Subject: [PATCH 029/911] Update api_def_GatherNd.pbtxt --- tensorflow/core/api_def/base_api/api_def_GatherNd.pbtxt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/api_def/base_api/api_def_GatherNd.pbtxt b/tensorflow/core/api_def/base_api/api_def_GatherNd.pbtxt index c156e8854c..342a1f6b05 100644 --- a/tensorflow/core/api_def/base_api/api_def_GatherNd.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_GatherNd.pbtxt @@ -25,7 +25,7 @@ END (K-1)-dimensional tensor of indices into `params`, where each element defines a slice of `params`: - output[\\\\(i_0, ..., i_{K-2}\\\\)] = params[indices[\\\\(i_0, ..., i_{K-2}\\\\)]] + output[\\(i_0, ..., i_{K-2}\\)] = params[indices[\\(i_0, ..., i_{K-2}\\)]] Whereas in @{tf.gather} `indices` defines slices into the first dimension of `params`, in `tf.gather_nd`, `indices` defines slices into the -- GitLab From 994a04808f71d19984420b70ce31fe66f06b174f Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Mon, 18 Jun 2018 17:37:42 -0700 Subject: [PATCH 030/911] Update api_def_MatrixExponential.pbtxt --- .../core/api_def/base_api/api_def_MatrixExponential.pbtxt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/api_def/base_api/api_def_MatrixExponential.pbtxt b/tensorflow/core/api_def/base_api/api_def_MatrixExponential.pbtxt index cfd8a6d391..d7b56aec87 100644 --- a/tensorflow/core/api_def/base_api/api_def_MatrixExponential.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_MatrixExponential.pbtxt @@ -18,7 +18,7 @@ END } summary: "Computes the matrix exponential of one or more square matrices:" description: < Date: Mon, 18 Jun 2018 17:38:01 -0700 Subject: [PATCH 031/911] Update api_def_MatrixLogarithm.pbtxt --- tensorflow/core/api_def/base_api/api_def_MatrixLogarithm.pbtxt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/api_def/base_api/api_def_MatrixLogarithm.pbtxt b/tensorflow/core/api_def/base_api/api_def_MatrixLogarithm.pbtxt index 05bf8ce76a..9e80064d15 100644 --- a/tensorflow/core/api_def/base_api/api_def_MatrixLogarithm.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_MatrixLogarithm.pbtxt @@ -20,7 +20,7 @@ END summary: "Computes the matrix logarithm of one or more square matrices:" description: < Date: Mon, 18 Jun 2018 17:38:27 -0700 Subject: [PATCH 032/911] Update api_def_Polygamma.pbtxt --- tensorflow/core/api_def/base_api/api_def_Polygamma.pbtxt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/api_def/base_api/api_def_Polygamma.pbtxt b/tensorflow/core/api_def/base_api/api_def_Polygamma.pbtxt index 9196f5dd19..10bf370f54 100644 --- a/tensorflow/core/api_def/base_api/api_def_Polygamma.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_Polygamma.pbtxt @@ -1,6 +1,6 @@ op { graph_op_name: "Polygamma" - summary: "Compute the polygamma function \\(\psi^{(n)}(x)\\)." + summary: "Compute the polygamma function \\\\(\\psi^{(n)}(x)\\\\)." description: < Date: Mon, 18 Jun 2018 17:39:39 -0700 Subject: [PATCH 033/911] Update api_def_Zeta.pbtxt --- tensorflow/core/api_def/base_api/api_def_Zeta.pbtxt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/api_def/base_api/api_def_Zeta.pbtxt b/tensorflow/core/api_def/base_api/api_def_Zeta.pbtxt index a87bdaed90..c02860a16a 100644 --- a/tensorflow/core/api_def/base_api/api_def_Zeta.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_Zeta.pbtxt @@ -1,6 +1,6 @@ op { graph_op_name: "Zeta" - summary: "Compute the Hurwitz zeta function \\(\zeta(x, q)\\)." + summary: "Compute the Hurwitz zeta function \\\\(\\zeta(x, q)\\\\)." description: < Date: Mon, 18 Jun 2018 20:11:26 -0700 Subject: [PATCH 034/911] eager env_ and env_rendezvous_mgr valid --- tensorflow/core/distributed_runtime/eager/eager_service_impl.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc b/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc index 94f39095e1..ca4ed02d85 100644 --- a/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc +++ b/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc @@ -81,7 +81,7 @@ Status GetNumRetvals(tensorflow::EagerContext* context, const string& op_name, Status EagerServiceImpl::CreateContext(const CreateContextRequest* request, CreateContextResponse* response) { if (env_ == nullptr || env_->rendezvous_mgr == nullptr) { - return errors::InvalidArgument("invalid eager env_ or env_->rendezvous_mgr."); + return errors::InvalidArgument("invalid eager env_ or env_->rendezvous_mgr."); } tensorflow::RemoteRendezvous* r = env_->rendezvous_mgr->Find(0); std::vector devices; -- GitLab From 4647f36e5a61c540c760f0952be5952c5f69118d Mon Sep 17 00:00:00 2001 From: "Li, Yiqiang" Date: Tue, 19 Jun 2018 12:32:23 +0800 Subject: [PATCH 035/911] [INTEL_MKL] Fix reorder creation failure in MklConcat op. --- tensorflow/core/kernels/mkl_concat_op.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/kernels/mkl_concat_op.cc b/tensorflow/core/kernels/mkl_concat_op.cc index 31d1b949ef..d054f0d404 100644 --- a/tensorflow/core/kernels/mkl_concat_op.cc +++ b/tensorflow/core/kernels/mkl_concat_op.cc @@ -704,14 +704,14 @@ class MklConcatOp : public OpKernel { if (input_tensors[k].NumElements() == 0) continue; - auto src_dims = TFShapeToMklDnnDims( - mkl_input_shapes[k].GetTfShape()); auto src_md = mkl_input_shapes[k].GetMklLayout(); srcs[k].SetUsrMem(src_md, &input_tensors[k]); - if (src_md.data.format != mkl_common_format) + if (src_md.data.format != mkl_common_format) { + memory::dims src_dims(src_md.data.dims, &src_md.data.dims[src_md.data.ndims]); src_md = memory::desc(src_dims, MklDnnType(), mkl_common_format); + } srcs_pd.push_back(memory::primitive_desc(src_md, cpu_engine)); } -- GitLab From 0a60bfccad8df81a57459e4393913b8fb2cb53c1 Mon Sep 17 00:00:00 2001 From: xxxx001 <352172@gmail.com> Date: Tue, 19 Jun 2018 21:41:14 -0700 Subject: [PATCH 036/911] eager env_ or env_RpcRendezvousMgr --- .../eager/eager_service_impl_test.cc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc b/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc index f865ebe1be..ae418287d3 100644 --- a/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc +++ b/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc @@ -122,9 +122,8 @@ tensorflow::FunctionDef MatMulFunction() { TEST(EagerServiceImplTest, BasicTest) { WorkerEnv worker_env; worker_env.env = Env::Default(); - tensorflow::RpcRendezvousMgr rm(&worker_env); - worker_env.rendezvous_mgr = &rm; - + + Status cons_status; TestEagerServiceImpl eager_service_impl(&worker_env); CreateContextRequest request; @@ -132,6 +131,12 @@ TEST(EagerServiceImplTest, BasicTest) { request.mutable_server_def()->set_task_index(0); CreateContextResponse response; + cons_status = eager_service_impl.CreateContext(&request, &response); + EXPECT_EQ(cons_status.error_message(), "invalid eager env_ or env_->rendezvous_mgr."); + + tensorflow::RpcRendezvousMgr rm(&worker_env); + worker_env.rendezvous_mgr = &rm; + TF_ASSERT_OK(eager_service_impl.CreateContext(&request, &response)); uint64 context_id = response.context_id(); -- GitLab From f33189da613fc82a38207fd2716269a42f622e5c Mon Sep 17 00:00:00 2001 From: Rafal Wojdyla Date: Wed, 20 Jun 2018 16:58:40 -0400 Subject: [PATCH 037/911] Update doc and add tests about multi_hot labels with vocabulary --- .../estimator/python/estimator/head.py | 3 ++- .../estimator/python/estimator/head_test.py | 27 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/estimator/python/estimator/head.py b/tensorflow/contrib/estimator/python/estimator/head.py index 9594e5132f..c9d86ef4ab 100644 --- a/tensorflow/contrib/estimator/python/estimator/head.py +++ b/tensorflow/contrib/estimator/python/estimator/head.py @@ -534,7 +534,8 @@ def multi_label_head(n_classes, * An integer `SparseTensor` of class indices. The `dense_shape` must be `[D0, D1, ... DN, ?]` and the values within `[0, n_classes)`. * If `label_vocabulary` is given, a string `SparseTensor`. The `dense_shape` - must be `[D0, D1, ... DN, ?]` and the values within `label_vocabulary`. + must be `[D0, D1, ... DN, ?]` and the values within `label_vocabulary` or a + multi-hot tensor of shape `[D0, D1, ... DN, n_classes]`. If `weight_column` is specified, weights must be of shape `[D0, D1, ... DN]`, or `[D0, D1, ... DN, 1]`. diff --git a/tensorflow/contrib/estimator/python/estimator/head_test.py b/tensorflow/contrib/estimator/python/estimator/head_test.py index b2b57fa06b..7b884402d4 100644 --- a/tensorflow/contrib/estimator/python/estimator/head_test.py +++ b/tensorflow/contrib/estimator/python/estimator/head_test.py @@ -568,6 +568,33 @@ class MultiLabelHead(test.TestCase): expected_loss=expected_loss, expected_metrics=expected_metrics) + def test_eval_with_label_vocabulary_with_multi_hot_input(self): + n_classes = 2 + head = head_lib.multi_label_head( + n_classes, label_vocabulary=['class0', 'class1']) + logits = np.array([[-1., 1.], [-1.5, 1.5]], dtype=np.float32) + labels_multi_hot = np.array([[1, 0], [1, 1]], dtype=np.int64) + # loss = labels * -log(sigmoid(logits)) + + # (1 - labels) * -log(1 - sigmoid(logits)) + # Sum over examples, divide by batch_size. + expected_loss = 0.5 * np.sum( + _sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits)) + keys = metric_keys.MetricKeys + expected_metrics = { + # Average loss over examples. + keys.LOSS_MEAN: expected_loss, + # auc and auc_pr cannot be reliably calculated for only 4 samples, but + # this assert tests that the algorithm remains consistent. + keys.AUC: 0.3333, + keys.AUC_PR: 0.7639, + } + self._test_eval( + head=head, + logits=logits, + labels=labels_multi_hot, + expected_loss=expected_loss, + expected_metrics=expected_metrics) + def test_eval_with_thresholds(self): n_classes = 2 thresholds = [0.25, 0.5, 0.75] -- GitLab From 1cf5178d0ddf32dddc13cdb00783aa0b79974fe8 Mon Sep 17 00:00:00 2001 From: xxxx001 <352172@gmail.com> Date: Wed, 20 Jun 2018 19:50:29 -0700 Subject: [PATCH 038/911] modify eager_service_impl.cc and eager_service_impl_test.cc --- .../core/distributed_runtime/eager/eager_service_impl.cc | 3 ++- .../distributed_runtime/eager/eager_service_impl_test.cc | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc b/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc index ca4ed02d85..428bd7423f 100644 --- a/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc +++ b/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc @@ -80,8 +80,9 @@ Status GetNumRetvals(tensorflow::EagerContext* context, const string& op_name, Status EagerServiceImpl::CreateContext(const CreateContextRequest* request, CreateContextResponse* response) { + //make sure env_ , env_->rendezvous_mgr available if (env_ == nullptr || env_->rendezvous_mgr == nullptr) { - return errors::InvalidArgument("invalid eager env_ or env_->rendezvous_mgr."); + return tensorflow::errors::Internal("invalid eager env_ or env_->rendezvous_mgr."); } tensorflow::RemoteRendezvous* r = env_->rendezvous_mgr->Find(0); std::vector devices; diff --git a/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc b/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc index ae418287d3..f65d129673 100644 --- a/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc +++ b/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc @@ -132,12 +132,16 @@ TEST(EagerServiceImplTest, BasicTest) { CreateContextResponse response; cons_status = eager_service_impl.CreateContext(&request, &response); - EXPECT_EQ(cons_status.error_message(), "invalid eager env_ or env_->rendezvous_mgr."); + + EXPECT_NE(cons_status,Status::OK()); + EXPECT_NE(cons_status.ok(),Status::OK().ok()); tensorflow::RpcRendezvousMgr rm(&worker_env); worker_env.rendezvous_mgr = &rm; - TF_ASSERT_OK(eager_service_impl.CreateContext(&request, &response)); + cons_status = eager_service_impl.CreateContext(&request, &response); + TF_ASSERT_OK(cons_status); + EXPECT_EQ(cons_status,Status::OK()); uint64 context_id = response.context_id(); -- GitLab From cb255e292b6a4378990d02557a37c89d751edb8a Mon Sep 17 00:00:00 2001 From: Christian Ertler Date: Thu, 21 Jun 2018 16:36:49 +0200 Subject: [PATCH 039/911] Adding tf.image.non_max_suppression_overlaps This commit introduces tf.image.non_max_suppression_overlaps. It allows to perform non-max-suppression with an overlap criterion different to IOU by providing a n-by-n matrix with precomputed overlap values for each box pair. --- ...pi_def_NonMaxSuppressionWithOverlaps.pbtxt | 62 +++++ ...pi_def_NonMaxSuppressionWithOverlaps.pbtxt | 4 + .../core/kernels/non_max_suppression_op.cc | 181 +++++++++++--- .../kernels/non_max_suppression_op_test.cc | 229 ++++++++++++++++++ tensorflow/core/ops/image_ops.cc | 32 +++ tensorflow/python/ops/image_ops_impl.py | 42 ++++ 6 files changed, 515 insertions(+), 35 deletions(-) create mode 100644 tensorflow/core/api_def/base_api/api_def_NonMaxSuppressionWithOverlaps.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_NonMaxSuppressionWithOverlaps.pbtxt diff --git a/tensorflow/core/api_def/base_api/api_def_NonMaxSuppressionWithOverlaps.pbtxt b/tensorflow/core/api_def/base_api/api_def_NonMaxSuppressionWithOverlaps.pbtxt new file mode 100644 index 0000000000..180edb15a4 --- /dev/null +++ b/tensorflow/core/api_def/base_api/api_def_NonMaxSuppressionWithOverlaps.pbtxt @@ -0,0 +1,62 @@ +op { + graph_op_name: "NonMaxSuppressionWithOverlaps" + in_arg { + name: "overlaps" + description: <