From 926259c411c1022812ffb7fe88ca61f0180bd778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Thu, 14 Dec 2017 09:51:09 +0800 Subject: [PATCH 001/791] TST: test case for string --- tensorflow/python/kernel_tests/scatter_nd_ops_test.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py index 9f57949515..83d69c651a 100644 --- a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py +++ b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py @@ -364,6 +364,16 @@ class ScatterNdTest(test.TestCase): del input_ # input_ is not used in scatter_nd return array_ops.scatter_nd(indices, updates, shape) + def testString(self): + indices = constant_op.constant([[4], [3], [1], [7]], dtype=dtypes.int32) + updates = constant_op.constant(["four", "three", "one", "seven"], dtype=dtypes.string) + expected = np.array(["", "one", "", "three", "four", "", "", "seven"]) + scatter = self.scatter_nd(indices, updates, shape=(8,)) + + with self.test_session() as sess: + result = sess.run(scatter) + self.assertTrue(np.array_equal(result, expected)) + def testRank3ValidShape(self): indices = array_ops.zeros([2, 2, 2], dtypes.int32) updates = array_ops.zeros([2, 2, 2], dtypes.int32) -- GitLab From 005840c6e2d2a4c25ecd293162a38a79dedf1a4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Thu, 14 Dec 2017 10:06:44 +0800 Subject: [PATCH 002/791] ENH: supports string for cpu --- tensorflow/core/kernels/scatter_nd_op.cc | 1 + tensorflow/core/kernels/scatter_nd_op_cpu_impl.h | 1 + 2 files changed, 2 insertions(+) diff --git a/tensorflow/core/kernels/scatter_nd_op.cc b/tensorflow/core/kernels/scatter_nd_op.cc index 3a95dd1773..0caa7bd317 100644 --- a/tensorflow/core/kernels/scatter_nd_op.cc +++ b/tensorflow/core/kernels/scatter_nd_op.cc @@ -241,6 +241,7 @@ class ScatterNdUpdateOp : public OpKernel { TF_CALL_NUMBER_TYPES(REGISTER_SCATTER_ND_ADD_SUB_CPU); TF_CALL_NUMBER_TYPES(REGISTER_SCATTER_ND_UPDATE_CPU); TF_CALL_NUMBER_TYPES(REGISTER_SCATTER_ND_CPU); +TF_CALL_string(REGISTER_SCATTER_ND_CPU); // Registers GPU kernels. #if GOOGLE_CUDA diff --git a/tensorflow/core/kernels/scatter_nd_op_cpu_impl.h b/tensorflow/core/kernels/scatter_nd_op_cpu_impl.h index cffc326174..155d354d85 100644 --- a/tensorflow/core/kernels/scatter_nd_op_cpu_impl.h +++ b/tensorflow/core/kernels/scatter_nd_op_cpu_impl.h @@ -160,6 +160,7 @@ struct ScatterNdFunctor { REGISTER_SCATTER_ND_INDEX(type, scatter_nd_op::UpdateOp::SUB); TF_CALL_ALL_TYPES(REGISTER_SCATTER_ND_UPDATE); +REGISTER_SCATTER_ND_INDEX(string, scatter_nd_op::UpdateOp::ADD); TF_CALL_NUMBER_TYPES(REGISTER_SCATTER_ND_MATH) #undef REGISTER_SCATTER_ND_MATH -- GitLab From d887d2bcfc819034b17e812a9a60460e2d61e447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Thu, 14 Dec 2017 12:14:40 +0800 Subject: [PATCH 003/791] TST: ignore NonAliasingAdd --- tensorflow/python/kernel_tests/scatter_nd_ops_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py index 83d69c651a..03b2f892c6 100644 --- a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py +++ b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py @@ -594,6 +594,10 @@ class ScatterNdNonAliasingAddTest(ScatterNdTest): shape, dtype=updates.dtype)) return array_ops.scatter_nd_non_aliasing_add(input_, indices, updates) + def testString(self): + # Not supported yet. + pass + if __name__ == "__main__": test.main() -- GitLab From 4b697e0d9472215c706bdb36bb72986cdce78edd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Thu, 14 Dec 2017 13:51:34 +0800 Subject: [PATCH 004/791] DOC: modify document --- tensorflow/core/ops/array_ops.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc index 5a31f433ce..933ebe6b63 100644 --- a/tensorflow/core/ops/array_ops.cc +++ b/tensorflow/core/ops/array_ops.cc @@ -5332,12 +5332,13 @@ REGISTER_OP("ScatterNd") .Attr("Tindices: {int32, int64}") .SetShapeFn(ScatterNdShape) .Doc(R"doc( -Scatter `updates` into a new (initially zero) tensor according to `indices`. +Scatter `updates` into a new (initially zero for numeric, empty for string) +tensor according to `indices`. -Creates a new tensor by applying sparse `updates` to individual -values or slices within a zero tensor of the given `shape` according to -indices. This operator is the inverse of the @{tf.gather_nd} operator which -extracts values or slices from a given tensor. +Creates a new tensor by applying sparse `updates` to individual values or +slices within a zero (or empty string) tensor of the given `shape` +according to indices. This operator is the inverse of the @{tf.gather_nd} +operator which extracts values or slices from a given tensor. **WARNING**: The order in which updates are applied is nondeterministic, so the output will be nondeterministic if `indices` contains duplicates. -- GitLab From 597403e03680d69b72dbfa669f7bbdc77ce21ec9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Wed, 20 Dec 2017 16:34:48 +0800 Subject: [PATCH 005/791] CLN: conform docstring --- tensorflow/core/ops/array_ops.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc index 933ebe6b63..89b6eb7162 100644 --- a/tensorflow/core/ops/array_ops.cc +++ b/tensorflow/core/ops/array_ops.cc @@ -5332,13 +5332,12 @@ REGISTER_OP("ScatterNd") .Attr("Tindices: {int32, int64}") .SetShapeFn(ScatterNdShape) .Doc(R"doc( -Scatter `updates` into a new (initially zero for numeric, empty for string) -tensor according to `indices`. +Scatter `updates` into a new empty tensor according to `indices`. Creates a new tensor by applying sparse `updates` to individual values or -slices within a zero (or empty string) tensor of the given `shape` -according to indices. This operator is the inverse of the @{tf.gather_nd} -operator which extracts values or slices from a given tensor. +slices within a tensor (initially zero for numeric, empty for string) of +the given `shape` according to indices. This operator is the inverse of the +@{tf.gather_nd} operator which extracts values or slices from a given tensor. **WARNING**: The order in which updates are applied is nondeterministic, so the output will be nondeterministic if `indices` contains duplicates. -- GitLab From 736e8c4ccb16718d11cf7c8e1fac843bf6e388a7 Mon Sep 17 00:00:00 2001 From: ManHyuk Date: Wed, 14 Feb 2018 18:26:20 +0900 Subject: [PATCH 006/791] fix typo --- tensorflow/core/lib/io/record_writer.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/lib/io/record_writer.cc b/tensorflow/core/lib/io/record_writer.cc index 3657243c5d..ebc5648269 100644 --- a/tensorflow/core/lib/io/record_writer.cc +++ b/tensorflow/core/lib/io/record_writer.cc @@ -49,7 +49,7 @@ RecordWriterOptions RecordWriterOptions::CreateRecordWriterOptions( #endif // IS_SLIM_BUILD } else if (compression_type != compression::kNone) { LOG(ERROR) << "Unsupported compression_type:" << compression_type - << ". No comprression will be used."; + << ". No compression will be used."; } return options; } -- GitLab From 617fa4e5fa634270c36a2a8762e6ce96bd38f2f8 Mon Sep 17 00:00:00 2001 From: ManHyuk Date: Wed, 14 Feb 2018 18:35:31 +0900 Subject: [PATCH 007/791] fix typo --- tensorflow/contrib/makefile/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/makefile/README.md b/tensorflow/contrib/makefile/README.md index b0228c5435..995230dfa8 100644 --- a/tensorflow/contrib/makefile/README.md +++ b/tensorflow/contrib/makefile/README.md @@ -155,7 +155,7 @@ CC_PREFIX=ccache tensorflow/contrib/makefile/build_all_android.sh -s tensorflow/ (add -T on subsequent builds to skip protobuf downloading/building) -#### Testing the the CUDA-enabled benchmark via adb: +#### Testing the CUDA-enabled benchmark via adb: Build binaries first as above, then run: ```bash -- GitLab From b81aaac898d93e17b4a280bb02547d2a60d490cb Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 15 Feb 2018 08:28:12 +0000 Subject: [PATCH 008/791] Fix warnings in tf.contrib.bayesflow.monte_carlo.expectation This fix fixes several warnings in tf.contrib.bayesflow.monte_carlo.expectation by switching to keepdims for tf.reduce_mean. Signed-off-by: Yong Tang --- tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py b/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py index 985177e897..5263e87ae6 100644 --- a/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py +++ b/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py @@ -328,7 +328,7 @@ def expectation(f, samples, log_prob=None, use_reparametrization=True, if not callable(f): raise ValueError('`f` must be a callable function.') if use_reparametrization: - return math_ops.reduce_mean(f(samples), axis=axis, keep_dims=keep_dims) + return math_ops.reduce_mean(f(samples), axis=axis, keepdims=keep_dims) else: if not callable(log_prob): raise ValueError('`log_prob` must be a callable function.') @@ -348,7 +348,7 @@ def expectation(f, samples, log_prob=None, use_reparametrization=True, # "Is there a floating point value of x, for which x-x == 0 is false?" # http://stackoverflow.com/q/2686644 fx += stop(fx) * (logpx - stop(logpx)) # Add zeros_like(logpx). - return math_ops.reduce_mean(fx, axis=axis, keep_dims=keep_dims) + return math_ops.reduce_mean(fx, axis=axis, keepdims=keep_dims) def _sample_mean(values): -- GitLab From 9c272adf248228408448db6219b238145f5a02ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Fri, 16 Feb 2018 10:38:50 +0800 Subject: [PATCH 009/791] DOC: move doc to api def file --- .../core/api_def/base_api/api_def_ScatterNd.pbtxt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorflow/core/api_def/base_api/api_def_ScatterNd.pbtxt b/tensorflow/core/api_def/base_api/api_def_ScatterNd.pbtxt index 4cb8c064fc..4e95895f54 100644 --- a/tensorflow/core/api_def/base_api/api_def_ScatterNd.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_ScatterNd.pbtxt @@ -25,12 +25,12 @@ A new tensor with the given shape and updates applied according to the indices. END } - summary: "Scatter `updates` into a new (initially zero) tensor according to `indices`." + summary: "Scatter `updates` into a new empty tensor according to `indices`." description: < Date: Mon, 19 Feb 2018 12:56:40 +0400 Subject: [PATCH 010/791] Add broadcasting functionality fro Div and Sub ops. --- tensorflow/contrib/lite/kernels/div.cc | 117 ++++++-- tensorflow/contrib/lite/kernels/div_test.cc | 174 ++++++++++++ .../internal/optimized/optimized_ops.h | 268 +++++++++++++++++- .../internal/reference/reference_ops.h | 257 +++++++++++++++++ tensorflow/contrib/lite/kernels/sub.cc | 135 +++++++-- tensorflow/contrib/lite/kernels/sub_test.cc | 213 ++++++++++++++ .../testing/generated_examples_zip_test.cc | 15 +- 7 files changed, 1122 insertions(+), 57 deletions(-) create mode 100644 tensorflow/contrib/lite/kernels/div_test.cc create mode 100644 tensorflow/contrib/lite/kernels/sub_test.cc diff --git a/tensorflow/contrib/lite/kernels/div.cc b/tensorflow/contrib/lite/kernels/div.cc index 44bd0dc85d..c77a0de9b7 100644 --- a/tensorflow/contrib/lite/kernels/div.cc +++ b/tensorflow/contrib/lite/kernels/div.cc @@ -37,7 +37,23 @@ constexpr int kInputTensor1 = 0; constexpr int kInputTensor2 = 1; constexpr int kOutputTensor = 0; +struct OpData { + bool requires_broadcast; +}; + +void* Init(TfLiteContext* context, const char* buffer, size_t length) { + auto* data = new OpData; + data->requires_broadcast = false; + return data; +} + +void Free(TfLiteContext* context, void* buffer) { + delete reinterpret_cast(buffer); +} + TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + OpData* data = reinterpret_cast(node->user_data); + TF_LITE_ENSURE_EQ(context, NumInputs(node), 2); TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); @@ -45,35 +61,85 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TfLiteTensor* input2 = GetInput(context, node, kInputTensor2); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); - TF_LITE_ENSURE_EQ(context, NumDimensions(input1), NumDimensions(input2)); - for (int i = 0; i < NumDimensions(input1); ++i) { - TF_LITE_ENSURE_EQ(context, SizeOfDimension(input1, i), - SizeOfDimension(input2, i)); - } + TF_LITE_ENSURE_EQ(context, input1->type, input2->type); + output->type = input2->type; + + data->requires_broadcast = !HaveSameShapes(input1, input2); - TF_LITE_ENSURE_EQ(context, input1->type, output->type); - TF_LITE_ENSURE_EQ(context, input2->type, output->type); + TfLiteIntArray* output_size = nullptr; + if (data->requires_broadcast) { + TF_LITE_ENSURE_OK(context, CalculateShapeForBroadcast( + context, input1, input2, &output_size)); + } else { + output_size = TfLiteIntArrayCopy(input1->dims); + } - TfLiteIntArray* output_size = TfLiteIntArrayCopy(input1->dims); return context->ResizeTensor(context, output, output_size); } template -void EvalDivFloat(TfLiteContext* context, TfLiteNode* node, - TfLiteDivParams* params, TfLiteTensor* input1, - TfLiteTensor* input2, TfLiteTensor* output) { +void EvalFloat(TfLiteContext* context, TfLiteNode* node, + TfLiteDivParams* params, const OpData* data, + TfLiteTensor* input1, TfLiteTensor* input2, + TfLiteTensor* output) { float output_activation_min, output_activation_max; CalculateActivationRangeFloat(params->activation, &output_activation_min, &output_activation_max); -#define TF_LITE_DIV(type) \ - type::Div(GetTensorData(input1), GetTensorDims(input1), \ - GetTensorData(input2), GetTensorDims(input2), \ - output_activation_min, output_activation_max, \ - GetTensorData(output), GetTensorDims(output)) +#define TF_LITE_DIV(type, opname) \ + type::opname(GetTensorData(input1), GetTensorDims(input1), \ + GetTensorData(input2), GetTensorDims(input2), \ + output_activation_min, output_activation_max, \ + GetTensorData(output), GetTensorDims(output)) + if (kernel_type == kReference) { + if (data->requires_broadcast) { + TF_LITE_DIV(reference_ops, BroadcastDiv); + } else { + TF_LITE_DIV(reference_ops, Div); + } + } else { + if (data->requires_broadcast) { + TF_LITE_DIV(optimized_ops, BroadcastDiv); + } else { + TF_LITE_DIV(optimized_ops, Div); + } + } +#undef TF_LITE_DIV +} + +template +void EvalQuantized(TfLiteContext* context, TfLiteNode* node, + TfLiteDivParams* params, const OpData* data, + TfLiteTensor* input1, TfLiteTensor* input2, + TfLiteTensor* output) { + auto input1_offset = -input1->params.zero_point; + auto input2_offset = -input2->params.zero_point; + auto output_offset = output->params.zero_point; + + int32_t output_multiplier; + int output_shift; + + double real_multiplier = + input1->params.scale * input2->params.scale / output->params.scale; + QuantizeMultiplierSmallerThanOne(real_multiplier, &output_multiplier, + &output_shift); + + int32 output_activation_min, output_activation_max; + CalculateActivationRangeUint8(params->activation, output, + &output_activation_min, &output_activation_max); + +#define TF_LITE_DIV(type, opname) \ + type::opname(GetTensorData(input1), GetTensorDims(input1), \ + input1_offset, GetTensorData(input2), \ + GetTensorDims(input2), input2_offset, output_offset, \ + output_multiplier, output_shift, output_activation_min, \ + output_activation_max, GetTensorData(output), \ + GetTensorDims(output)); + // The quantized version of Div doesn't support activations, so we + // always use BroadcastDiv. if (kernel_type == kReference) { - TF_LITE_DIV(reference_ops); + TF_LITE_DIV(reference_ops, BroadcastDiv); } else { - TF_LITE_DIV(optimized_ops); + TF_LITE_DIV(optimized_ops, BroadcastDiv); } #undef TF_LITE_DIV } @@ -81,15 +147,20 @@ void EvalDivFloat(TfLiteContext* context, TfLiteNode* node, template TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { auto* params = reinterpret_cast(node->builtin_data); + OpData* data = reinterpret_cast(node->user_data); TfLiteTensor* input1 = GetInput(context, node, kInputTensor1); TfLiteTensor* input2 = GetInput(context, node, kInputTensor2); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); if (output->type == kTfLiteFloat32) { - EvalDivFloat(context, node, params, input1, input2, output); + EvalFloat(context, node, params, data, input1, input2, output); + } else if (output->type == kTfLiteUInt8) { + EvalQuantized(context, node, params, data, input1, input2, + output); } else { - context->ReportError(context, "Inputs and outputs not all float types."); + context->ReportError(context, + "Div only supports FLOAT32 and quantized UINT8 now."); return kTfLiteError; } @@ -99,19 +170,19 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace div TfLiteRegistration* Register_DIV_REF() { - static TfLiteRegistration r = {nullptr, nullptr, div::Prepare, + static TfLiteRegistration r = {div::Init, div::Free, div::Prepare, div::Eval}; return &r; } TfLiteRegistration* Register_DIV_GENERIC_OPT() { - static TfLiteRegistration r = {nullptr, nullptr, div::Prepare, + static TfLiteRegistration r = {div::Init, div::Free, div::Prepare, div::Eval}; return &r; } TfLiteRegistration* Register_DIV_NEON_OPT() { - static TfLiteRegistration r = {nullptr, nullptr, div::Prepare, + static TfLiteRegistration r = {div::Init, div::Free, div::Prepare, div::Eval}; return &r; } diff --git a/tensorflow/contrib/lite/kernels/div_test.cc b/tensorflow/contrib/lite/kernels/div_test.cc new file mode 100644 index 0000000000..78918a0d79 --- /dev/null +++ b/tensorflow/contrib/lite/kernels/div_test.cc @@ -0,0 +1,174 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include "tensorflow/contrib/lite/interpreter.h" +#include "tensorflow/contrib/lite/kernels/register.h" +#include "tensorflow/contrib/lite/kernels/test_util.h" +#include "tensorflow/contrib/lite/model.h" + +namespace tflite { +namespace { + +using ::testing::ElementsAreArray; + +class BaseDivOpModel : public SingleOpModel { + public: + BaseDivOpModel(const TensorData& input1, const TensorData& input2, + const TensorData& output, + ActivationFunctionType activation_type) { + input1_ = AddInput(input1); + input2_ = AddInput(input2); + output_ = AddOutput(output); + SetBuiltinOp(BuiltinOperator_DIV, BuiltinOptions_DivOptions, + CreateDivOptions(builder_, activation_type).Union()); + BuildInterpreter({GetShape(input1_), GetShape(input2_)}); + } + + int input1() { return input1_; } + int input2() { return input2_; } + + protected: + int input1_; + int input2_; + int output_; +}; + +class FloatDivOpModel : public BaseDivOpModel { + public: + using BaseDivOpModel::BaseDivOpModel; + + std::vector GetOutput() { return ExtractVector(output_); } +}; + +// For quantized Div, the error shouldn't exceed (2*step + step^2). +// The param min=-1.0 & max=1.0 is used in the following tests. +// The tolerance value is ~0.0157. +const float kQuantizedStep = 2.0 / 255.0; +const float kQuantizedTolerance = + 2.0 * kQuantizedStep + kQuantizedStep * kQuantizedStep; + +class QuantizedDivOpModel : public BaseDivOpModel { + public: + using BaseDivOpModel::BaseDivOpModel; + + std::vector GetDequantizedOutput() { + return Dequantize(ExtractVector(output_), + GetScale(output_), GetZeroPoint(output_)); + } +}; + +TEST(FloatDivOpTest, NoActivation) { + FloatDivOpModel m({TensorType_FLOAT32, {1, 2, 2, 1}}, + {TensorType_FLOAT32, {1, 2, 2, 1}}, + {TensorType_FLOAT32, {}}, ActivationFunctionType_NONE); + m.PopulateTensor(m.input1(), {-0.2, 0.2, -1.2, 0.8}); + m.PopulateTensor(m.input2(), {0.5, 0.2, -1.5, 0.5}); + m.Invoke(); + EXPECT_THAT(m.GetOutput(), + ElementsAreArray(ArrayFloatNear({-0.4, 1.0, 0.8, 1.6}))); +} + +TEST(FloatDivOpTest, ActivationRELU_N1_TO_1) { + FloatDivOpModel m( + {TensorType_FLOAT32, {1, 2, 2, 1}}, {TensorType_FLOAT32, {1, 2, 2, 1}}, + {TensorType_FLOAT32, {}}, ActivationFunctionType_RELU_N1_TO_1); + m.PopulateTensor(m.input1(), {-0.2, 0.2, -1.2, 0.8}); + m.PopulateTensor(m.input2(), {0.1, 0.2, -1.5, 0.5}); + m.Invoke(); + EXPECT_THAT(m.GetOutput(), + ElementsAreArray(ArrayFloatNear({-1.0, 1.0, 0.8, 1.0}))); +} + +TEST(FloatDivOpTest, VariousInputShapes) { + std::vector> test_shapes = { + {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}}; + for (int i = 0; i < test_shapes.size(); ++i) { + FloatDivOpModel m({TensorType_FLOAT32, test_shapes[i]}, + {TensorType_FLOAT32, test_shapes[i]}, + {TensorType_FLOAT32, {}}, ActivationFunctionType_NONE); + m.PopulateTensor(m.input1(), {-2.0, 0.2, 0.3, 0.8, 1.1, -2.0}); + m.PopulateTensor(m.input2(), {0.1, 0.2, 0.6, 0.5, -1.1, -0.1}); + m.Invoke(); + EXPECT_THAT( + m.GetOutput(), + ElementsAreArray(ArrayFloatNear({-20.0, 1.0, 0.5, 1.6, -1.0, 20.0}))) + << "With shape number " << i; + } +} + +TEST(FloatDivOpTest, WithBroadcast) { + std::vector> test_shapes = { + {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}}; + for (int i = 0; i < test_shapes.size(); ++i) { + FloatDivOpModel m({TensorType_FLOAT32, test_shapes[i]}, + {TensorType_FLOAT32, {}}, // always a scalar + {TensorType_FLOAT32, {}}, ActivationFunctionType_NONE); + m.PopulateTensor(m.input1(), {-0.2, 0.2, 0.07, 0.08, 0.11, -0.123}); + m.PopulateTensor(m.input2(), {0.1}); + m.Invoke(); + EXPECT_THAT( + m.GetOutput(), + ElementsAreArray(ArrayFloatNear({-2.0, 2.0, 0.7, 0.8, 1.1, -1.23}))) + << "With shape number " << i; + } +} + +TEST(QuantizedDivOpTest, NoActivation) { + QuantizedDivOpModel m({TensorType_UINT8, {1, 2, 2, 1}, -1.0, 1.0}, + {TensorType_UINT8, {1, 2, 2, 1}, -1.0, 1.0}, + {TensorType_UINT8, {}, -1.0, 1.0}, + ActivationFunctionType_NONE); + m.QuantizeAndPopulate(m.input1(), {-0.6, 0.2, 0.9, -0.7}); + m.QuantizeAndPopulate(m.input2(), {0.8, 0.4, 0.9, -0.8}); + m.Invoke(); + EXPECT_THAT(m.GetDequantizedOutput(), + ElementsAreArray(ArrayFloatNear({-0.75, 0.5, 1.0, 0.875}, + kQuantizedTolerance))); +} + +// for quantized Div, the error shouldn't exceed 2*step +float GetTolerance(int min, int max) { + float kQuantizedStep = (max - min) / 255.0; + float kQuantizedTolerance = 2.0 * kQuantizedStep; + return kQuantizedTolerance; +} + +TEST(QuantizedDivOpTest, WithBroadcast) { + float kQuantizedTolerance = GetTolerance(-3.0, 3.0); + std::vector> test_shapes = { + {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}}; + for (int i = 0; i < test_shapes.size(); ++i) { + QuantizedDivOpModel m({TensorType_UINT8, test_shapes[i], -3.0, 3.0}, + {TensorType_UINT8, {}, -3.0, 3.0}, // always a scalar + {TensorType_UINT8, {}, -3.0, 3.0}, + ActivationFunctionType_NONE); + m.QuantizeAndPopulate(m.input1(), {-0.2, 0.2, 0.07, 0.08, 0.11, -0.123}); + m.QuantizeAndPopulate(m.input2(), {0.1}); + m.Invoke(); + EXPECT_THAT(m.GetDequantizedOutput(), + ElementsAreArray(ArrayFloatNear( + {-2.0, 2.0, 0.7, 0.8, 1.1, -1.23}, kQuantizedTolerance))) + << "With shape number " << i; + } +} + +} // namespace +} // namespace tflite + +int main(int argc, char** argv) { + ::tflite::LogToStderr(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h index dec58fea4f..d12a3eca1d 100644 --- a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h +++ b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h @@ -1928,6 +1928,126 @@ inline void Div(const float* input1_data, const Dims<4>& input1_dims, } } +// TODO(jiawen): We can implement BroadcastDiv on buffers of arbitrary +// dimensionality if the runtime code does a single loop over one dimension +// that handles broadcasting as the base case. The code generator would then +// generate max(D1, D2) nested for loops. +// TODO(benoitjacob): BroadcastDiv is intentionally duplicated from +// reference_ops.h. Once an optimized version is implemented and NdArrayDesc +// is no longer referenced in this file, move NdArrayDesc from types.h to +// reference_ops.h. +template +void BroadcastDiv(const T* input1_data, const Dims<4>& input1_dims, + const T* input2_data, const Dims<4>& input2_dims, + T output_activation_min, T output_activation_max, + T* output_data, const Dims<4>& output_dims) { + gemmlowp::ScopedProfilingLabel label("BroadcastDiv"); + + NdArrayDesc<4> desc1; + NdArrayDesc<4> desc2; + NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2); + + // In Tensorflow, the dimensions are canonically named (batch_number, row, + // col, channel), with extents (batches, height, width, depth), with the + // trailing dimension changing most rapidly (channels has the smallest stride, + // typically 1 element). + // + // In generated C code, we store arrays with the dimensions reversed. The + // first dimension has smallest stride. + // + // We name our variables by their Tensorflow convention, but generate C code + // nesting loops such that the innermost loop has the smallest stride for the + // best cache behavior. + for (int b = 0; b < ArraySize(output_dims, 3); ++b) { + for (int y = 0; y < ArraySize(output_dims, 2); ++y) { + for (int x = 0; x < ArraySize(output_dims, 1); ++x) { + for (int c = 0; c < ArraySize(output_dims, 0); ++c) { + output_data[Offset(output_dims, c, x, y, b)] = + ActivationFunctionWithMinMax( + input1_data[SubscriptToIndex(desc1, c, x, y, b)] / + input2_data[SubscriptToIndex(desc2, c, x, y, b)], + output_activation_min, output_activation_max); + } + } + } + } +} + +// legacy, for compatibility with old checked-in code +template +void BroadcastDiv(const T* input1_data, const Dims<4>& input1_dims, + const T* input2_data, const Dims<4>& input2_dims, + T* output_data, const Dims<4>& output_dims) { + T output_activation_min, output_activation_max; + GetActivationMinMax(Ac, &output_activation_min, &output_activation_max); + + BroadcastDiv(input1_data, input1_dims, input2_data, input2_dims, + output_activation_min, output_activation_max, output_data, + output_dims); +} + +inline void BroadcastDiv(const uint8* input1_data, const Dims<4>& input1_dims, + int32 input1_offset, const uint8* input2_data, + const Dims<4>& input2_dims, int32 input2_offset, + int32 output_offset, int32 output_multiplier, + int output_shift, int32 output_activation_min, + int32 output_activation_max, uint8* output_data, + const Dims<4>& output_dims) { + gemmlowp::ScopedProfilingLabel label("BroadcastDiv/8bit"); + + NdArrayDesc<4> desc1; + NdArrayDesc<4> desc2; + NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2); + + // In Tensorflow, the dimensions are canonically named (batch_number, row, + // col, channel), with extents (batches, height, width, depth), with the + // trailing dimension changing most rapidly (channels has the smallest stride, + // typically 1 element). + // + // In generated C code, we store arrays with the dimensions reversed. The + // first dimension has smallest stride. + // + // We name our variables by their Tensorflow convention, but generate C code + // nesting loops such that the innermost loop has the smallest stride for the + // best cache behavior. + for (int b = 0; b < ArraySize(output_dims, 3); ++b) { + for (int y = 0; y < ArraySize(output_dims, 2); ++y) { + for (int x = 0; x < ArraySize(output_dims, 1); ++x) { + for (int c = 0; c < ArraySize(output_dims, 0); ++c) { + const int32 input1_val = + input1_offset + input1_data[SubscriptToIndex(desc1, c, x, y, b)]; + const int32 input2_val = + input2_offset + input2_data[SubscriptToIndex(desc2, c, x, y, b)]; + const int32 unclamped_result = + output_offset + + MultiplyByQuantizedMultiplierSmallerThanOne( + input1_val / input2_val, output_multiplier, output_shift); + const int32 clamped_output = + std::min(output_activation_max, + std::max(output_activation_min, unclamped_result)); + output_data[Offset(output_dims, c, x, y, b)] = + static_cast(clamped_output); + } + } + } + } +} + +// legacy, for compatibility with old checked-in code +template +inline void BroadcastDiv(const uint8* input1_data, const Dims<4>& input1_dims, + int32 input1_offset, const uint8* input2_data, + const Dims<4>& input2_dims, int32 input2_offset, + int32 output_offset, int32 output_multiplier, + int output_shift, int32 output_activation_min, + int32 output_activation_max, uint8* output_data, + const Dims<4>& output_dims) { + BroadcastDiv(input1_data, input1_dims, input1_offset, input2_data, + input2_dims, input2_offset, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, + output_data, output_dims); +} + // TODO(aselle): This is not actually optimized yet. inline void Sub(const float* input1_data, const Dims<4>& input1_dims, const float* input2_data, const Dims<4>& input2_dims, @@ -1955,6 +2075,152 @@ inline void Sub(const float* input1_data, const Dims<4>& input1_dims, } } } + +// TODO(jiawen): We can implement BroadcastSub on buffers of arbitrary +// dimensionality if the runtime code does a single loop over one dimension +// that handles broadcasting as the base case. The code generator would then +// generate max(D1, D2) nested for loops. +// TODO(benoitjacob): BroadcastSub is intentionally duplicated from +// reference_ops.h. Once an optimized version is implemented and NdArrayDesc +// is no longer referenced in this file, move NdArrayDesc from types.h to +// reference_ops.h. +template +void BroadcastSub(const T* input1_data, const Dims<4>& input1_dims, + const T* input2_data, const Dims<4>& input2_dims, + T output_activation_min, T output_activation_max, + T* output_data, const Dims<4>& output_dims) { + gemmlowp::ScopedProfilingLabel label("BroadcastSub"); + + NdArrayDesc<4> desc1; + NdArrayDesc<4> desc2; + NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2); + + // In Tensorflow, the dimensions are canonically named (batch_number, row, + // col, channel), with extents (batches, height, width, depth), with the + // trailing dimension changing most rapidly (channels has the smallest stride, + // typically 1 element). + // + // In generated C code, we store arrays with the dimensions reversed. The + // first dimension has smallest stride. + // + // We name our variables by their Tensorflow convention, but generate C code + // nesting loops such that the innermost loop has the smallest stride for the + // best cache behavior. + for (int b = 0; b < ArraySize(output_dims, 3); ++b) { + for (int y = 0; y < ArraySize(output_dims, 2); ++y) { + for (int x = 0; x < ArraySize(output_dims, 1); ++x) { + for (int c = 0; c < ArraySize(output_dims, 0); ++c) { + output_data[Offset(output_dims, c, x, y, b)] = + ActivationFunctionWithMinMax( + input1_data[SubscriptToIndex(desc1, c, x, y, b)] - + input2_data[SubscriptToIndex(desc2, c, x, y, b)], + output_activation_min, output_activation_max); + } + } + } + } +} + +// legacy, for compatibility with old checked-in code +template +void BroadcastSub(const T* input1_data, const Dims<4>& input1_dims, + const T* input2_data, const Dims<4>& input2_dims, + T* output_data, const Dims<4>& output_dims) { + T output_activation_min, output_activation_max; + GetActivationMinMax(Ac, &output_activation_min, &output_activation_max); + + BroadcastSub(input1_data, input1_dims, input2_data, input2_dims, + output_activation_min, output_activation_max, output_data, + output_dims); +} + +inline void BroadcastSub(int left_shift, const uint8* input1_data, + const Dims<4>& input1_dims, int32 input1_offset, + int32 input1_multiplier, int input1_shift, + const uint8* input2_data, const Dims<4>& input2_dims, + int32 input2_offset, int32 input2_multiplier, + int input2_shift, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_data, + const Dims<4>& output_dims) { + gemmlowp::ScopedProfilingLabel label("BroadcastSub/8bit"); + + NdArrayDesc<4> desc1; + NdArrayDesc<4> desc2; + NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2); + + // In Tensorflow, the dimensions are canonically named (batch_number, row, + // col, channel), with extents (batches, height, width, depth), with the + // trailing dimension changing most rapidly (channels has the smallest stride, + // typically 1 element). + // + // In generated C code, we store arrays with the dimensions reversed. The + // first dimension has smallest stride. + // + // We name our variables by their Tensorflow convention, but generate C code + // nesting loops such that the innermost loop has the smallest stride for the + // best cache behavior. + for (int b = 0; b < ArraySize(output_dims, 3); ++b) { + for (int y = 0; y < ArraySize(output_dims, 2); ++y) { + for (int x = 0; x < ArraySize(output_dims, 1); ++x) { + for (int c = 0; c < ArraySize(output_dims, 0); ++c) { + const int32 input1_val = + input1_offset + input1_data[SubscriptToIndex(desc1, c, x, y, b)]; + const int32 input2_val = + input2_offset + input2_data[SubscriptToIndex(desc2, c, x, y, b)]; + const int32 shifted_input1_val = input1_val * (1 << left_shift); + const int32 shifted_input2_val = input2_val * (1 << left_shift); + const int32 scaled_input1_val = + MultiplyByQuantizedMultiplierSmallerThanOne( + shifted_input1_val, input1_multiplier, input1_shift); + const int32 scaled_input2_val = + MultiplyByQuantizedMultiplierSmallerThanOne( + shifted_input2_val, input2_multiplier, input2_shift); + const int32 raw_sum = scaled_input1_val - scaled_input2_val; + const int32 raw_output = + MultiplyByQuantizedMultiplierSmallerThanOne( + raw_sum, output_multiplier, output_shift) + + output_offset; + const int32 clamped_output = + std::min(output_activation_max, + std::max(output_activation_min, raw_output)); + output_data[Offset(output_dims, c, x, y, b)] = + static_cast(clamped_output); + } + } + } + } +} + +template +inline void BroadcastSub(int left_shift, const uint8* input1_data, + const Dims<4>& input1_dims, int32 input1_offset, + int32 input1_multiplier, int input1_shift, + const uint8* input2_data, const Dims<4>& input2_dims, + int32 input2_offset, int32 input2_multiplier, + int input2_shift, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_data, + const Dims<4>& output_dims) { + static_assert(Ac == FusedActivationFunctionType::kNone || + Ac == FusedActivationFunctionType::kRelu || + Ac == FusedActivationFunctionType::kRelu6 || + Ac == FusedActivationFunctionType::kRelu1, + ""); + TFLITE_DCHECK_LE(output_activation_min, output_activation_max); + if (Ac == FusedActivationFunctionType::kNone) { + TFLITE_DCHECK_EQ(output_activation_min, 0); + TFLITE_DCHECK_EQ(output_activation_max, 255); + } + BroadcastSub(left_shift, input1_data, input1_dims, input1_offset, + input1_multiplier, input1_shift, input2_data, input2_dims, + input2_offset, input2_multiplier, input2_shift, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_data, output_dims); +} + template void Concatenation(int concat_dim, const Scalar* const* input_data, const Dims<4>* const* input_dims, int inputs_count, @@ -2866,7 +3132,7 @@ inline void Softmax(const uint8* input_data, const Dims<4>& input_dims, using FixedPointAccum = gemmlowp::FixedPoint; using FixedPoint0 = gemmlowp::FixedPoint; - gemmlowp::ScopedProfilingLabel label("Softmax/8bit"); +gemmlowp::ScopedProfilingLabel label("Softmax/8bit"); const int batches = MatchingArraySize(input_dims, 3, output_dims, 3); const int height = MatchingArraySize(input_dims, 2, output_dims, 2); const int width = MatchingArraySize(input_dims, 1, output_dims, 1); diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h index 5f4d5be323..c7b7687622 100644 --- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h +++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h @@ -1208,6 +1208,122 @@ inline void Div(const float* input1_data, const Dims<4>& input1_dims, } } +// TODO(jiawen): We can implement BroadcastDiv on buffers of arbitrary +// dimensionality if the runtime code does a single loop over one dimension +// that handles broadcasting as the base case. The code generator would then +// generate max(D1, D2) nested for loops. +template +void BroadcastDiv(const T* input1_data, const Dims<4>& input1_dims, + const T* input2_data, const Dims<4>& input2_dims, + T output_activation_min, T output_activation_max, + T* output_data, const Dims<4>& output_dims) { + gemmlowp::ScopedProfilingLabel label("BroadcastDiv"); + + NdArrayDesc<4> desc1; + NdArrayDesc<4> desc2; + NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2); + + // In Tensorflow, the dimensions are canonically named (batch_number, row, + // col, channel), with extents (batches, height, width, depth), with the + // trailing dimension changing most rapidly (channels has the smallest + // stride, typically 1 element). + // + // In generated C code, we store arrays with the dimensions reversed. The + // first dimension has smallest stride. + // + // We name our variables by their Tensorflow convention, but generate C code + // nesting loops such that the innermost loop has the smallest stride for + // the best cache behavior. + for (int b = 0; b < ArraySize(output_dims, 3); ++b) { + for (int y = 0; y < ArraySize(output_dims, 2); ++y) { + for (int x = 0; x < ArraySize(output_dims, 1); ++x) { + for (int c = 0; c < ArraySize(output_dims, 0); ++c) { + output_data[Offset(output_dims, c, x, y, b)] = + ActivationFunctionWithMinMax( + input1_data[SubscriptToIndex(desc1, c, x, y, b)] / + input2_data[SubscriptToIndex(desc2, c, x, y, b)], + output_activation_min, output_activation_max); + } + } + } + } +} + +// legacy, for compatibility with old checked-in code +template +void BroadcastDiv(const T* input1_data, const Dims<4>& input1_dims, + const T* input2_data, const Dims<4>& input2_dims, + T* output_data, const Dims<4>& output_dims) { + T output_activation_min, output_activation_max; + GetActivationMinMax(Ac, &output_activation_min, &output_activation_max); + + BroadcastDiv(input1_data, input1_dims, input2_data, input2_dims, + output_activation_min, output_activation_max, output_data, + output_dims); +} + +inline void BroadcastDiv(const uint8* input1_data, const Dims<4>& input1_dims, + int32 input1_offset, const uint8* input2_data, + const Dims<4>& input2_dims, int32 input2_offset, + int32 output_offset, int32 output_multiplier, + int output_shift, int32 output_activation_min, + int32 output_activation_max, uint8* output_data, + const Dims<4>& output_dims) { + gemmlowp::ScopedProfilingLabel label("BroadcastDiv/8bit"); + + NdArrayDesc<4> desc1; + NdArrayDesc<4> desc2; + NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2); + + // In Tensorflow, the dimensions are canonically named (batch_number, row, + // col, channel), with extents (batches, height, width, depth), with the + // trailing dimension changing most rapidly (channels has the smallest + // stride, typically 1 element). + // + // In generated C code, we store arrays with the dimensions reversed. The + // first dimension has smallest stride. + // + // We name our variables by their Tensorflow convention, but generate C code + // nesting loops such that the innermost loop has the smallest stride for + // the best cache behavior. + for (int b = 0; b < ArraySize(output_dims, 3); ++b) { + for (int y = 0; y < ArraySize(output_dims, 2); ++y) { + for (int x = 0; x < ArraySize(output_dims, 1); ++x) { + for (int c = 0; c < ArraySize(output_dims, 0); ++c) { + const int32 input1_val = + input1_offset + input1_data[SubscriptToIndex(desc1, c, x, y, b)]; + const int32 input2_val = + input2_offset + input2_data[SubscriptToIndex(desc2, c, x, y, b)]; + const int32 unclamped_result = + output_offset + + MultiplyByQuantizedMultiplierSmallerThanOne( + input1_val / input2_val, output_multiplier, output_shift); + const int32 clamped_output = + std::min(output_activation_max, + std::max(output_activation_min, unclamped_result)); + output_data[Offset(output_dims, c, x, y, b)] = + static_cast(clamped_output); + } + } + } + } +} + +// legacy, for compatibility with old checked-in code +template +inline void BroadcastDiv(const uint8* input1_data, const Dims<4>& input1_dims, + int32 input1_offset, const uint8* input2_data, + const Dims<4>& input2_dims, int32 input2_offset, + int32 output_offset, int32 output_multiplier, + int output_shift, int32 output_activation_min, + int32 output_activation_max, uint8* output_data, + const Dims<4>& output_dims) { + BroadcastDiv(input1_data, input1_dims, input1_offset, input2_data, + input2_dims, input2_offset, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, + output_data, output_dims); +} + inline void Sub(const float* input1_data, const Dims<4>& input1_dims, const float* input2_data, const Dims<4>& input2_dims, float output_activation_min, float output_activation_max, @@ -1235,6 +1351,147 @@ inline void Sub(const float* input1_data, const Dims<4>& input1_dims, } } +// TODO(jiawen): We can implement BroadcastSub on buffers of arbitrary +// dimensionality if the runtime code does a single loop over one dimension +// that handles broadcasting as the base case. The code generator would then +// generate max(D1, D2) nested for loops. +template +void BroadcastSub(const T* input1_data, const Dims<4>& input1_dims, + const T* input2_data, const Dims<4>& input2_dims, + T output_activation_min, T output_activation_max, + T* output_data, const Dims<4>& output_dims) { + gemmlowp::ScopedProfilingLabel label("BroadcastSub"); + + NdArrayDesc<4> desc1; + NdArrayDesc<4> desc2; + NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2); + + // In Tensorflow, the dimensions are canonically named (batch_number, row, + // col, channel), with extents (batches, height, width, depth), with the + // trailing dimension changing most rapidly (channels has the smallest stride, + // typically 1 element). + // + // In generated C code, we store arrays with the dimensions reversed. The + // first dimension has smallest stride. + // + // We name our variables by their Tensorflow convention, but generate C code + // nesting loops such that the innermost loop has the smallest stride for the + // best cache behavior. + for (int b = 0; b < ArraySize(output_dims, 3); ++b) { + for (int y = 0; y < ArraySize(output_dims, 2); ++y) { + for (int x = 0; x < ArraySize(output_dims, 1); ++x) { + for (int c = 0; c < ArraySize(output_dims, 0); ++c) { + output_data[Offset(output_dims, c, x, y, b)] = + ActivationFunctionWithMinMax( + input1_data[SubscriptToIndex(desc1, c, x, y, b)] - + input2_data[SubscriptToIndex(desc2, c, x, y, b)], + output_activation_min, output_activation_max); + } + } + } + } +} + +// legacy, for compatibility with old checked-in code +template +void BroadcastSub(const T* input1_data, const Dims<4>& input1_dims, + const T* input2_data, const Dims<4>& input2_dims, + T* output_data, const Dims<4>& output_dims) { + T output_activation_min, output_activation_max; + GetActivationMinMax(Ac, &output_activation_min, &output_activation_max); + + BroadcastSub(input1_data, input1_dims, input2_data, input2_dims, + output_activation_min, output_activation_max, output_data, + output_dims); +} + +inline void BroadcastSub(int left_shift, const uint8* input1_data, + const Dims<4>& input1_dims, int32 input1_offset, + int32 input1_multiplier, int input1_shift, + const uint8* input2_data, const Dims<4>& input2_dims, + int32 input2_offset, int32 input2_multiplier, + int input2_shift, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_data, + const Dims<4>& output_dims) { + gemmlowp::ScopedProfilingLabel label("BroadcastSub/8bit"); + + NdArrayDesc<4> desc1; + NdArrayDesc<4> desc2; + NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2); + + // In Tensorflow, the dimensions are canonically named (batch_number, row, + // col, channel), with extents (batches, height, width, depth), with the + // trailing dimension changing most rapidly (channels has the smallest stride, + // typically 1 element). + // + // In generated C code, we store arrays with the dimensions reversed. The + // first dimension has smallest stride. + // + // We name our variables by their Tensorflow convention, but generate C code + // nesting loops such that the innermost loop has the smallest stride for the + // best cache behavior. + for (int b = 0; b < ArraySize(output_dims, 3); ++b) { + for (int y = 0; y < ArraySize(output_dims, 2); ++y) { + for (int x = 0; x < ArraySize(output_dims, 1); ++x) { + for (int c = 0; c < ArraySize(output_dims, 0); ++c) { + const int32 input1_val = + input1_offset + input1_data[SubscriptToIndex(desc1, c, x, y, b)]; + const int32 input2_val = + input2_offset + input2_data[SubscriptToIndex(desc2, c, x, y, b)]; + const int32 shifted_input1_val = input1_val * (1 << left_shift); + const int32 shifted_input2_val = input2_val * (1 << left_shift); + const int32 scaled_input1_val = + MultiplyByQuantizedMultiplierSmallerThanOne( + shifted_input1_val, input1_multiplier, input1_shift); + const int32 scaled_input2_val = + MultiplyByQuantizedMultiplierSmallerThanOne( + shifted_input2_val, input2_multiplier, input2_shift); + const int32 raw_sum = scaled_input1_val - scaled_input2_val; + const int32 raw_output = + MultiplyByQuantizedMultiplierSmallerThanOne( + raw_sum, output_multiplier, output_shift) + + output_offset; + const int32 clamped_output = + std::min(output_activation_max, + std::max(output_activation_min, raw_output)); + output_data[Offset(output_dims, c, x, y, b)] = + static_cast(clamped_output); + } + } + } + } +} + +template +inline void BroadcastSub(int left_shift, const uint8* input1_data, + const Dims<4>& input1_dims, int32 input1_offset, + int32 input1_multiplier, int input1_shift, + const uint8* input2_data, const Dims<4>& input2_dims, + int32 input2_offset, int32 input2_multiplier, + int input2_shift, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_data, + const Dims<4>& output_dims) { + static_assert(Ac == FusedActivationFunctionType::kNone || + Ac == FusedActivationFunctionType::kRelu || + Ac == FusedActivationFunctionType::kRelu6 || + Ac == FusedActivationFunctionType::kRelu1, + ""); + TFLITE_DCHECK_LE(output_activation_min, output_activation_max); + if (Ac == FusedActivationFunctionType::kNone) { + TFLITE_DCHECK_EQ(output_activation_min, 0); + TFLITE_DCHECK_EQ(output_activation_max, 255); + } + BroadcastSub(left_shift, input1_data, input1_dims, input1_offset, + input1_multiplier, input1_shift, input2_data, input2_dims, + input2_offset, input2_multiplier, input2_shift, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_data, output_dims); +} + template void Concatenation(int concat_dim, const Scalar* const* input_data, const Dims<4>* const* input_dims, int inputs_count, diff --git a/tensorflow/contrib/lite/kernels/sub.cc b/tensorflow/contrib/lite/kernels/sub.cc index ddaf498d5b..410585a293 100644 --- a/tensorflow/contrib/lite/kernels/sub.cc +++ b/tensorflow/contrib/lite/kernels/sub.cc @@ -26,7 +26,7 @@ namespace ops { namespace builtin { namespace sub { -// This file has three implementation of Div. +// This file has three implementation of Sub. enum KernelType { kReference, kGenericOptimized, // Neon-free @@ -37,7 +37,23 @@ constexpr int kInputTensor1 = 0; constexpr int kInputTensor2 = 1; constexpr int kOutputTensor = 0; +struct OpData { + bool requires_broadcast; +}; + +void* Init(TfLiteContext* context, const char* buffer, size_t length) { + auto* data = new OpData; + data->requires_broadcast = false; + return data; +} + +void Free(TfLiteContext* context, void* buffer) { + delete reinterpret_cast(buffer); +} + TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + OpData* data = reinterpret_cast(node->user_data); + TF_LITE_ENSURE_EQ(context, NumInputs(node), 2); TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); @@ -45,51 +61,122 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TfLiteTensor* input2 = GetInput(context, node, kInputTensor2); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); - TF_LITE_ENSURE_EQ(context, NumDimensions(input1), NumDimensions(input2)); - for (int i = 0; i < NumDimensions(input1); ++i) { - TF_LITE_ENSURE_EQ(context, SizeOfDimension(input1, i), - SizeOfDimension(input2, i)); - } + TF_LITE_ENSURE_EQ(context, input1->type, input2->type); + output->type = input2->type; - TF_LITE_ENSURE_EQ(context, input1->type, output->type); - TF_LITE_ENSURE_EQ(context, input2->type, output->type); + data->requires_broadcast = !HaveSameShapes(input1, input2); + + TfLiteIntArray* output_size = nullptr; + if (data->requires_broadcast) { + TF_LITE_ENSURE_OK(context, CalculateShapeForBroadcast( + context, input1, input2, &output_size)); + } else { + output_size = TfLiteIntArrayCopy(input1->dims); + } - TfLiteIntArray* output_size = TfLiteIntArrayCopy(input1->dims); return context->ResizeTensor(context, output, output_size); } template void EvalSubFloat(TfLiteContext* context, TfLiteNode* node, - TfLiteSubParams* params, TfLiteTensor* input1, - TfLiteTensor* input2, TfLiteTensor* output) { + TfLiteSubParams* params, const OpData* data, + TfLiteTensor* input1, TfLiteTensor* input2, + TfLiteTensor* output) { float output_activation_min, output_activation_max; CalculateActivationRangeFloat(params->activation, &output_activation_min, &output_activation_max); -#define TF_LITE_Sub(type) \ - type::Sub(GetTensorData(input1), GetTensorDims(input1), \ - GetTensorData(input2), GetTensorDims(input2), \ - output_activation_min, output_activation_max, \ - GetTensorData(output), GetTensorDims(output)) +#define TF_LITE_SUB(type, opname) \ + type::opname(GetTensorData(input1), GetTensorDims(input1), \ + GetTensorData(input2), GetTensorDims(input2), \ + output_activation_min, output_activation_max, \ + GetTensorData(output), GetTensorDims(output)) + if (kernel_type == kReference) { + if (data->requires_broadcast) { + TF_LITE_SUB(reference_ops, BroadcastSub); + } else { + TF_LITE_SUB(reference_ops, Sub); + } + } else { + if (data->requires_broadcast) { + TF_LITE_SUB(optimized_ops, BroadcastSub); + } else { + TF_LITE_SUB(optimized_ops, Sub); + } + } +#undef TF_LITE_SUB +} + +template +void EvalSubQuantized(TfLiteContext* context, TfLiteNode* node, + TfLiteSubParams* params, const OpData* data, + TfLiteTensor* input1, TfLiteTensor* input2, + TfLiteTensor* output) { + auto input1_offset = -input1->params.zero_point; + auto input2_offset = -input2->params.zero_point; + auto output_offset = output->params.zero_point; + const int left_shift = 20; + const double twice_max_input_scale = + 2 * std::max(input1->params.scale, input2->params.scale); + const double real_input1_multiplier = + input1->params.scale / twice_max_input_scale; + const double real_input2_multiplier = + input2->params.scale / twice_max_input_scale; + const double real_output_multiplier = + twice_max_input_scale / ((1 << left_shift) * output->params.scale); + + int32 input1_multiplier; + int input1_shift; + QuantizeMultiplierSmallerThanOne(real_input1_multiplier, &input1_multiplier, + &input1_shift); + int32 input2_multiplier; + int input2_shift; + QuantizeMultiplierSmallerThanOne(real_input2_multiplier, &input2_multiplier, + &input2_shift); + int32 output_multiplier; + int output_shift; + QuantizeMultiplierSmallerThanOne(real_output_multiplier, &output_multiplier, + &output_shift); + + int32 output_activation_min, output_activation_max; + CalculateActivationRangeUint8(params->activation, output, + &output_activation_min, &output_activation_max); + +#define TF_LITE_SUB(type, opname) \ + type::opname(left_shift, GetTensorData(input1), \ + GetTensorDims(input1), input1_offset, input1_multiplier, \ + input1_shift, GetTensorData(input2), \ + GetTensorDims(input2), input2_offset, input2_multiplier, \ + input2_shift, output_offset, output_multiplier, output_shift, \ + output_activation_min, output_activation_max, \ + GetTensorData(output), GetTensorDims(output)); + // The quantized version of Sub doesn't support activations, so we + // always use BroadcastSub. if (kernel_type == kReference) { - TF_LITE_Sub(reference_ops); + TF_LITE_SUB(reference_ops, BroadcastSub); } else { - TF_LITE_Sub(optimized_ops); + TF_LITE_SUB(optimized_ops, BroadcastSub); } -#undef TF_LITE_Sub +#undef TF_LITE_SUB } template TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { auto* params = reinterpret_cast(node->builtin_data); + OpData* data = reinterpret_cast(node->user_data); TfLiteTensor* input1 = GetInput(context, node, kInputTensor1); TfLiteTensor* input2 = GetInput(context, node, kInputTensor2); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); if (output->type == kTfLiteFloat32) { - EvalSubFloat(context, node, params, input1, input2, output); + EvalSubFloat(context, node, params, data, input1, input2, + output); + } else if (output->type == kTfLiteUInt8) { + EvalSubQuantized(context, node, params, data, input1, input2, + output); } else { - context->ReportError(context, "Inputs and outputs not all float types."); + context->ReportError(context, + "Inputs and outputs not all float|unit8 types."); return kTfLiteError; } @@ -99,19 +186,19 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace sub TfLiteRegistration* Register_SUB_REF() { - static TfLiteRegistration r = {nullptr, nullptr, sub::Prepare, + static TfLiteRegistration r = {sub::Init, sub::Free, sub::Prepare, sub::Eval}; return &r; } TfLiteRegistration* Register_SUB_GENERIC_OPT() { - static TfLiteRegistration r = {nullptr, nullptr, sub::Prepare, + static TfLiteRegistration r = {sub::Init, sub::Free, sub::Prepare, sub::Eval}; return &r; } TfLiteRegistration* Register_SUB_NEON_OPT() { - static TfLiteRegistration r = {nullptr, nullptr, sub::Prepare, + static TfLiteRegistration r = {sub::Init, sub::Free, sub::Prepare, sub::Eval}; return &r; } diff --git a/tensorflow/contrib/lite/kernels/sub_test.cc b/tensorflow/contrib/lite/kernels/sub_test.cc new file mode 100644 index 0000000000..b2c6d05f62 --- /dev/null +++ b/tensorflow/contrib/lite/kernels/sub_test.cc @@ -0,0 +1,213 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include "tensorflow/contrib/lite/interpreter.h" +#include "tensorflow/contrib/lite/kernels/register.h" +#include "tensorflow/contrib/lite/kernels/test_util.h" +#include "tensorflow/contrib/lite/model.h" + +namespace tflite { +namespace { + +using ::testing::ElementsAreArray; + +class BaseSubOpModel : public SingleOpModel { + public: + BaseSubOpModel(const TensorData& input1, const TensorData& input2, + const TensorData& output, + ActivationFunctionType activation_type) { + input1_ = AddInput(input1); + input2_ = AddInput(input2); + output_ = AddOutput(output); + SetBuiltinOp(BuiltinOperator_Sub, BuiltinOptions_SubOptions, + CreateSubOptions(builder_, activation_type).Union()); + BuildInterpreter({GetShape(input1_), GetShape(input2_)}); + } + + int input1() { return input1_; } + int input2() { return input2_; } + + protected: + int input1_; + int input2_; + int output_; +}; + +class FloatSubOpModel : public BaseSubOpModel { + public: + using BaseSubOpModel::BaseSubOpModel; + + std::vector GetOutput() { return ExtractVector(output_); } +}; + +class QuantizedSubOpModel : public BaseSubOpModel { + public: + using BaseSubOpModel::BaseSubOpModel; + + std::vector GetDequantizedOutput() { + return Dequantize(ExtractVector(output_), + GetScale(output_), GetZeroPoint(output_)); + } +}; + +// for quantized Sub, the error shouldn't exceed 2*step +float GetTolerance(int min, int max) { + float kQuantizedStep = (max - min) / 255.0; + float kQuantizedTolerance = 2.0 * kQuantizedStep; + return kQuantizedTolerance; +} + +TEST(FloatSubOpModel, NoActivation) { + FloatSubOpModel m({TensorType_FLOAT32, {1, 2, 2, 1}}, + {TensorType_FLOAT32, {1, 2, 2, 1}}, + {TensorType_FLOAT32, {}}, ActivationFunctionType_NONE); + m.PopulateTensor(m.input1(), {-2.0, 0.2, 1.7, 0.5}); + m.PopulateTensor(m.input2(), {0.1, 0.2, 0.3, 0.8}); + m.Invoke(); + EXPECT_THAT(m.GetOutput(), ElementsAreArray({-2.1, 0.0, 1.4, -0.3})); +} + +TEST(FloatSubOpModel, ActivationRELU_N1_TO_1) { + FloatSubOpModel m( + {TensorType_FLOAT32, {1, 2, 2, 1}}, {TensorType_FLOAT32, {1, 2, 2, 1}}, + {TensorType_FLOAT32, {}}, ActivationFunctionType_RELU_N1_TO_1); + m.PopulateTensor(m.input1(), {-2.0, 0.2, 1.7, 0.5}); + m.PopulateTensor(m.input2(), {0.1, 0.2, 0.3, 0.8}); + m.Invoke(); + EXPECT_THAT(m.GetOutput(), ElementsAreArray({-1.0, 0.0, 1.0, -0.3})); +} + +TEST(FloatSubOpModel, VariousInputShapes) { + std::vector> test_shapes = { + {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}}; + for (int i = 0; i < test_shapes.size(); ++i) { + FloatSubOpModel m({TensorType_FLOAT32, test_shapes[i]}, + {TensorType_FLOAT32, test_shapes[i]}, + {TensorType_FLOAT32, {}}, ActivationFunctionType_NONE); + m.PopulateTensor(m.input1(), {-2.0, 0.2, 1.7, 0.5, -1.1, 2.0}); + m.PopulateTensor(m.input2(), {0.1, 0.2, 0.3, 0.8, -1.1, 0.1}); + m.Invoke(); + EXPECT_THAT(m.GetOutput(), + ElementsAreArray({-2.1, 0.0, 1.4, -0.3, 0.0, 1.9})) + << "With shape number " << i; + } +} + +TEST(FloatSubOpModel, WithBroadcast) { + std::vector> test_shapes = { + {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}}; + for (int i = 0; i < test_shapes.size(); ++i) { + FloatSubOpModel m({TensorType_FLOAT32, test_shapes[i]}, + {TensorType_FLOAT32, {}}, // always a scalar + {TensorType_FLOAT32, {}}, ActivationFunctionType_NONE); + m.PopulateTensor(m.input1(), {-2.0, 0.2, 1.7, 0.5, -1.1, 2.0}); + m.PopulateTensor(m.input2(), {0.5}); + m.Invoke(); + EXPECT_THAT( + m.GetOutput(), + ElementsAreArray(ArrayFloatNear({-2.5, -0.3, 1.2, 0.0, -1.6, 1.5}))) + << "With shape number " << i; + } +} + +TEST(QuantizedSubOpModel, QuantizedTestsNoActivation) { + float kQuantizedTolerance = GetTolerance(-1.0, 1.0); + std::vector> inputs1 = { + {0.1, 0.2, 0.3, 0.4}, {-0.2, 0.2, 0.4, 0.7}, {-0.01, 0.2, 0.7, 0.3}}; + std::vector> inputs2 = { + {0.6, 0.4, 0.3, 0.1}, {0.6, 0.4, 0.5, -0.2}, {0.6, 0.4, -0.18, 0.5}}; + std::vector> results = { + {-0.5, -0.2, 0.0, 0.3}, {-0.8, -0.2, -0.1, 0.9}, {-0.61, -0.2, 0.88, -0.2}}; + for (int i = 0; i < inputs1.size(); ++i) { + QuantizedSubOpModel m({TensorType_UINT8, {1, 2, 2, 1}, -1.0, 1.0}, + {TensorType_UINT8, {1, 2, 2, 1}, -1.0, 1.0}, + {TensorType_UINT8, {}, -1.0, 1.0}, + ActivationFunctionType_NONE); + m.QuantizeAndPopulate(m.input1(), inputs1[i]); + m.QuantizeAndPopulate(m.input2(), inputs2[i]); + m.Invoke(); + EXPECT_THAT(m.GetDequantizedOutput(), ElementsAreArray(ArrayFloatNear( + results[i], kQuantizedTolerance))) + << "With test number " << i; + } +} + +TEST(QuantizedSubOpModel, QuantizedTestsActivationRELU_N1_TO_1) { + float kQuantizedTolerance = GetTolerance(-1.0, 1.0); + std::vector> inputs1 = {{-0.8, 0.2, 0.9, 0.7}, + {-0.8, 0.2, 0.7, 0.5}}; + std::vector> inputs2 = {{0.6, 0.4, 0.9, -0.8}, + {0.6, 0.4, -0.8, 0.3}}; + std::vector> results = {{-1.0, -0.2, 0.0, 1.0}, + {-1.0, -0.2, 1.0, 0.2}}; + for (int i = 0; i < inputs1.size(); ++i) { + QuantizedSubOpModel m({TensorType_UINT8, {1, 2, 2, 1}, -1.0, 1.0}, + {TensorType_UINT8, {1, 2, 2, 1}, -1.0, 1.0}, + {TensorType_UINT8, {}, -1.0, 1.0}, + ActivationFunctionType_RELU_N1_TO_1); + m.QuantizeAndPopulate(m.input1(), inputs1[i]); + m.QuantizeAndPopulate(m.input2(), inputs2[i]); + m.Invoke(); + EXPECT_THAT(m.GetDequantizedOutput(), ElementsAreArray(ArrayFloatNear( + results[i], kQuantizedTolerance))) + << "With test number " << i; + } +} + +TEST(QuantizedSubOpModel, QuantizedVariousInputShapes) { + float kQuantizedTolerance = GetTolerance(-3.0, 3.0); + std::vector> test_shapes = { + {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}}; + for (int i = 0; i < test_shapes.size(); ++i) { + QuantizedSubOpModel m({TensorType_UINT8, test_shapes[i], -3.0, 3.0}, + {TensorType_UINT8, test_shapes[i], -3.0, 3.0}, + {TensorType_UINT8, {}, -3.0, 3.0}, + ActivationFunctionType_NONE); + m.QuantizeAndPopulate(m.input1(), {-2.0, 0.2, 0.7, 0.8, 1.1, 2.0}); + m.QuantizeAndPopulate(m.input2(), {0.1, 0.3, 0.3, 0.5, 1.1, 0.1}); + m.Invoke(); + EXPECT_THAT(m.GetDequantizedOutput(), + ElementsAreArray(ArrayFloatNear({-2.1, -0.1, 0.4, 0.3, 0.0, 1.9}, + kQuantizedTolerance))) + << "With shape number " << i; + } +} + +TEST(QuantizedSubOpModel, QuantizedWithBroadcast) { + float kQuantizedTolerance = GetTolerance(-3.0, 3.0); + std::vector> test_shapes = { + {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}}; + for (int i = 0; i < test_shapes.size(); ++i) { + QuantizedSubOpModel m({TensorType_UINT8, test_shapes[i], -3.0, 3.0}, + {TensorType_UINT8, {}, -3.0, 3.0}, + {TensorType_UINT8, {}, -3.0, 3.0}, + ActivationFunctionType_NONE); + m.QuantizeAndPopulate(m.input1(), {-2.0, 0.2, 0.7, 0.8, 1.1, 2.0}); + m.QuantizeAndPopulate(m.input2(), {0.7}); + m.Invoke(); + EXPECT_THAT(m.GetDequantizedOutput(), + ElementsAreArray(ArrayFloatNear({-2.7, -0.5, 0.0, 0.1, 0.4, 1.3}, + kQuantizedTolerance))) + << "With shape number " << i; + } +} + +} // namespace +} // namespace tflite +int main(int argc, char** argv) { + ::tflite::LogToStderr(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc index 49766cedac..1e177d5f6e 100644 --- a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc +++ b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc @@ -47,9 +47,6 @@ tensorflow::Env* env = tensorflow::Env::Default(); // Key is a substring of the test name and value is a bug number. // TODO(ahentz): make sure we clean this list up frequently. std::map kBrokenTests = { - // Sub and Div don't support broadcasting. - {R"(^\/diva.*input_shape_1=\[1,3,4,3\],input_shape_2=\[3\])", "68500195"}, - {R"(^\/suba.*input_shape_1=\[1,3,4,3\],input_shape_2=\[3\])", "68500195"}, // Add only supports float32. (and "constant" tests use Add) {R"(^\/adda.*int32)", "68808744"}, @@ -235,22 +232,23 @@ TEST_P(OpsTest, RunStuff) { INSTANTIATE_TESTS(add) INSTANTIATE_TESTS(avg_pool) -INSTANTIATE_TESTS(space_to_batch_nd) INSTANTIATE_TESTS(batch_to_space_nd) INSTANTIATE_TESTS(concat) INSTANTIATE_TESTS(constant) INSTANTIATE_TESTS(control_dep) INSTANTIATE_TESTS(conv) INSTANTIATE_TESTS(depthwiseconv) +INSTANTIATE_TESTS(div) INSTANTIATE_TESTS(exp) INSTANTIATE_TESTS(fully_connected) INSTANTIATE_TESTS(fused_batch_norm) INSTANTIATE_TESTS(gather) INSTANTIATE_TESTS(global_batch_norm) -INSTANTIATE_TESTS(l2norm) INSTANTIATE_TESTS(l2_pool) +INSTANTIATE_TESTS(l2norm) INSTANTIATE_TESTS(local_response_norm) INSTANTIATE_TESTS(max_pool) +INSTANTIATE_TESTS(mean) INSTANTIATE_TESTS(mul) INSTANTIATE_TESTS(pad) INSTANTIATE_TESTS(relu) @@ -260,14 +258,13 @@ INSTANTIATE_TESTS(reshape) INSTANTIATE_TESTS(resize_bilinear) INSTANTIATE_TESTS(sigmoid) INSTANTIATE_TESTS(softmax) +INSTANTIATE_TESTS(space_to_batch_nd) INSTANTIATE_TESTS(space_to_depth) -INSTANTIATE_TESTS(sub) INSTANTIATE_TESTS(split) -INSTANTIATE_TESTS(div) -INSTANTIATE_TESTS(transpose) -INSTANTIATE_TESTS(mean) INSTANTIATE_TESTS(squeeze) INSTANTIATE_TESTS(strided_slice) +INSTANTIATE_TESTS(sub) +INSTANTIATE_TESTS(transpose) } // namespace testing } // namespace tflite -- GitLab From 779d457008ab7ea2c11f4d73370099a1e56c0652 Mon Sep 17 00:00:00 2001 From: ManHyuk Date: Sun, 25 Feb 2018 21:39:52 +0900 Subject: [PATCH 011/791] fix typo --- .../python/kernel_tests/linalg/linear_operator_diag_test.py | 2 +- tensorflow/python/ops/linalg/linear_operator_diag.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py index 343d158498..8cb9f9e621 100644 --- a/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py +++ b/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py @@ -129,7 +129,7 @@ class LinearOperatorDiagTest( with self.test_session() as sess: x = random_ops.random_normal(shape=(2, 2, 3, 4)) - # This LinearOperatorDiag will be brodacast to (2, 2, 3, 3) during solve + # This LinearOperatorDiag will be broadcast to (2, 2, 3, 3) during solve # and matmul with 'x' as the argument. diag = random_ops.random_uniform(shape=(2, 1, 3)) operator = linalg.LinearOperatorDiag(diag, is_self_adjoint=True) diff --git a/tensorflow/python/ops/linalg/linear_operator_diag.py b/tensorflow/python/ops/linalg/linear_operator_diag.py index b3ec3d5b7c..e180e83026 100644 --- a/tensorflow/python/ops/linalg/linear_operator_diag.py +++ b/tensorflow/python/ops/linalg/linear_operator_diag.py @@ -67,7 +67,7 @@ class LinearOperatorDiag(linear_operator.LinearOperator): operator = LinearOperatorDiag(diag) # Create a shape [2, 1, 4, 2] vector. Note that this shape is compatible - # since the batch dimensions, [2, 1], are brodcast to + # since the batch dimensions, [2, 1], are broadcast to # operator.batch_shape = [2, 3]. y = tf.random_normal(shape=[2, 1, 4, 2]) x = operator.solve(y) -- GitLab From b569035378ef4a8595c64e5f398d74244cac376e Mon Sep 17 00:00:00 2001 From: ManHyuk Date: Sun, 25 Feb 2018 21:44:12 +0900 Subject: [PATCH 012/791] fix typo --- tensorflow/contrib/slim/python/slim/data/parallel_reader.py | 2 +- tensorflow/python/ops/distributions/special_math.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/slim/python/slim/data/parallel_reader.py b/tensorflow/contrib/slim/python/slim/data/parallel_reader.py index ad5e985487..b3343aef47 100644 --- a/tensorflow/contrib/slim/python/slim/data/parallel_reader.py +++ b/tensorflow/contrib/slim/python/slim/data/parallel_reader.py @@ -221,7 +221,7 @@ def parallel_read(data_sources, the data will be cycled through indefinitely. num_readers: a integer, number of Readers to create. reader_kwargs: an optional dict, of kwargs for the reader. - shuffle: boolean, wether should shuffle the files and the records by using + shuffle: boolean, whether should shuffle the files and the records by using RandomShuffleQueue as common_queue. dtypes: A list of types. The length of dtypes must equal the number of elements in each record. If it is None it will default to diff --git a/tensorflow/python/ops/distributions/special_math.py b/tensorflow/python/ops/distributions/special_math.py index bed4cbb2c1..1d605c5dfc 100644 --- a/tensorflow/python/ops/distributions/special_math.py +++ b/tensorflow/python/ops/distributions/special_math.py @@ -213,7 +213,7 @@ def _ndtri(p): # Compute x for p <= exp(-2): x = z - log(z)/z - (1/z) P(1/z) / Q(1/z), # where z = sqrt(-2. * log(p)), and P/Q are chosen between two different - # arrays based on wether p < exp(-32). + # arrays based on whether p < exp(-32). z = math_ops.sqrt(-2. * math_ops.log(sanitized_mcp)) first_term = z - math_ops.log(z) / z second_term_small_p = (_create_polynomial(1. / z, p2) -- GitLab From f1f70ef5c268d6ce41bdab4867ed0f2e19d6f924 Mon Sep 17 00:00:00 2001 From: Hovhannes Harutyunyan Date: Mon, 26 Feb 2018 10:52:11 +0400 Subject: [PATCH 013/791] Remove code that was written for compatibility with old checked-in code. Update code to have 80 characters per line. --- tensorflow/contrib/lite/kernels/div_test.cc | 3 +- .../internal/optimized/optimized_ops.h | 41 ------------------- .../internal/reference/reference_ops.h | 41 ------------------- tensorflow/contrib/lite/kernels/sub_test.cc | 18 +++++--- 4 files changed, 15 insertions(+), 88 deletions(-) diff --git a/tensorflow/contrib/lite/kernels/div_test.cc b/tensorflow/contrib/lite/kernels/div_test.cc index 78918a0d79..e67e0ec034 100644 --- a/tensorflow/contrib/lite/kernels/div_test.cc +++ b/tensorflow/contrib/lite/kernels/div_test.cc @@ -154,7 +154,8 @@ TEST(QuantizedDivOpTest, WithBroadcast) { {TensorType_UINT8, {}, -3.0, 3.0}, // always a scalar {TensorType_UINT8, {}, -3.0, 3.0}, ActivationFunctionType_NONE); - m.QuantizeAndPopulate(m.input1(), {-0.2, 0.2, 0.07, 0.08, 0.11, -0.123}); + m.QuantizeAndPopulate(m.input1(), {-0.2, 0.2, 0.07, + 0.08, 0.11, -0.123}); m.QuantizeAndPopulate(m.input2(), {0.1}); m.Invoke(); EXPECT_THAT(m.GetDequantizedOutput(), diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h index d12a3eca1d..b19f46beaa 100644 --- a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h +++ b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h @@ -1973,19 +1973,6 @@ void BroadcastDiv(const T* input1_data, const Dims<4>& input1_dims, } } -// legacy, for compatibility with old checked-in code -template -void BroadcastDiv(const T* input1_data, const Dims<4>& input1_dims, - const T* input2_data, const Dims<4>& input2_dims, - T* output_data, const Dims<4>& output_dims) { - T output_activation_min, output_activation_max; - GetActivationMinMax(Ac, &output_activation_min, &output_activation_max); - - BroadcastDiv(input1_data, input1_dims, input2_data, input2_dims, - output_activation_min, output_activation_max, output_data, - output_dims); -} - inline void BroadcastDiv(const uint8* input1_data, const Dims<4>& input1_dims, int32 input1_offset, const uint8* input2_data, const Dims<4>& input2_dims, int32 input2_offset, @@ -2033,21 +2020,6 @@ inline void BroadcastDiv(const uint8* input1_data, const Dims<4>& input1_dims, } } -// legacy, for compatibility with old checked-in code -template -inline void BroadcastDiv(const uint8* input1_data, const Dims<4>& input1_dims, - int32 input1_offset, const uint8* input2_data, - const Dims<4>& input2_dims, int32 input2_offset, - int32 output_offset, int32 output_multiplier, - int output_shift, int32 output_activation_min, - int32 output_activation_max, uint8* output_data, - const Dims<4>& output_dims) { - BroadcastDiv(input1_data, input1_dims, input1_offset, input2_data, - input2_dims, input2_offset, output_offset, output_multiplier, - output_shift, output_activation_min, output_activation_max, - output_data, output_dims); -} - // TODO(aselle): This is not actually optimized yet. inline void Sub(const float* input1_data, const Dims<4>& input1_dims, const float* input2_data, const Dims<4>& input2_dims, @@ -2121,19 +2093,6 @@ void BroadcastSub(const T* input1_data, const Dims<4>& input1_dims, } } -// legacy, for compatibility with old checked-in code -template -void BroadcastSub(const T* input1_data, const Dims<4>& input1_dims, - const T* input2_data, const Dims<4>& input2_dims, - T* output_data, const Dims<4>& output_dims) { - T output_activation_min, output_activation_max; - GetActivationMinMax(Ac, &output_activation_min, &output_activation_max); - - BroadcastSub(input1_data, input1_dims, input2_data, input2_dims, - output_activation_min, output_activation_max, output_data, - output_dims); -} - inline void BroadcastSub(int left_shift, const uint8* input1_data, const Dims<4>& input1_dims, int32 input1_offset, int32 input1_multiplier, int input1_shift, diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h index c7b7687622..847075e207 100644 --- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h +++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h @@ -1249,19 +1249,6 @@ void BroadcastDiv(const T* input1_data, const Dims<4>& input1_dims, } } -// legacy, for compatibility with old checked-in code -template -void BroadcastDiv(const T* input1_data, const Dims<4>& input1_dims, - const T* input2_data, const Dims<4>& input2_dims, - T* output_data, const Dims<4>& output_dims) { - T output_activation_min, output_activation_max; - GetActivationMinMax(Ac, &output_activation_min, &output_activation_max); - - BroadcastDiv(input1_data, input1_dims, input2_data, input2_dims, - output_activation_min, output_activation_max, output_data, - output_dims); -} - inline void BroadcastDiv(const uint8* input1_data, const Dims<4>& input1_dims, int32 input1_offset, const uint8* input2_data, const Dims<4>& input2_dims, int32 input2_offset, @@ -1309,21 +1296,6 @@ inline void BroadcastDiv(const uint8* input1_data, const Dims<4>& input1_dims, } } -// legacy, for compatibility with old checked-in code -template -inline void BroadcastDiv(const uint8* input1_data, const Dims<4>& input1_dims, - int32 input1_offset, const uint8* input2_data, - const Dims<4>& input2_dims, int32 input2_offset, - int32 output_offset, int32 output_multiplier, - int output_shift, int32 output_activation_min, - int32 output_activation_max, uint8* output_data, - const Dims<4>& output_dims) { - BroadcastDiv(input1_data, input1_dims, input1_offset, input2_data, - input2_dims, input2_offset, output_offset, output_multiplier, - output_shift, output_activation_min, output_activation_max, - output_data, output_dims); -} - inline void Sub(const float* input1_data, const Dims<4>& input1_dims, const float* input2_data, const Dims<4>& input2_dims, float output_activation_min, float output_activation_max, @@ -1392,19 +1364,6 @@ void BroadcastSub(const T* input1_data, const Dims<4>& input1_dims, } } -// legacy, for compatibility with old checked-in code -template -void BroadcastSub(const T* input1_data, const Dims<4>& input1_dims, - const T* input2_data, const Dims<4>& input2_dims, - T* output_data, const Dims<4>& output_dims) { - T output_activation_min, output_activation_max; - GetActivationMinMax(Ac, &output_activation_min, &output_activation_max); - - BroadcastSub(input1_data, input1_dims, input2_data, input2_dims, - output_activation_min, output_activation_max, output_data, - output_dims); -} - inline void BroadcastSub(int left_shift, const uint8* input1_data, const Dims<4>& input1_dims, int32 input1_offset, int32 input1_multiplier, int input1_shift, diff --git a/tensorflow/contrib/lite/kernels/sub_test.cc b/tensorflow/contrib/lite/kernels/sub_test.cc index b2c6d05f62..1fd0ee2a0e 100644 --- a/tensorflow/contrib/lite/kernels/sub_test.cc +++ b/tensorflow/contrib/lite/kernels/sub_test.cc @@ -125,11 +125,17 @@ TEST(FloatSubOpModel, WithBroadcast) { TEST(QuantizedSubOpModel, QuantizedTestsNoActivation) { float kQuantizedTolerance = GetTolerance(-1.0, 1.0); std::vector> inputs1 = { - {0.1, 0.2, 0.3, 0.4}, {-0.2, 0.2, 0.4, 0.7}, {-0.01, 0.2, 0.7, 0.3}}; + {0.1, 0.2, 0.3, 0.4}, + {-0.2, 0.2, 0.4, 0.7}, + {-0.01, 0.2, 0.7, 0.3}}; std::vector> inputs2 = { - {0.6, 0.4, 0.3, 0.1}, {0.6, 0.4, 0.5, -0.2}, {0.6, 0.4, -0.18, 0.5}}; + {0.6, 0.4, 0.3, 0.1}, + {0.6, 0.4, 0.5, -0.2}, + {0.6, 0.4, -0.18, 0.5}}; std::vector> results = { - {-0.5, -0.2, 0.0, 0.3}, {-0.8, -0.2, -0.1, 0.9}, {-0.61, -0.2, 0.88, -0.2}}; + {-0.5, -0.2, 0.0, 0.3}, + {-0.8, -0.2, -0.1, 0.9}, + {-0.61, -0.2, 0.88, -0.2}}; for (int i = 0; i < inputs1.size(); ++i) { QuantizedSubOpModel m({TensorType_UINT8, {1, 2, 2, 1}, -1.0, 1.0}, {TensorType_UINT8, {1, 2, 2, 1}, -1.0, 1.0}, @@ -179,7 +185,8 @@ TEST(QuantizedSubOpModel, QuantizedVariousInputShapes) { m.QuantizeAndPopulate(m.input2(), {0.1, 0.3, 0.3, 0.5, 1.1, 0.1}); m.Invoke(); EXPECT_THAT(m.GetDequantizedOutput(), - ElementsAreArray(ArrayFloatNear({-2.1, -0.1, 0.4, 0.3, 0.0, 1.9}, + ElementsAreArray(ArrayFloatNear({-2.1, -0.1, 0.4, + 0.3, 0.0, 1.9}, kQuantizedTolerance))) << "With shape number " << i; } @@ -198,7 +205,8 @@ TEST(QuantizedSubOpModel, QuantizedWithBroadcast) { m.QuantizeAndPopulate(m.input2(), {0.7}); m.Invoke(); EXPECT_THAT(m.GetDequantizedOutput(), - ElementsAreArray(ArrayFloatNear({-2.7, -0.5, 0.0, 0.1, 0.4, 1.3}, + ElementsAreArray(ArrayFloatNear({-2.7, -0.5, 0.0, + 0.1, 0.4, 1.3}, kQuantizedTolerance))) << "With shape number " << i; } -- GitLab From 0489bf25930ea0dc4b7d8ffc792b0390bfbc06bc Mon Sep 17 00:00:00 2001 From: Jingwen Date: Tue, 27 Feb 2018 18:30:09 -0500 Subject: [PATCH 014/791] Include cstring in logging.cc for use of strrchr() --- tensorflow/core/platform/default/logging.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/core/platform/default/logging.cc b/tensorflow/core/platform/default/logging.cc index 2b874da198..c6e5777c26 100644 --- a/tensorflow/core/platform/default/logging.cc +++ b/tensorflow/core/platform/default/logging.cc @@ -21,6 +21,7 @@ limitations under the License. #include #include #include +#include #endif #include -- GitLab From ef4e8ad826c8946f8ff3e0f7e1b3bb3bec61010c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Wed, 21 Feb 2018 15:06:04 +0800 Subject: [PATCH 015/791] CLN: extract ApplyAdamBaseOp --- tensorflow/core/kernels/training_ops.cc | 146 +++++++++++++++--- tensorflow/core/kernels/training_ops.h | 13 ++ .../core/kernels/training_ops_gpu.cu.cc | 30 ++++ tensorflow/core/ops/training_ops.cc | 37 +++++ 4 files changed, 202 insertions(+), 24 deletions(-) diff --git a/tensorflow/core/kernels/training_ops.cc b/tensorflow/core/kernels/training_ops.cc index 233aa03c32..7d383d980a 100644 --- a/tensorflow/core/kernels/training_ops.cc +++ b/tensorflow/core/kernels/training_ops.cc @@ -328,6 +328,45 @@ struct ApplyAdamSYCL { template struct ApplyAdam : ApplyAdamNonCuda {}; +template +struct ApplyAdaMaxNonCuda { + void operator()(const Device& d, typename TTypes::Flat var, + typename TTypes::Flat m, typename TTypes::Flat v, + typename TTypes::ConstScalar beta1_power, + typename TTypes::ConstScalar beta2_power, + typename TTypes::ConstScalar lr, + typename TTypes::ConstScalar beta1, + typename TTypes::ConstScalar beta2, + typename TTypes::ConstScalar epsilon, + typename TTypes::ConstFlat grad, bool use_nesterov) { + if (use_nesterov) { + LOG(WARNING) << "AdaMax doesn't support use_nesterov yet, ignore it."; + } + m.device(d) += (grad - m) * (T(1) - beta1()); + // v == u + v.device(d) = (beta2() * v).cwiseMax(grad.abs()); + // var == θ + var.device(d) -= (lr * m) / ((T(1) - beta1_power()) * v); + } +}; + +#ifdef TENSORFLOW_USE_SYCL +template +struct ApplyAdaMaxSYCL { + void operator()(const SYCLDevice& d, typename TTypes::Flat var, + typename TTypes::Flat m, typename TTypes::Flat v, + T beta1_power, T beta2_power, T lr, T beta1, T beta2, + T epsilon, typename TTypes::ConstFlat grad) { + m.device(d) += (grad - m) * (T(1) - beta1); + v.device(d) = (beta2 * v).cwiseMax(grad.abs()); + var.device(d) -= (lr * m) / ((T(1) - beta1_power) * v); + } +}; +#endif // TENSORFLOW_USE_SYCL + +template +struct ApplyAdaMax : ApplyAdaMaxNonCuda {}; + template struct ApplyRMSProp { void operator()(const CPUDevice& d, typename TTypes::Flat var, @@ -2477,10 +2516,12 @@ TF_CALL_double(REGISTER_CPU_KERNELS); #undef REGISTER_CPU_KERNELS #undef REGISTER_KERNELS -template -class ApplyAdamOp : public OpKernel { +template + class Functor> +class ApplyAdamBaseOp : public OpKernel { public: - explicit ApplyAdamOp(OpKernelConstruction* ctx) : OpKernel(ctx) { + explicit ApplyAdamBaseOp(OpKernelConstruction* ctx) : OpKernel(ctx) { OP_REQUIRES_OK(ctx, ctx->GetAttr("use_locking", &use_exclusive_lock_)); OP_REQUIRES_OK(ctx, ctx->GetAttr("use_nesterov", &use_nesterov_)); } @@ -2553,11 +2594,11 @@ class ApplyAdamOp : public OpKernel { grad.shape().DebugString())); const Device& device = ctx->template eigen_device(); - functor::ApplyAdam()( - device, var.flat(), m.flat(), v.flat(), - beta1_power.scalar(), beta2_power.scalar(), lr.scalar(), - beta1.scalar(), beta2.scalar(), epsilon.scalar(), - grad.flat(), use_nesterov_); + auto functor = Functor(); + functor(device, var.flat(), m.flat(), v.flat(), + beta1_power.scalar(), beta2_power.scalar(), lr.scalar(), + beta1.scalar(), beta2.scalar(), epsilon.scalar(), + grad.flat(), use_nesterov_); MaybeForwardRefInputToRefOutput(ctx, 0, 0); } @@ -2568,10 +2609,11 @@ class ApplyAdamOp : public OpKernel { }; #ifdef TENSORFLOW_USE_SYCL -template -class ApplyAdamOp : public OpKernel { +template class Functor> +class ApplyAdamBaseOp : public OpKernel { public: - explicit ApplyAdamOp(OpKernelConstruction* ctx) : OpKernel(ctx) { + explicit ApplyAdamBaseOp(OpKernelConstruction* ctx) : OpKernel(ctx) { OP_REQUIRES_OK(ctx, ctx->GetAttr("use_locking", &use_exclusive_lock_)); } @@ -2672,9 +2714,10 @@ class ApplyAdamOp : public OpKernel { var.shape().DebugString(), " ", grad.shape().DebugString())); - functor::ApplyAdamSYCL()(device, var.flat(), m.flat(), v.flat(), - beta1_power, beta2_power, lr, beta1, beta2, - epsilon, grad.flat()); + auto functor = Functor(); + functor(device, var.flat(), m.flat(), v.flat(), + beta1_power, beta2_power, lr, beta1, beta2, + epsilon, grad.flat()); MaybeForwardRefInputToRefOutput(ctx, 0, 0); } @@ -2684,28 +2727,28 @@ class ApplyAdamOp : public OpKernel { }; #endif // TENSORFLOW_USE_SYCL -#define REGISTER_KERNELS(D, T) \ +#define REGISTER_KERNELS(D, T, F) \ REGISTER_KERNEL_BUILDER( \ Name("ApplyAdam").Device(DEVICE_##D).TypeConstraint("T"), \ - ApplyAdamOp); \ + ApplyAdamBaseOp); \ REGISTER_KERNEL_BUILDER(Name("ResourceApplyAdam") \ .HostMemory("var") \ .HostMemory("m") \ .HostMemory("v") \ .Device(DEVICE_##D) \ .TypeConstraint("T"), \ - ApplyAdamOp); -#define REGISTER_CPU_KERNELS(T) REGISTER_KERNELS(CPU, T); - + ApplyAdamBaseOp); +#define REGISTER_CPU_KERNELS(T) REGISTER_KERNELS(CPU, T, functor::ApplyAdam); TF_CALL_half(REGISTER_CPU_KERNELS); TF_CALL_float(REGISTER_CPU_KERNELS); TF_CALL_double(REGISTER_CPU_KERNELS); +#undef REGISTER_CPU_KERNELS #ifdef TENSORFLOW_USE_SYCL -#define REGISTER_SYCL_KERNELS(T) REGISTER_KERNELS(SYCL, T); - +#define REGISTER_SYCL_KERNELS(T) REGISTER_KERNELS(SYCL, T, functor::ApplyAdamSYCL); TF_CALL_float(REGISTER_SYCL_KERNELS); TF_CALL_double(REGISTER_SYCL_KERNELS); +#undef REGISTER_SYCL_KERNELS #endif #if GOOGLE_CUDA @@ -2730,11 +2773,66 @@ DECLARE_GPU_SPEC(double); #undef DECLARE_GPU_SPEC } // namespace functor -REGISTER_KERNELS(GPU, Eigen::half); -REGISTER_KERNELS(GPU, float); -REGISTER_KERNELS(GPU, double); +#define REGISTER_GPU_KERNELS(T) REGISTER_KERNELS(GPU, T, functor::ApplyAdam); +REGISTER_GPU_KERNELS(Eigen::half); +REGISTER_GPU_KERNELS(float); +REGISTER_GPU_KERNELS(double); +#undef REGISTER_GPU_KERNELS #endif +#undef REGISTER_KERNELS + +#define REGISTER_KERNELS(D, T, F) \ + REGISTER_KERNEL_BUILDER( \ + Name("ApplyAdaMax").Device(DEVICE_##D).TypeConstraint("T"), \ + ApplyAdamBaseOp); \ + REGISTER_KERNEL_BUILDER(Name("ResourceApplyAdaMax") \ + .HostMemory("var") \ + .HostMemory("m") \ + .HostMemory("v") \ + .Device(DEVICE_##D) \ + .TypeConstraint("T"), \ + ApplyAdamBaseOp); +#define REGISTER_CPU_KERNELS(T) REGISTER_KERNELS(CPU, T, functor::ApplyAdaMax); +TF_CALL_half(REGISTER_CPU_KERNELS); +TF_CALL_float(REGISTER_CPU_KERNELS); +TF_CALL_double(REGISTER_CPU_KERNELS); #undef REGISTER_CPU_KERNELS + +#ifdef TENSORFLOW_USE_SYCL +#define REGISTER_SYCL_KERNELS(T) REGISTER_KERNELS(SYCL, T, functor::ApplyAdaMaxSYCL); +TF_CALL_float(REGISTER_SYCL_KERNELS); +TF_CALL_double(REGISTER_SYCL_KERNELS); +#undef REGISTER_SYCL_KERNELS +#endif + +#if GOOGLE_CUDA +// Forward declarations of the functor specializations for GPU. +namespace functor { +#define DECLARE_GPU_SPEC(T) \ + template <> \ + void ApplyAdaMax::operator()( \ + const GPUDevice& d, typename TTypes::Flat var, \ + typename TTypes::Flat m, typename TTypes::Flat v, \ + typename TTypes::ConstScalar beta1_power, \ + typename TTypes::ConstScalar beta2_power, \ + typename TTypes::ConstScalar lr, \ + typename TTypes::ConstScalar beta1, \ + typename TTypes::ConstScalar beta2, \ + typename TTypes::ConstScalar epsilon, \ + typename TTypes::ConstFlat grad, bool use_nesterov); \ + extern template struct ApplyAdaMax; +DECLARE_GPU_SPEC(Eigen::half); +DECLARE_GPU_SPEC(float); +DECLARE_GPU_SPEC(double); +#undef DECLARE_GPU_SPEC +} // namespace functor + +#define REGISTER_GPU_KERNELS(T) REGISTER_KERNELS(GPU, T, functor::ApplyAdaMax); +REGISTER_GPU_KERNELS(Eigen::half); +REGISTER_GPU_KERNELS(float); +REGISTER_GPU_KERNELS(double); +#undef REGISTER_GPU_KERNELS +#endif #undef REGISTER_KERNELS template diff --git a/tensorflow/core/kernels/training_ops.h b/tensorflow/core/kernels/training_ops.h index 7ee956053a..46a5290210 100644 --- a/tensorflow/core/kernels/training_ops.h +++ b/tensorflow/core/kernels/training_ops.h @@ -139,6 +139,19 @@ struct ApplyAdam { typename TTypes::ConstFlat grad, bool use_nesterov); }; +template +struct ApplyAdaMax { + void operator()(const Device& d, typename TTypes::Flat var, + typename TTypes::Flat m, typename TTypes::Flat v, + typename TTypes::ConstScalar beta1_power, + typename TTypes::ConstScalar beta2_power, + typename TTypes::ConstScalar lr, + typename TTypes::ConstScalar beta1, + typename TTypes::ConstScalar beta2, + typename TTypes::ConstScalar epsilon, + typename TTypes::ConstFlat grad, bool use_nesterov); +}; + template struct ApplyRMSProp { void operator()(const Device& d, typename TTypes::Flat var, diff --git a/tensorflow/core/kernels/training_ops_gpu.cu.cc b/tensorflow/core/kernels/training_ops_gpu.cu.cc index 0376a3b2c6..1776c108ab 100644 --- a/tensorflow/core/kernels/training_ops_gpu.cu.cc +++ b/tensorflow/core/kernels/training_ops_gpu.cu.cc @@ -142,6 +142,32 @@ struct ApplyAdam { } }; +template +struct ApplyAdaMax { + void operator()(const GPUDevice& d, typename TTypes::Flat var, + typename TTypes::Flat m, typename TTypes::Flat v, + typename TTypes::ConstScalar beta1_power, + typename TTypes::ConstScalar beta2_power, + typename TTypes::ConstScalar lr, + typename TTypes::ConstScalar beta1, + typename TTypes::ConstScalar beta2, + typename TTypes::ConstScalar epsilon, + typename TTypes::ConstFlat grad, bool use_nesterov) { + Eigen::array::Tensor::Index, 1> bcast; + bcast[0] = grad.dimension(0); + Eigen::Sizes<1> single; + const auto one = static_cast(1.0); + m.device(d) = + m + (beta1.constant(one) - beta1).reshape(single).broadcast(bcast) * + (grad - m); + v.device(d) = + (beta2.reshape(single).broadcast(bcast) * v).cwiseMax(grad.abs()); + var.device(d) -= + (lr * m) / ((beta1_power.constant(one) - + beta1_power).reshape(single).broadcast(bcast) * v); + } +}; + template struct ApplyRMSProp { void operator()(const GPUDevice& d, typename TTypes::Flat var, @@ -278,6 +304,10 @@ template struct functor::ApplyAdam; template struct functor::ApplyAdam; template struct functor::ApplyAdam; +template struct functor::ApplyAdaMax; +template struct functor::ApplyAdaMax; +template struct functor::ApplyAdaMax; + template struct functor::ApplyRMSProp; template struct functor::ApplyRMSProp; template struct functor::ApplyRMSProp; diff --git a/tensorflow/core/ops/training_ops.cc b/tensorflow/core/ops/training_ops.cc index 6ce9595fb6..6f107db3ea 100644 --- a/tensorflow/core/ops/training_ops.cc +++ b/tensorflow/core/ops/training_ops.cc @@ -737,6 +737,43 @@ REGISTER_OP("ResourceApplyAdam") return ApplyAdamShapeFn(c, false /* sparse */); }); +REGISTER_OP("ApplyAdaMax") + .Input("var: Ref(T)") + .Input("m: Ref(T)") + .Input("v: Ref(T)") + .Input("beta1_power: T") + .Input("beta2_power: T") + .Input("lr: T") + .Input("beta1: T") + .Input("beta2: T") + .Input("epsilon: T") + .Input("grad: T") + .Output("out: Ref(T)") + .Attr("T: numbertype") + .Attr("use_locking: bool = false") + .Attr("use_nesterov: bool = false") + .SetShapeFn([](InferenceContext* c) { + return ApplyAdamShapeFn(c, false /* sparse */); + }); + +REGISTER_OP("ResourceApplyAdaMax") + .Input("var: resource") + .Input("m: resource") + .Input("v: resource") + .Input("beta1_power: T") + .Input("beta2_power: T") + .Input("lr: T") + .Input("beta1: T") + .Input("beta2: T") + .Input("epsilon: T") + .Input("grad: T") + .Attr("T: numbertype") + .Attr("use_locking: bool = false") + .Attr("use_nesterov: bool = false") + .SetShapeFn([](InferenceContext* c) { + return ApplyAdamShapeFn(c, false /* sparse */); + }); + static Status ApplyRMSPropShapeFn(InferenceContext* c, bool sparse) { ShapeHandle unused; ShapeHandle s = ShapeOrHandleShape(c, 0); // var -- GitLab From 4d31dac8111b963ed427969c71c6957c929d3e5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Wed, 21 Feb 2018 20:29:46 +0800 Subject: [PATCH 016/791] ENH: add AdaMaxOptimizer in python side --- tensorflow/contrib/opt/BUILD | 20 +++ tensorflow/contrib/opt/__init__.py | 2 + .../contrib/opt/python/training/adamax.py | 72 ++++++++++ .../opt/python/training/adamax_test.py | 124 ++++++++++++++++++ tensorflow/core/kernels/training_ops.cc | 2 +- 5 files changed, 219 insertions(+), 1 deletion(-) create mode 100644 tensorflow/contrib/opt/python/training/adamax.py create mode 100644 tensorflow/contrib/opt/python/training/adamax_test.py diff --git a/tensorflow/contrib/opt/BUILD b/tensorflow/contrib/opt/BUILD index 86ceda71b7..a86d150f7a 100644 --- a/tensorflow/contrib/opt/BUILD +++ b/tensorflow/contrib/opt/BUILD @@ -14,6 +14,7 @@ py_library( name = "opt_py", srcs = [ "__init__.py", + "python/training/adamax.py", "python/training/addsign.py", "python/training/drop_stale_gradient_optimizer.py", "python/training/elastic_average_optimizer.py", @@ -48,6 +49,25 @@ py_library( ], ) +py_test( + name = "adamax_test", + srcs = ["python/training/adamax_test.py"], + srcs_version = "PY2AND3", + tags = [ + "no_oss", # b/73507407 + "notsan", # b/31055119 + ], + deps = [ + ":opt_py", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:math_ops", + "//tensorflow/python:training", + "//third_party/py/numpy", + ], +) + py_test( name = "external_optimizer_test", srcs = ["python/training/external_optimizer_test.py"], diff --git a/tensorflow/contrib/opt/__init__.py b/tensorflow/contrib/opt/__init__.py index 6c1bb1adc0..4c13c8e247 100644 --- a/tensorflow/contrib/opt/__init__.py +++ b/tensorflow/contrib/opt/__init__.py @@ -19,6 +19,7 @@ from __future__ import division from __future__ import print_function # pylint: disable=wildcard-import +from tensorflow.contrib.opt.python.training.adamax import * from tensorflow.contrib.opt.python.training.addsign import * from tensorflow.contrib.opt.python.training.drop_stale_gradient_optimizer import * from tensorflow.contrib.opt.python.training.external_optimizer import * @@ -36,6 +37,7 @@ from tensorflow.python.util.all_util import remove_undocumented _allowed_symbols = [ + 'AdaMaxOptimizer', 'PowerSignOptimizer', 'AddSignOptimizer', 'DelayCompensatedGradientDescentOptimizer', diff --git a/tensorflow/contrib/opt/python/training/adamax.py b/tensorflow/contrib/opt/python/training/adamax.py new file mode 100644 index 0000000000..4e0c541d3a --- /dev/null +++ b/tensorflow/contrib/opt/python/training/adamax.py @@ -0,0 +1,72 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""AdaMax for TensorFlow.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.eager import context +from tensorflow.python.framework import ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.training import optimizer +from tensorflow.python.training import adam +from tensorflow.python.training import training_ops +from tensorflow.python.util.tf_export import tf_export + + +@tf_export("train.AdaMaxOptimizer") +class AdaMaxOptimizer(adam.AdamOptimizer): + """Optimizer that implements the AdaMax algorithm. + + See [Kingma et al., 2014](http://arxiv.org/abs/1412.6980) + ([pdf](http://arxiv.org/pdf/1412.6980.pdf)). + """ + + def _apply_dense(self, grad, var): + m = self.get_slot(var, "m") + v = self.get_slot(var, "v") + beta1_power, beta2_power = self._get_beta_accumulators() + return training_ops.apply_ada_max( + var, m, v, + math_ops.cast(beta1_power, var.dtype.base_dtype), + math_ops.cast(beta2_power, var.dtype.base_dtype), + math_ops.cast(self._lr_t, var.dtype.base_dtype), + math_ops.cast(self._beta1_t, var.dtype.base_dtype), + math_ops.cast(self._beta2_t, var.dtype.base_dtype), + math_ops.cast(self._epsilon_t, var.dtype.base_dtype), + grad, use_locking=self._use_locking).op + + def _resource_apply_dense(self, grad, var): + m = self.get_slot(var, "m") + v = self.get_slot(var, "v") + beta1_power, beta2_power = self._get_beta_accumulators() + return training_ops.resource_apply_ada_max( + var.handle, m.handle, v.handle, + math_ops.cast(beta1_power, grad.dtype.base_dtype), + math_ops.cast(beta2_power, grad.dtype.base_dtype), + math_ops.cast(self._lr_t, grad.dtype.base_dtype), + math_ops.cast(self._beta1_t, grad.dtype.base_dtype), + math_ops.cast(self._beta2_t, grad.dtype.base_dtype), + math_ops.cast(self._epsilon_t, grad.dtype.base_dtype), + grad, use_locking=self._use_locking) + + def _apply_sparse_shared(self, grad, var, indices, scatter_add): + raise NotImplementedError() + + def _apply_sparse(self, grad, var): + raise NotImplementedError() diff --git a/tensorflow/contrib/opt/python/training/adamax_test.py b/tensorflow/contrib/opt/python/training/adamax_test.py new file mode 100644 index 0000000000..a1499118dd --- /dev/null +++ b/tensorflow/contrib/opt/python/training/adamax_test.py @@ -0,0 +1,124 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for AdaMax.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.contrib.opt.python.training import adamax +from tensorflow.python.client import session +from tensorflow.python.eager import context +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import variables +from tensorflow.python.platform import test + + +def adamax_update_numpy(param, + g_t, + t, + m, + v, + alpha=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8): + m_t = beta1 * m + (1 - beta1) * g_t + v_t = np.maximum(beta2 * v, np.abs(g_t)) + param_t = param - (alpha / (1 - beta1**t)) * m_t / v_t + return param_t, m_t, v_t + + +class AdaMaxOptimizerTest(test.TestCase): + + def doTestBasic(self, use_resource=False): + for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): + with self.test_session(graph=ops.Graph()): + # Initialize variables for numpy implementation. + m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) + + if use_resource: + var0 = resource_variable_ops.ResourceVariable( + var0_np, name="var0_%d" % i) + var1 = resource_variable_ops.ResourceVariable( + var1_np, name="var1_%d" % i) + else: + var0 = variables.Variable(var0_np) + var1 = variables.Variable(var1_np) + grads0 = constant_op.constant(grads0_np) + grads1 = constant_op.constant(grads1_np) + + opt = adamax.AdaMaxOptimizer() + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + opt_variables = opt.variables() + beta1_power, beta2_power = opt._get_beta_accumulators() + self.assertTrue(beta1_power is not None) + self.assertTrue(beta2_power is not None) + self.assertIn(beta1_power, opt_variables) + self.assertIn(beta2_power, opt_variables) + + with ops.Graph().as_default(): + # Shouldn't return non-slot variables from other graphs. + self.assertEqual(0, len(opt.variables())) + + if context.in_graph_mode(): + self.evaluate(variables.global_variables_initializer()) + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) + + beta1_power, beta2_power = opt._get_beta_accumulators() + + # Run 3 steps of Adam + for t in range(1, 4): + if context.in_graph_mode(): + self.evaluate(update) + elif t > 1: + opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + + self.assertAllCloseAccordingToType(0.9**(t + 1), + self.evaluate(beta1_power)) + self.assertAllCloseAccordingToType(0.999**(t + 1), + self.evaluate(beta2_power)) + + var0_np, m0, v0 = adamax_update_numpy(var0_np, grads0_np, t, m0, v0) + var1_np, m1, v1 = adamax_update_numpy(var1_np, grads1_np, t, m1, v1) + + # Validate updated params + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) + if use_resource: + self.assertEqual("var0_%d/Adam:0" % (i,), + opt.get_slot(var=var0, name="m").name) + + def testBasic(self): + with self.test_session(): + self.doTestBasic(use_resource=False) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/core/kernels/training_ops.cc b/tensorflow/core/kernels/training_ops.cc index 7d383d980a..b3b53d9ee0 100644 --- a/tensorflow/core/kernels/training_ops.cc +++ b/tensorflow/core/kernels/training_ops.cc @@ -346,7 +346,7 @@ struct ApplyAdaMaxNonCuda { // v == u v.device(d) = (beta2() * v).cwiseMax(grad.abs()); // var == θ - var.device(d) -= (lr * m) / ((T(1) - beta1_power()) * v); + var.device(d) -= (lr() * m) / ((T(1) - beta1_power()) * v); } }; -- GitLab From ba258d530f1af5fbcc8c1b72637dc7b2177a48c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Fri, 2 Mar 2018 19:33:30 +0800 Subject: [PATCH 017/791] ENH: support sparse grad --- .../contrib/opt/python/training/adamax.py | 51 +++++++++++++++++-- .../opt/python/training/adamax_test.py | 2 +- tensorflow/core/kernels/training_ops.cc | 4 +- .../core/kernels/training_ops_gpu.cu.cc | 5 +- 4 files changed, 52 insertions(+), 10 deletions(-) diff --git a/tensorflow/contrib/opt/python/training/adamax.py b/tensorflow/contrib/opt/python/training/adamax.py index 4e0c541d3a..137fce769f 100644 --- a/tensorflow/contrib/opt/python/training/adamax.py +++ b/tensorflow/contrib/opt/python/training/adamax.py @@ -18,12 +18,12 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.python.eager import context from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.training import optimizer +from tensorflow.python.ops import state_ops from tensorflow.python.training import adam from tensorflow.python.training import training_ops from tensorflow.python.util.tf_export import tf_export @@ -65,8 +65,49 @@ class AdaMaxOptimizer(adam.AdamOptimizer): math_ops.cast(self._epsilon_t, grad.dtype.base_dtype), grad, use_locking=self._use_locking) - def _apply_sparse_shared(self, grad, var, indices, scatter_add): - raise NotImplementedError() + def _apply_sparse_shared(self, grad, var, indices, + scatter_add, scatter_update): + beta1_power, beta2_power = self._get_beta_accumulators() + beta1_power = math_ops.cast(beta1_power, var.dtype.base_dtype) + beta2_power = math_ops.cast(beta2_power, var.dtype.base_dtype) + lr_t = math_ops.cast(self._lr_t, var.dtype.base_dtype) + beta1_t = math_ops.cast(self._beta1_t, var.dtype.base_dtype) + beta2_t = math_ops.cast(self._beta2_t, var.dtype.base_dtype) + epsilon_t = math_ops.cast(self._epsilon_t, var.dtype.base_dtype) + # m_t = beta1 * m + (1 - beta1) * g_t + m = self.get_slot(var, "m") + m_slice = array_ops.gather(m, indices) + m_t_slice = m_slice * beta1_t + grad * (1 - beta1_t) + with ops.control_dependencies([m_t_slice]): + m_t = scatter_update(m, indices, m_t_slice) + # u_t = max(beta2 * u, abs(g_t)) + v = self.get_slot(var, "v") + v_slice = array_ops.gather(v, indices) + v_t_slice = math_ops.maximum(v_slice * beta2_t, math_ops.abs(grad)) + with ops.control_dependencies([v_t_slice]): + v_t = scatter_update(v, indices, v_t_slice) + # theta_t = theta - lr / (1 - beta1^t) * m_t / u_t + var_slice = -lr_t / (1 - beta1_power) * (m_t_slice / + (v_t_slice + epsilon_t)) + with ops.control_dependencies([var_slice]): + var_update = scatter_add(var, indices, var_slice) + return control_flow_ops.group(*[var_update, m_t, v_t]) def _apply_sparse(self, grad, var): - raise NotImplementedError() + return self._apply_sparse_shared( + grad.values, var, grad.indices, + lambda x, i, v: state_ops.scatter_add( # pylint: disable=g-long-lambda + x, i, v, use_locking=self._use_locking), + lambda x, i, v: state_ops.scatter_update( # pylint: disable=g-long-lambda + x, i, v, use_locking=self._use_locking)) + + def _resource_scatter_update(self, x, i, v): + with ops.control_dependencies( + [resource_variable_ops.resource_scatter_update( + x.handle, i, v)]): + return x.value() + + def _resource_apply_sparse(self, grad, var, indices): + return self._apply_sparse_shared( + grad, var, indices, + self._resource_scatter_add, self._resource_scatter_update) diff --git a/tensorflow/contrib/opt/python/training/adamax_test.py b/tensorflow/contrib/opt/python/training/adamax_test.py index a1499118dd..0e2ba0987a 100644 --- a/tensorflow/contrib/opt/python/training/adamax_test.py +++ b/tensorflow/contrib/opt/python/training/adamax_test.py @@ -45,7 +45,7 @@ def adamax_update_numpy(param, epsilon=1e-8): m_t = beta1 * m + (1 - beta1) * g_t v_t = np.maximum(beta2 * v, np.abs(g_t)) - param_t = param - (alpha / (1 - beta1**t)) * m_t / v_t + param_t = param - (alpha / (1 - beta1**t)) * m_t / (v_t + epsilon) return param_t, m_t, v_t diff --git a/tensorflow/core/kernels/training_ops.cc b/tensorflow/core/kernels/training_ops.cc index b3b53d9ee0..0387e3011e 100644 --- a/tensorflow/core/kernels/training_ops.cc +++ b/tensorflow/core/kernels/training_ops.cc @@ -346,7 +346,7 @@ struct ApplyAdaMaxNonCuda { // v == u v.device(d) = (beta2() * v).cwiseMax(grad.abs()); // var == θ - var.device(d) -= (lr() * m) / ((T(1) - beta1_power()) * v); + var.device(d) -= lr() / (T(1) - beta1_power()) * (m / (v + epsilon())); } }; @@ -359,7 +359,7 @@ struct ApplyAdaMaxSYCL { T epsilon, typename TTypes::ConstFlat grad) { m.device(d) += (grad - m) * (T(1) - beta1); v.device(d) = (beta2 * v).cwiseMax(grad.abs()); - var.device(d) -= (lr * m) / ((T(1) - beta1_power) * v); + var.device(d) -= lr / (T(1) - beta1_power) * (m / (v + epsilon)); } }; #endif // TENSORFLOW_USE_SYCL diff --git a/tensorflow/core/kernels/training_ops_gpu.cu.cc b/tensorflow/core/kernels/training_ops_gpu.cu.cc index 1776c108ab..54c06b130c 100644 --- a/tensorflow/core/kernels/training_ops_gpu.cu.cc +++ b/tensorflow/core/kernels/training_ops_gpu.cu.cc @@ -163,8 +163,9 @@ struct ApplyAdaMax { v.device(d) = (beta2.reshape(single).broadcast(bcast) * v).cwiseMax(grad.abs()); var.device(d) -= - (lr * m) / ((beta1_power.constant(one) - - beta1_power).reshape(single).broadcast(bcast) * v); + lr / (beta1_power.constant(one) - + beta1_power).reshape(single).broadcast(bcast) * + (m / (v + epsilon)); } }; -- GitLab From f6f5a6019970bb8d667819da7d6316a8088a0b78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Sat, 3 Mar 2018 10:02:43 +0800 Subject: [PATCH 018/791] DOC: add docment --- .../contrib/opt/python/training/adamax.py | 51 ++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/opt/python/training/adamax.py b/tensorflow/contrib/opt/python/training/adamax.py index 137fce769f..ddae06bec7 100644 --- a/tensorflow/contrib/opt/python/training/adamax.py +++ b/tensorflow/contrib/opt/python/training/adamax.py @@ -29,7 +29,6 @@ from tensorflow.python.training import training_ops from tensorflow.python.util.tf_export import tf_export -@tf_export("train.AdaMaxOptimizer") class AdaMaxOptimizer(adam.AdamOptimizer): """Optimizer that implements the AdaMax algorithm. @@ -37,6 +36,56 @@ class AdaMaxOptimizer(adam.AdamOptimizer): ([pdf](http://arxiv.org/pdf/1412.6980.pdf)). """ + def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8, + use_locking=False, name="AdaMax"): + """Construct a new AdaMax optimizer. + + Initialization: + + ``` + m_0 <- 0 (Initialize initial 1st moment vector) + v_0 <- 0 (Initialize the exponentially weighted infinity norm) + t <- 0 (Initialize timestep) + ``` + + The update rule for `variable` with gradient `g` uses an optimization + described at the end of section7.1 of the paper: + + ``` + t <- t + 1 + lr_t <- learning_rate * sqrt(1 - beta2^t) / (1 - beta1^t) + + m_t <- beta1 * m_{t-1} + (1 - beta1) * g + v_t <- max(beta2 * v_{t-1}, abs(g)) + variable <- variable - lr_t / (1 - beta1^t) * m_t / (v_t + epsilon) + ``` + + Similar to AdamOptimizer, the epsilon is added for numerical stability + (especially to get rid of division by zero when v_t = 0). + + Contrast to AdamOptimizer, the sparse implementation of this algorithm + (used when the gradient is an IndexedSlices object, typically because of + `tf.gather` or an embedding lookup in the forward pass) only updates + variable slices and corresponding `m_t`, `v_t` terms when that part of + the variable was used in the forward pass. This means that the sparse + behavior is contrast to the dense behavior (similar to some momentum + implementations which ignore momentum unless a variable slice was actually + used). + + Args: + learning_rate: A Tensor or a floating point value. The learning rate. + beta1: A float value or a constant float tensor. + The exponential decay rate for the 1st moment estimates. + beta2: A float value or a constant float tensor. + The exponential decay rate for the exponentially weighted infinity norm. + epsilon: A small constant for numerical stability. + use_locking: If True use locks for update operations. + name: Optional name for the operations created when applying gradients. + Defaults to "AdaMax". + """ + super(AdaMaxOptimizer, self).__init__(learning_rate, beta1, beta2, + epsilon, use_locking, name) + def _apply_dense(self, grad, var): m = self.get_slot(var, "m") v = self.get_slot(var, "v") -- GitLab From f750e21a63c8836b9e7243ce786af2de3f65cc3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Sat, 3 Mar 2018 12:31:54 +0800 Subject: [PATCH 019/791] TST: add more tests --- .../contrib/opt/python/training/adamax.py | 2 +- .../opt/python/training/adamax_test.py | 243 +++++++++++++++++- 2 files changed, 233 insertions(+), 12 deletions(-) diff --git a/tensorflow/contrib/opt/python/training/adamax.py b/tensorflow/contrib/opt/python/training/adamax.py index ddae06bec7..36d49d4cbf 100644 --- a/tensorflow/contrib/opt/python/training/adamax.py +++ b/tensorflow/contrib/opt/python/training/adamax.py @@ -159,4 +159,4 @@ class AdaMaxOptimizer(adam.AdamOptimizer): def _resource_apply_sparse(self, grad, var, indices): return self._apply_sparse_shared( grad, var, indices, - self._resource_scatter_add, self._resource_scatter_update) + self._resource_scatter_add, self._resource_scatter_update) diff --git a/tensorflow/contrib/opt/python/training/adamax_test.py b/tensorflow/contrib/opt/python/training/adamax_test.py index 0e2ba0987a..e91e5cb96a 100644 --- a/tensorflow/contrib/opt/python/training/adamax_test.py +++ b/tensorflow/contrib/opt/python/training/adamax_test.py @@ -35,22 +35,142 @@ from tensorflow.python.platform import test def adamax_update_numpy(param, - g_t, - t, - m, - v, - alpha=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8): + g_t, + t, + m, + v, + alpha=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8): m_t = beta1 * m + (1 - beta1) * g_t v_t = np.maximum(beta2 * v, np.abs(g_t)) - param_t = param - (alpha / (1 - beta1**t)) * m_t / (v_t + epsilon) + param_t = param - (alpha / (1 - beta1**t)) * (m_t / (v_t + epsilon)) + return param_t, m_t, v_t + + +def adamax_sparse_update_numpy(param, + indices, + g_t, + t, + m, + v, + alpha=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8): + m_t, v_t, param_t = np.copy(m), np.copy(v), np.copy(param) + m_t_slice = beta1 * m[indices] + (1 - beta1) * g_t + v_t_slice = np.maximum(beta2 * v[indices], np.abs(g_t)) + param_t_slice = param[indices] - ((alpha / (1 - beta1**t)) * + (m_t_slice / (v_t_slice + epsilon))) + m_t[indices] = m_t_slice + v_t[indices] = v_t_slice + param_t[indices] = param_t_slice return param_t, m_t, v_t class AdaMaxOptimizerTest(test.TestCase): + def doTestSparse(self, use_resource=False): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.test_session(): + # Initialize variables for numpy implementation. + zero_slots = lambda: np.zeros((3), dtype=dtype.as_numpy_dtype) + m0, v0, m1, v1 = zero_slots(), zero_slots(), zero_slots(), zero_slots() + var0_np = np.array([1.0, 2.0, 3.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([4.0, 5.0, 6.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) + + if use_resource: + var0 = resource_variable_ops.ResourceVariable(var0_np) + var1 = resource_variable_ops.ResourceVariable(var1_np) + else: + var0 = variables.Variable(var0_np) + var1 = variables.Variable(var1_np) + grads0_np_indices = np.array([0, 1], dtype=np.int32) + grads0 = ops.IndexedSlices( + constant_op.constant(grads0_np), + constant_op.constant(grads0_np_indices), constant_op.constant([2])) + grads1_np_indices = np.array([2, 1], dtype=np.int32) + grads1 = ops.IndexedSlices( + constant_op.constant(grads1_np), + constant_op.constant(grads1_np_indices), constant_op.constant([2])) + opt = adamax.AdaMaxOptimizer() + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + variables.global_variables_initializer().run() + + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0, 3.0], var0.eval()) + self.assertAllClose([4.0, 5.0, 6.0], var1.eval()) + + beta1_power, beta2_power = opt._get_beta_accumulators() + + # Run 3 steps of AdaMax + for t in range(1, 4): + self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) + self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval()) + update.run() + + var0_np, m0, v0 = adamax_sparse_update_numpy( + var0_np, grads0_np_indices, grads0_np, t, m0, v0) + var1_np, m1, v1 = adamax_sparse_update_numpy( + var1_np, grads1_np_indices, grads1_np, t, m1, v1) + + # Validate updated params + self.assertAllCloseAccordingToType(var0_np, var0.eval()) + self.assertAllCloseAccordingToType(var1_np, var1.eval()) + + def testSparse(self): + self.doTestSparse(use_resource=False) + + def testResourceSparse(self): + self.doTestSparse(use_resource=True) + + def testSparseDevicePlacement(self): + for index_dtype in [dtypes.int32, dtypes.int64]: + with self.test_session(force_gpu=test.is_gpu_available()): + # If a GPU is available, tests that all optimizer ops can be placed on + # it (i.e. they have GPU kernels). + var = variables.Variable([[1.0], [2.0]]) + indices = constant_op.constant([0, 1], dtype=index_dtype) + gathered_sum = math_ops.reduce_sum(array_ops.gather(var, indices)) + optimizer = adamax.AdaMaxOptimizer(3.0) + minimize_op = optimizer.minimize(gathered_sum) + variables.global_variables_initializer().run() + minimize_op.run() + + def testSparseRepeatedIndices(self): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.test_session(): + repeated_index_update_var = variables.Variable( + [[1.0], [2.0]], dtype=dtype) + aggregated_update_var = variables.Variable( + [[1.0], [2.0]], dtype=dtype) + grad_repeated_index = ops.IndexedSlices( + constant_op.constant( + [0.1, 0.1], shape=[2, 1], dtype=dtype), + constant_op.constant([1, 1]), + constant_op.constant([2, 1])) + grad_aggregated = ops.IndexedSlices( + constant_op.constant( + [0.2], shape=[1, 1], dtype=dtype), + constant_op.constant([1]), + constant_op.constant([2, 1])) + repeated_update = adamax.AdaMaxOptimizer().apply_gradients( + [(grad_repeated_index, repeated_index_update_var)]) + aggregated_update = adamax.AdaMaxOptimizer().apply_gradients( + [(grad_aggregated, aggregated_update_var)]) + variables.global_variables_initializer().run() + self.assertAllClose(aggregated_update_var.eval(), + repeated_index_update_var.eval()) + for _ in range(3): + repeated_update.run() + aggregated_update.run() + self.assertAllClose(aggregated_update_var.eval(), + repeated_index_update_var.eval()) + def doTestBasic(self, use_resource=False): for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): with self.test_session(graph=ops.Graph()): @@ -93,7 +213,7 @@ class AdaMaxOptimizerTest(test.TestCase): beta1_power, beta2_power = opt._get_beta_accumulators() - # Run 3 steps of Adam + # Run 3 steps of AdaMax for t in range(1, 4): if context.in_graph_mode(): self.evaluate(update) @@ -112,13 +232,114 @@ class AdaMaxOptimizerTest(test.TestCase): self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) if use_resource: - self.assertEqual("var0_%d/Adam:0" % (i,), + self.assertEqual("var0_%d/AdaMax:0" % (i,), opt.get_slot(var=var0, name="m").name) def testBasic(self): with self.test_session(): self.doTestBasic(use_resource=False) + @test_util.run_in_graph_and_eager_modes(reset_test=True) + def testResourceBasic(self): + self.doTestBasic(use_resource=True) + + def testTensorLearningRate(self): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.test_session(): + # Initialize variables for numpy implementation. + m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) + + var0 = variables.Variable(var0_np) + var1 = variables.Variable(var1_np) + grads0 = constant_op.constant(grads0_np) + grads1 = constant_op.constant(grads1_np) + opt = adamax.AdaMaxOptimizer(constant_op.constant(0.001)) + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + variables.global_variables_initializer().run() + + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], var0.eval()) + self.assertAllClose([3.0, 4.0], var1.eval()) + + beta1_power, beta2_power = opt._get_beta_accumulators() + + # Run 3 steps of AdaMax + for t in range(1, 4): + self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) + self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval()) + update.run() + + var0_np, m0, v0 = adamax_update_numpy(var0_np, grads0_np, t, m0, v0) + var1_np, m1, v1 = adamax_update_numpy(var1_np, grads1_np, t, m1, v1) + + # Validate updated params + self.assertAllCloseAccordingToType(var0_np, var0.eval()) + self.assertAllCloseAccordingToType(var1_np, var1.eval()) + + def testSharing(self): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.test_session(): + # Initialize variables for numpy implementation. + m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) + + var0 = variables.Variable(var0_np) + var1 = variables.Variable(var1_np) + grads0 = constant_op.constant(grads0_np) + grads1 = constant_op.constant(grads1_np) + opt = adamax.AdaMaxOptimizer() + update1 = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + update2 = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + variables.global_variables_initializer().run() + + beta1_power, beta2_power = opt._get_beta_accumulators() + + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], var0.eval()) + self.assertAllClose([3.0, 4.0], var1.eval()) + + # Run 3 steps of intertwined AdaMax1 and AdaMax2. + for t in range(1, 4): + self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) + self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval()) + if t % 2 == 0: + update1.run() + else: + update2.run() + + var0_np, m0, v0 = adamax_update_numpy(var0_np, grads0_np, t, m0, v0) + var1_np, m1, v1 = adamax_update_numpy(var1_np, grads1_np, t, m1, v1) + + # Validate updated params + self.assertAllCloseAccordingToType(var0_np, var0.eval()) + self.assertAllCloseAccordingToType(var1_np, var1.eval()) + + def testTwoSessions(self): + optimizer = adamax.AdaMaxOptimizer() + g = ops.Graph() + with g.as_default(): + with session.Session(): + var0 = variables.Variable(np.array([1.0, 2.0]), name="v0") + grads0 = constant_op.constant(np.array([0.1, 0.1])) + optimizer.apply_gradients([(grads0, var0)]) + + gg = ops.Graph() + with gg.as_default(): + with session.Session(): + var0 = variables.Variable(np.array([1.0, 2.0]), name="v0") + grads0 = constant_op.constant(np.array([0.1, 0.1])) + + # If the optimizer saves any state not keyed by graph the following line + # fails. + optimizer.apply_gradients([(grads0, var0)]) + if __name__ == "__main__": test.main() -- GitLab From 8b5e4ad404ba16919ad4f17a763ee5383d61a400 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Sat, 3 Mar 2018 17:39:56 +0800 Subject: [PATCH 020/791] DOC: add apidef --- .../contrib/opt/python/training/adamax.py | 3 +- .../base_api/api_def_ApplyAdaMax.pbtxt | 89 +++++++++++++++++++ .../api_def_ResourceApplyAdaMax.pbtxt | 83 +++++++++++++++++ 3 files changed, 173 insertions(+), 2 deletions(-) create mode 100644 tensorflow/core/api_def/base_api/api_def_ApplyAdaMax.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_ResourceApplyAdaMax.pbtxt diff --git a/tensorflow/contrib/opt/python/training/adamax.py b/tensorflow/contrib/opt/python/training/adamax.py index 36d49d4cbf..fe5522a170 100644 --- a/tensorflow/contrib/opt/python/training/adamax.py +++ b/tensorflow/contrib/opt/python/training/adamax.py @@ -53,11 +53,10 @@ class AdaMaxOptimizer(adam.AdamOptimizer): ``` t <- t + 1 - lr_t <- learning_rate * sqrt(1 - beta2^t) / (1 - beta1^t) m_t <- beta1 * m_{t-1} + (1 - beta1) * g v_t <- max(beta2 * v_{t-1}, abs(g)) - variable <- variable - lr_t / (1 - beta1^t) * m_t / (v_t + epsilon) + variable <- variable - learning_rate / (1 - beta1^t) * m_t / (v_t + epsilon) ``` Similar to AdamOptimizer, the epsilon is added for numerical stability diff --git a/tensorflow/core/api_def/base_api/api_def_ApplyAdaMax.pbtxt b/tensorflow/core/api_def/base_api/api_def_ApplyAdaMax.pbtxt new file mode 100644 index 0000000000..106c30ca83 --- /dev/null +++ b/tensorflow/core/api_def/base_api/api_def_ApplyAdaMax.pbtxt @@ -0,0 +1,89 @@ +op { + graph_op_name: "ApplyAdaMax" + in_arg { + name: "var" + description: < Date: Mon, 5 Mar 2018 17:41:00 +0000 Subject: [PATCH 021/791] Update the documentation of `softmax_cross_entropy` This fix updates the documentation of `softmax_cross_entropy`, and removed the shape restrictions of `onehot_labels` and `logits`. They only needs to be of the same shape, not necessary `[batch_size, num_classes]`. This fix fixes 16263. Signed-off-by: Yong Tang --- tensorflow/python/ops/losses/losses_impl.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/ops/losses/losses_impl.py b/tensorflow/python/ops/losses/losses_impl.py index 7386976e93..04c13cb6c6 100644 --- a/tensorflow/python/ops/losses/losses_impl.py +++ b/tensorflow/python/ops/losses/losses_impl.py @@ -710,11 +710,16 @@ def softmax_cross_entropy( new_onehot_labels = onehot_labels * (1 - label_smoothing) + label_smoothing / num_classes + Note that `onehot_labels` and `logits` must have the same shape, + e.g. `[batch_size, num_classes]`. The shape of `weights` must be + broadcastable to loss, whose shape is decided by the shape of `logits`. + In case the shape of `logits` is `[batch_size, num_classes]`, loss is + a `Tensor` of shape `[batch_size]`. + Args: - onehot_labels: `[batch_size, num_classes]` target one-hot-encoded labels. - logits: `[batch_size, num_classes]` logits outputs of the network . - weights: Optional `Tensor` whose rank is either 0, or rank 1 and is - broadcastable to the loss which is a `Tensor` of shape `[batch_size]`. + onehot_labels: One-hot-encoded labels. + logits: Logits outputs of the network. + weights: Optional `Tensor` that is broadcastable to loss. label_smoothing: If greater than 0 then smooth the labels. scope: the scope for the operations performed in computing the loss. loss_collection: collection to which the loss will be added. -- GitLab From 82e34cd19f554509113d438ca98ad76e42fdf4e9 Mon Sep 17 00:00:00 2001 From: Hovhannes Harutyunyan Date: Wed, 7 Mar 2018 09:14:53 +0400 Subject: [PATCH 022/791] Remove quantized versiaon of Div till fixing it. --- .../internal/optimized/optimized_ops.h | 47 ------------------- 1 file changed, 47 deletions(-) diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h index b19f46beaa..9c181fddad 100644 --- a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h +++ b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h @@ -1973,53 +1973,6 @@ void BroadcastDiv(const T* input1_data, const Dims<4>& input1_dims, } } -inline void BroadcastDiv(const uint8* input1_data, const Dims<4>& input1_dims, - int32 input1_offset, const uint8* input2_data, - const Dims<4>& input2_dims, int32 input2_offset, - int32 output_offset, int32 output_multiplier, - int output_shift, int32 output_activation_min, - int32 output_activation_max, uint8* output_data, - const Dims<4>& output_dims) { - gemmlowp::ScopedProfilingLabel label("BroadcastDiv/8bit"); - - NdArrayDesc<4> desc1; - NdArrayDesc<4> desc2; - NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2); - - // In Tensorflow, the dimensions are canonically named (batch_number, row, - // col, channel), with extents (batches, height, width, depth), with the - // trailing dimension changing most rapidly (channels has the smallest stride, - // typically 1 element). - // - // In generated C code, we store arrays with the dimensions reversed. The - // first dimension has smallest stride. - // - // We name our variables by their Tensorflow convention, but generate C code - // nesting loops such that the innermost loop has the smallest stride for the - // best cache behavior. - for (int b = 0; b < ArraySize(output_dims, 3); ++b) { - for (int y = 0; y < ArraySize(output_dims, 2); ++y) { - for (int x = 0; x < ArraySize(output_dims, 1); ++x) { - for (int c = 0; c < ArraySize(output_dims, 0); ++c) { - const int32 input1_val = - input1_offset + input1_data[SubscriptToIndex(desc1, c, x, y, b)]; - const int32 input2_val = - input2_offset + input2_data[SubscriptToIndex(desc2, c, x, y, b)]; - const int32 unclamped_result = - output_offset + - MultiplyByQuantizedMultiplierSmallerThanOne( - input1_val / input2_val, output_multiplier, output_shift); - const int32 clamped_output = - std::min(output_activation_max, - std::max(output_activation_min, unclamped_result)); - output_data[Offset(output_dims, c, x, y, b)] = - static_cast(clamped_output); - } - } - } - } -} - // TODO(aselle): This is not actually optimized yet. inline void Sub(const float* input1_data, const Dims<4>& input1_dims, const float* input2_data, const Dims<4>& input2_dims, -- GitLab From f82d009d878dc675a307e69f89ba9f4dfdcd6c71 Mon Sep 17 00:00:00 2001 From: imsheridan Date: Wed, 7 Mar 2018 21:58:39 +0800 Subject: [PATCH 023/791] Fix broken link of typical distributed configuration in graphs.md --- tensorflow/docs_src/programmers_guide/graphs.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tensorflow/docs_src/programmers_guide/graphs.md b/tensorflow/docs_src/programmers_guide/graphs.md index e69b717432..ca74b17542 100644 --- a/tensorflow/docs_src/programmers_guide/graphs.md +++ b/tensorflow/docs_src/programmers_guide/graphs.md @@ -210,9 +210,8 @@ with tf.device("/device:GPU:0"): # Operations created in this context will be pinned to the GPU. result = tf.matmul(weights, img) ``` -If you are deploying TensorFlow in a @{$deploy/distributed$typical distributed configuration}, -you might specify the job name and task ID to place variables on -a task in the parameter server job (`"/job:ps"`), and the other operations on + +If you are deploying TensorFlow in a typical @{$deploy/distributed} configuration, you might specify the job name and task ID to place variables on a task in the parameter server job (`"/job:ps"`), and the other operations on task in the worker job (`"/job:worker"`): ```python -- GitLab From 04b6127510793b4c5aaa540b60b68ffdf3fd48ce Mon Sep 17 00:00:00 2001 From: imsheridan Date: Wed, 7 Mar 2018 22:23:50 +0800 Subject: [PATCH 024/791] revert the minor space nit --- tensorflow/docs_src/programmers_guide/graphs.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tensorflow/docs_src/programmers_guide/graphs.md b/tensorflow/docs_src/programmers_guide/graphs.md index ca74b17542..3b5e3e5a9a 100644 --- a/tensorflow/docs_src/programmers_guide/graphs.md +++ b/tensorflow/docs_src/programmers_guide/graphs.md @@ -210,8 +210,9 @@ with tf.device("/device:GPU:0"): # Operations created in this context will be pinned to the GPU. result = tf.matmul(weights, img) ``` - -If you are deploying TensorFlow in a typical @{$deploy/distributed} configuration, you might specify the job name and task ID to place variables on a task in the parameter server job (`"/job:ps"`), and the other operations on +If you are deploying TensorFlow in a typical @{$deploy/distributed} configuration, +you might specify the job name and task ID to place variables on +a task in the parameter server job (`"/job:ps"`), and the other operations on task in the worker job (`"/job:worker"`): ```python -- GitLab From 2548a3d2cf035a229d35ab6257bee511aa3a8e23 Mon Sep 17 00:00:00 2001 From: imsheridan Date: Thu, 8 Mar 2018 00:15:22 +0800 Subject: [PATCH 025/791] fix some typo --- tensorflow/docs_src/programmers_guide/graphs.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/docs_src/programmers_guide/graphs.md b/tensorflow/docs_src/programmers_guide/graphs.md index 3b5e3e5a9a..f28660d44a 100644 --- a/tensorflow/docs_src/programmers_guide/graphs.md +++ b/tensorflow/docs_src/programmers_guide/graphs.md @@ -505,10 +505,10 @@ multiple graphs in the same process. As noted above, TensorFlow provides a "default graph" that is implicitly passed to all API functions in the same context. For many applications, a single graph is sufficient. However, TensorFlow also provides methods for manipulating -the default graph, which can be useful in more advanced used cases. For example: +the default graph, which can be useful in more advanced use cases. For example: * A @{tf.Graph} defines the namespace for @{tf.Operation} objects: each - operation in a single graph must have a unique name. TensorFlow will + operation in a single graph must have an unique name. TensorFlow will "uniquify" the names of operations by appending `"_1"`, `"_2"`, and so on to their names if the requested name is already taken. Using multiple explicitly created graphs gives you more control over what name is given to each -- GitLab From cee41f9d10b81ce3b49f566ddd448a7f3f2872c3 Mon Sep 17 00:00:00 2001 From: KB Sriram Date: Wed, 7 Mar 2018 08:11:03 -0800 Subject: [PATCH 026/791] C++ gradient for StridedSlice See https://github.com/tensorflow/tensorflow/issues/9645 --- tensorflow/cc/gradients/array_grad.cc | 36 ++++++++++++++++++++++ tensorflow/cc/gradients/array_grad_test.cc | 24 +++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/tensorflow/cc/gradients/array_grad.cc b/tensorflow/cc/gradients/array_grad.cc index 6545e4ee3e..ff348fadb2 100644 --- a/tensorflow/cc/gradients/array_grad.cc +++ b/tensorflow/cc/gradients/array_grad.cc @@ -385,6 +385,42 @@ Status MirrorPadGradGrad(const Scope& scope, const Operation& op, } REGISTER_GRADIENT_OP("MirrorPadGrad", MirrorPadGradGrad); +Status StridedSliceGradHelper(const Scope& scope, const Operation& op, + const std::vector& grad_inputs, + std::vector* grad_outputs) { + Input x = Shape(scope, op.input(0)); + Input begin = op.input(1); + Input end = op.input(2); + Input strides = op.input(3); + int64 begin_mask; + int64 end_mask; + int64 ellipsis_mask; + int64 new_axis_mask; + int64 shrink_axis_mask; + TF_RETURN_IF_ERROR( + GetNodeAttr(op.node()->attrs(), "begin_mask", &begin_mask)); + TF_RETURN_IF_ERROR(GetNodeAttr(op.node()->attrs(), "end_mask", &end_mask)); + TF_RETURN_IF_ERROR( + GetNodeAttr(op.node()->attrs(), "ellipsis_mask", &ellipsis_mask)); + TF_RETURN_IF_ERROR( + GetNodeAttr(op.node()->attrs(), "new_axis_mask", &new_axis_mask)); + TF_RETURN_IF_ERROR( + GetNodeAttr(op.node()->attrs(), "shrink_axis_mask", &shrink_axis_mask)); + grad_outputs->push_back( + StridedSliceGrad(scope, x, begin, end, strides, grad_inputs[0], + StridedSliceGrad::BeginMask(begin_mask) + .EndMask(end_mask) + .EllipsisMask(ellipsis_mask) + .NewAxisMask(new_axis_mask) + .ShrinkAxisMask(shrink_axis_mask))); + // No gradients returned for begin, end and strides + grad_outputs->push_back(NoGradient()); + grad_outputs->push_back(NoGradient()); + grad_outputs->push_back(NoGradient()); + return scope.status(); +} +REGISTER_GRADIENT_OP("StridedSlice", StridedSliceGradHelper); + } // anonymous namespace } // namespace ops } // namespace tensorflow diff --git a/tensorflow/cc/gradients/array_grad_test.cc b/tensorflow/cc/gradients/array_grad_test.cc index 4a215fcc92..2a2180297c 100644 --- a/tensorflow/cc/gradients/array_grad_test.cc +++ b/tensorflow/cc/gradients/array_grad_test.cc @@ -354,5 +354,29 @@ TEST_F(ArrayGradTest, MirrorPadGradGrad_Symmetric) { RunTest(x, x_shape, y, y_shape); } +TEST_F(ArrayGradTest, StridedSliceGrad) { + TensorShape x_shape({6, 4, 4}); + auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(x_shape)); + + // y = x[2:6:2, 1:3, 1:3] + auto y = StridedSlice(scope_, x, {2, 1, 1}, {6, 3, 3}, {2, 1, 1}); + // y.shape = [2, 2, 2]; + RunTest(x, x_shape, y, {2, 2, 2}); + + // y = x[2:6:2, 1:3, 1:3] + // begin_mask = 1<<1 (ignore begin_index = 1) + // end_mask = 1<<2 (ignore end_index = 2) + y = StridedSlice(scope_, x, {2, 1, 1}, {6, 3, 3}, {2, 1, 1}, + StridedSlice::BeginMask(1<<1).EndMask(1<<2)); + // y.shape = [2, 3, 3]; + RunTest(x, x_shape, y, {2, 3, 3}); + + // y = [tf.newaxis, 2:6:2, 1:3, 1:3] + y = StridedSlice(scope_, x, {0, 2, 1, 1}, {0, 6, 3, 3}, {1, 2, 1, 1}, + StridedSlice::NewAxisMask(1<<0)); + // y.shape = [1, 2, 2, 2]; + RunTest(x, x_shape, y, {1, 2, 2, 2}); +} + } // namespace } // namespace tensorflow -- GitLab From e31fb25f4e3989a846a8e54d789a3bf5efff0cea Mon Sep 17 00:00:00 2001 From: KB Sriram Date: Thu, 8 Mar 2018 07:40:24 -0800 Subject: [PATCH 027/791] Clang-format fixes. --- tensorflow/cc/gradients/array_grad_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/cc/gradients/array_grad_test.cc b/tensorflow/cc/gradients/array_grad_test.cc index 2a2180297c..de3bd0fc9e 100644 --- a/tensorflow/cc/gradients/array_grad_test.cc +++ b/tensorflow/cc/gradients/array_grad_test.cc @@ -367,13 +367,13 @@ TEST_F(ArrayGradTest, StridedSliceGrad) { // begin_mask = 1<<1 (ignore begin_index = 1) // end_mask = 1<<2 (ignore end_index = 2) y = StridedSlice(scope_, x, {2, 1, 1}, {6, 3, 3}, {2, 1, 1}, - StridedSlice::BeginMask(1<<1).EndMask(1<<2)); + StridedSlice::BeginMask(1 << 1).EndMask(1 << 2)); // y.shape = [2, 3, 3]; RunTest(x, x_shape, y, {2, 3, 3}); // y = [tf.newaxis, 2:6:2, 1:3, 1:3] y = StridedSlice(scope_, x, {0, 2, 1, 1}, {0, 6, 3, 3}, {1, 2, 1, 1}, - StridedSlice::NewAxisMask(1<<0)); + StridedSlice::NewAxisMask(1 << 0)); // y.shape = [1, 2, 2, 2]; RunTest(x, x_shape, y, {1, 2, 2, 2}); } -- GitLab From d6533df7cd3ef19b39081a64fcb0bed5f83c7ee0 Mon Sep 17 00:00:00 2001 From: Giuseppe Date: Thu, 8 Mar 2018 17:49:29 +0100 Subject: [PATCH 028/791] Fix markdown error in layers tutorial. --- tensorflow/docs_src/tutorials/layers.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tensorflow/docs_src/tutorials/layers.md b/tensorflow/docs_src/tutorials/layers.md index ee03f440c9..b24d3f4cad 100644 --- a/tensorflow/docs_src/tutorials/layers.md +++ b/tensorflow/docs_src/tutorials/layers.md @@ -192,8 +192,7 @@ dive deeper into the `tf.layers` code used to create each layer, as well as how to calculate loss, configure the training op, and generate predictions. If you're already experienced with CNNs and @{$get_started/custom_estimators$TensorFlow `Estimator`s}, and find the above code intuitive, you may want to skim these sections or just -skip ahead to ["Training and Evaluating the CNN MNIST -Classifier"](#training-and-evaluating-the-cnn-mnist-classifier). +skip ahead to ["Training and Evaluating the CNN MNIST Classifier"](#training_and_evaluating_the_cnn_mnist_classifier). ### Input Layer @@ -534,9 +533,8 @@ if mode == tf.estimator.ModeKeys.TRAIN: ``` > Note: For a more in-depth look at configuring training ops for Estimator model -> functions, see @{$get_started/custom_estimators#defining-the-training-op-for-the-model$"Defining -> the training op for the model"} in the @{$get_started/custom_estimators$"Creating Estimations in -> tf.estimator"} tutorial. +> functions, see @{$get_started/custom_estimators#defining-the-training-op-for-the-model$"Defining the training op for the model"} +> in the @{$get_started/custom_estimators$"Creating Estimations in tf.estimator"} tutorial. ### Add evaluation metrics -- GitLab From b4db970c338123ee3156bb0e216193bde35d4b17 Mon Sep 17 00:00:00 2001 From: imsheridan Date: Tue, 13 Mar 2018 00:04:33 +0800 Subject: [PATCH 029/791] fix broken link of tensor-like type --- tensorflow/docs_src/programmers_guide/graphs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/docs_src/programmers_guide/graphs.md b/tensorflow/docs_src/programmers_guide/graphs.md index f28660d44a..81fd99cb4a 100644 --- a/tensorflow/docs_src/programmers_guide/graphs.md +++ b/tensorflow/docs_src/programmers_guide/graphs.md @@ -362,7 +362,7 @@ operations that are needed to compute the result. @{tf.Session.run} requires you to specify a list of **fetches**, which determine the return values, and may be a @{tf.Operation}, a @{tf.Tensor}, or -a [tensor-like type](#tensor-like-objects) such as @{tf.Variable}. These fetches +a [tensor-like type](#tensor-like_objects) such as @{tf.Variable}. These fetches determine what **subgraph** of the overall @{tf.Graph} must be executed to produce the result: this is the subgraph that contains all operations named in the fetch list, plus all operations whose outputs are used to compute the value -- GitLab From 1f03b013ef00c128cf8331f274524a23d86ac458 Mon Sep 17 00:00:00 2001 From: imsheridan Date: Tue, 13 Mar 2018 16:44:57 +0800 Subject: [PATCH 030/791] revert wrong typo fix --- tensorflow/docs_src/programmers_guide/graphs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/docs_src/programmers_guide/graphs.md b/tensorflow/docs_src/programmers_guide/graphs.md index 81fd99cb4a..69eb6df5f6 100644 --- a/tensorflow/docs_src/programmers_guide/graphs.md +++ b/tensorflow/docs_src/programmers_guide/graphs.md @@ -508,7 +508,7 @@ is sufficient. However, TensorFlow also provides methods for manipulating the default graph, which can be useful in more advanced use cases. For example: * A @{tf.Graph} defines the namespace for @{tf.Operation} objects: each - operation in a single graph must have an unique name. TensorFlow will + operation in a single graph must have a unique name. TensorFlow will "uniquify" the names of operations by appending `"_1"`, `"_2"`, and so on to their names if the requested name is already taken. Using multiple explicitly created graphs gives you more control over what name is given to each -- GitLab From d751b6bfa84dae1be9835fc40cc3094a8205a74e Mon Sep 17 00:00:00 2001 From: imsheridan Date: Tue, 13 Mar 2018 23:11:47 +0800 Subject: [PATCH 031/791] Fix link of typical distributed configuration --- tensorflow/docs_src/programmers_guide/graphs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/docs_src/programmers_guide/graphs.md b/tensorflow/docs_src/programmers_guide/graphs.md index 69eb6df5f6..e4095cf7dd 100644 --- a/tensorflow/docs_src/programmers_guide/graphs.md +++ b/tensorflow/docs_src/programmers_guide/graphs.md @@ -210,7 +210,7 @@ with tf.device("/device:GPU:0"): # Operations created in this context will be pinned to the GPU. result = tf.matmul(weights, img) ``` -If you are deploying TensorFlow in a typical @{$deploy/distributed} configuration, +If you are deploying TensorFlow in a @{$distributed$typical distributed configuration}, you might specify the job name and task ID to place variables on a task in the parameter server job (`"/job:ps"`), and the other operations on task in the worker job (`"/job:worker"`): -- GitLab From b618740a8754e85a2a6ee142028105f76a4d5d58 Mon Sep 17 00:00:00 2001 From: Wenhao Hu Date: Fri, 16 Mar 2018 00:11:38 +0900 Subject: [PATCH 032/791] implement matrix 2-norm --- tensorflow/python/ops/linalg_ops.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tensorflow/python/ops/linalg_ops.py b/tensorflow/python/ops/linalg_ops.py index 37470e00d7..110b766a6e 100644 --- a/tensorflow/python/ops/linalg_ops.py +++ b/tensorflow/python/ops/linalg_ops.py @@ -454,7 +454,7 @@ def norm(tensor, This function can compute several different vector norms (the 1-norm, the Euclidean or 2-norm, the inf-norm, and in general the p-norm for p > 0) and - matrix norms (Frobenius, 1-norm, and inf-norm). + matrix norms (Frobenius, 1-norm, 2-norm and inf-norm). Args: tensor: `Tensor` of types `float32`, `float64`, `complex64`, `complex128` @@ -465,7 +465,7 @@ def norm(tensor, Some restrictions apply: a) The Frobenius norm `fro` is not defined for vectors, b) If axis is a 2-tuple (matrix norm), only 'euclidean', 'fro', `1`, - `np.inf` are supported. + `2`, `np.inf` are supported. See the description of `axis` on how to compute norms for a batch of vectors or matrices stored in a tensor. axis: If `axis` is `None` (the default), the input is considered a vector @@ -521,8 +521,7 @@ def norm(tensor, axis[0] == axis[1]): raise ValueError( "'axis' must be None, an integer, or a tuple of 2 unique integers") - # TODO(rmlarsen): Implement matrix 2-norm using tf.svd(). - supported_matrix_norms = ['euclidean', 'fro', 1, np.inf] + supported_matrix_norms = ['euclidean', 'fro', 1, 2, np.inf] if ord not in supported_matrix_norms: raise ValueError("'ord' must be a supported matrix norm in %s, got %s" % (supported_matrix_norms, ord)) @@ -539,10 +538,20 @@ def norm(tensor, with ops.name_scope(name, 'norm', [tensor]): tensor = ops.convert_to_tensor(tensor) + rank = len(tensor.get_shape().as_list()) + axis = tuple(map(lambda i: i if i >= 0 else i + rank, axis)) + if ord in ['fro', 'euclidean', 2, 2.0]: - # TODO(rmlarsen): Move 2-norm to a separate clause once we support it for - # matrices. - result = math_ops.sqrt( + if is_matrix_norm and ord in [2, 2.0]: + axes = list(range(rank)) + perm_before = list(filter(lambda i: i not in axis, axes)) + list(axis) + perm_after = list(map(lambda i: perm_before.index(i), axes)) + result = array_ops.transpose(array_ops.expand_dims(math_ops.reduce_max( + gen_linalg_ops.svd(array_ops.transpose(tensor, perm=perm_before), + compute_uv=False)[0], axis=-1, keepdims=True), + axis=-1), perm=perm_after) + else: + result = math_ops.sqrt( math_ops.reduce_sum( tensor * math_ops.conj(tensor), axis, keepdims=True)) else: -- GitLab From a280a1d0cfd64831857826db639a3ee0180094de Mon Sep 17 00:00:00 2001 From: Wenhao Hu Date: Fri, 16 Mar 2018 00:32:34 +0900 Subject: [PATCH 033/791] follow python coding style --- tensorflow/python/ops/linalg_ops.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/ops/linalg_ops.py b/tensorflow/python/ops/linalg_ops.py index 110b766a6e..b467711e3b 100644 --- a/tensorflow/python/ops/linalg_ops.py +++ b/tensorflow/python/ops/linalg_ops.py @@ -546,14 +546,15 @@ def norm(tensor, axes = list(range(rank)) perm_before = list(filter(lambda i: i not in axis, axes)) + list(axis) perm_after = list(map(lambda i: perm_before.index(i), axes)) - result = array_ops.transpose(array_ops.expand_dims(math_ops.reduce_max( - gen_linalg_ops.svd(array_ops.transpose(tensor, perm=perm_before), - compute_uv=False)[0], axis=-1, keepdims=True), - axis=-1), perm=perm_after) + result = array_ops.transpose(array_ops.expand_dims( + math_ops.reduce_max(gen_linalg_ops.svd( + array_ops.transpose(tensor, perm=perm_before), + compute_uv=False)[0], axis=-1, keepdims=True), axis=-1), + perm=perm_after) else: result = math_ops.sqrt( - math_ops.reduce_sum( - tensor * math_ops.conj(tensor), axis, keepdims=True)) + math_ops.reduce_sum( + tensor * math_ops.conj(tensor), axis, keepdims=True)) else: result = math_ops.abs(tensor) if ord == 1: -- GitLab From cc10ac9b7d593375a7cee0c167c20989dc29e8cf Mon Sep 17 00:00:00 2001 From: Wenhao Hu Date: Fri, 16 Mar 2018 00:40:05 +0900 Subject: [PATCH 034/791] remove unnecessary lambda --- tensorflow/python/ops/linalg_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/ops/linalg_ops.py b/tensorflow/python/ops/linalg_ops.py index b467711e3b..db6ce71125 100644 --- a/tensorflow/python/ops/linalg_ops.py +++ b/tensorflow/python/ops/linalg_ops.py @@ -545,7 +545,7 @@ def norm(tensor, if is_matrix_norm and ord in [2, 2.0]: axes = list(range(rank)) perm_before = list(filter(lambda i: i not in axis, axes)) + list(axis) - perm_after = list(map(lambda i: perm_before.index(i), axes)) + perm_after = list(map(perm_before.index, axes)) result = array_ops.transpose(array_ops.expand_dims( math_ops.reduce_max(gen_linalg_ops.svd( array_ops.transpose(tensor, perm=perm_before), -- GitLab From b21ceeb518ca9462a247d8be05870f12bebad201 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 15 Mar 2018 23:13:25 -0700 Subject: [PATCH 035/791] Enhancement with deprecated_argument_lookup for argmax This fix makes some enhancement for argmax, using deprecated_argument_lookup instread of customerized logic. Signed-off-by: Yong Tang --- tensorflow/python/ops/math_ops.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py index e18d0e9501..9a88b71398 100644 --- a/tensorflow/python/ops/math_ops.py +++ b/tensorflow/python/ops/math_ops.py @@ -208,11 +208,9 @@ def argmax(input, name=None, dimension=None, output_type=dtypes.int64): - if dimension is not None: - if axis is not None: - raise ValueError("Cannot specify both 'axis' and 'dimension'") - axis = dimension - elif axis is None: + axis = deprecation.deprecated_argument_lookup( + "axis", axis, "dimension", dimension) + if axis is None: axis = 0 return gen_math_ops.arg_max(input, axis, name=name, output_type=output_type) -- GitLab From 82571ca199869f60fe2036d15d0071031d997b47 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 15 Mar 2018 23:15:37 -0700 Subject: [PATCH 036/791] Enhancement with deprecated_argument_lookup for argmin This fix makes some enhancement for argmin, using deprecated_argument_lookup instread of customerized logic. Signed-off-by: Yong Tang --- tensorflow/python/ops/math_ops.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py index 9a88b71398..a2892d206d 100644 --- a/tensorflow/python/ops/math_ops.py +++ b/tensorflow/python/ops/math_ops.py @@ -226,11 +226,9 @@ def argmin(input, name=None, dimension=None, output_type=dtypes.int64): - if dimension is not None: - if axis is not None: - raise ValueError("Cannot specify both 'axis' and 'dimension'") - axis = dimension - elif axis is None: + axis = deprecation.deprecated_argument_lookup( + "axis", axis, "dimension", dimension) + if axis is None: axis = 0 return gen_math_ops.arg_min(input, axis, name=name, output_type=output_type) -- GitLab From 52fef7f6b8b41d4fffa92bddcb78d96eb6333051 Mon Sep 17 00:00:00 2001 From: ManHyuk Date: Fri, 16 Mar 2018 16:03:26 +0900 Subject: [PATCH 037/791] fix typo --- tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc b/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc index 272410c693..7651a03fe5 100644 --- a/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc +++ b/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc @@ -398,7 +398,7 @@ TEST_F(FoldOldBatchNormsTest, TestFoldFusedBatchNorms) { } TEST_F(FoldOldBatchNormsTest, TestFoldFusedBatchNormsWithConcat) { - // Test axis is not 3, so all weigths and offsets are fused to each of inputs + // Test axis is not 3, so all weights and offsets are fused to each of inputs // of conv2d. TestFoldFusedBatchNormsWithConcat(/*split=*/true); // Test axis = 3, BatchNorm weights and offsets will be split before fused -- GitLab From 20424e92417b520d7ea8c7323eee46538d2b909f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Sat, 17 Mar 2018 09:30:24 +0800 Subject: [PATCH 038/791] CLN: remove the unused import: tf_export --- tensorflow/contrib/opt/python/training/adamax.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/contrib/opt/python/training/adamax.py b/tensorflow/contrib/opt/python/training/adamax.py index fe5522a170..65918831e9 100644 --- a/tensorflow/contrib/opt/python/training/adamax.py +++ b/tensorflow/contrib/opt/python/training/adamax.py @@ -26,7 +26,6 @@ from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import state_ops from tensorflow.python.training import adam from tensorflow.python.training import training_ops -from tensorflow.python.util.tf_export import tf_export class AdaMaxOptimizer(adam.AdamOptimizer): -- GitLab From b5ebb7e9e5f5ae59e6db93bb5950f4bb68bf9e18 Mon Sep 17 00:00:00 2001 From: Wenhao Hu Date: Sun, 18 Mar 2018 00:48:46 +0900 Subject: [PATCH 039/791] update norm_op_test --- tensorflow/python/kernel_tests/norm_op_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tensorflow/python/kernel_tests/norm_op_test.py b/tensorflow/python/kernel_tests/norm_op_test.py index d85512fae6..d6625b69ef 100644 --- a/tensorflow/python/kernel_tests/norm_op_test.py +++ b/tensorflow/python/kernel_tests/norm_op_test.py @@ -85,8 +85,6 @@ def _GetNormOpTest(dtype_, shape_, ord_, axis_, keep_dims_, use_static_shape_): if ((not is_matrix_norm and ord_ == "fro") or (is_matrix_norm and is_fancy_p_norm)): self.skipTest("Not supported by neither numpy.linalg.norm nor tf.norm") - if is_matrix_norm and ord_ == 2: - self.skipTest("Not supported by tf.norm") if ord_ == 'euclidean' or (axis_ is None and len(shape) > 2): self.skipTest("Not supported by numpy.linalg.norm") matrix = np.random.randn(*shape_).astype(dtype_) -- GitLab From c53160a2a5decdae30bda6e8f40b45f3b4dd9f8e Mon Sep 17 00:00:00 2001 From: Wenhao Hu Date: Sun, 18 Mar 2018 00:49:13 +0900 Subject: [PATCH 040/791] use tf function instead of np --- tensorflow/python/ops/linalg_ops.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/tensorflow/python/ops/linalg_ops.py b/tensorflow/python/ops/linalg_ops.py index db6ce71125..d8150d85b9 100644 --- a/tensorflow/python/ops/linalg_ops.py +++ b/tensorflow/python/ops/linalg_ops.py @@ -24,6 +24,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import functional_ops from tensorflow.python.ops import gen_linalg_ops from tensorflow.python.ops import math_ops # pylint: disable=wildcard-import @@ -538,19 +539,27 @@ def norm(tensor, with ops.name_scope(name, 'norm', [tensor]): tensor = ops.convert_to_tensor(tensor) - rank = len(tensor.get_shape().as_list()) - axis = tuple(map(lambda i: i if i >= 0 else i + rank, axis)) if ord in ['fro', 'euclidean', 2, 2.0]: if is_matrix_norm and ord in [2, 2.0]: - axes = list(range(rank)) - perm_before = list(filter(lambda i: i not in axis, axes)) + list(axis) - perm_after = list(map(perm_before.index, axes)) - result = array_ops.transpose(array_ops.expand_dims( - math_ops.reduce_max(gen_linalg_ops.svd( - array_ops.transpose(tensor, perm=perm_before), - compute_uv=False)[0], axis=-1, keepdims=True), axis=-1), - perm=perm_after) + rank = array_ops.rank(tensor) + axis = functional_ops.map_fn( + lambda i: control_flow_ops.cond(i >= 0, lambda: i, + lambda: i + rank), + ops.convert_to_tensor(axis)).eval() + axes = math_ops.range(rank) + perm_before = array_ops.concat( + [array_ops.setdiff1d(axes, axis)[0], axis], axis=0) + perm_after = functional_ops.map_fn( + lambda i: math_ops.cast( + array_ops.squeeze( + array_ops.where(math_ops.equal(perm_before, i))), + dtype=dtypes.int32), axes) + permed = array_ops.transpose(tensor, perm=perm_before) + matrix_2_norm = array_ops.expand_dims( + math_ops.reduce_max(gen_linalg_ops.svd(permed, compute_uv=False)[0], + axis=-1, keepdims=True), axis=-1) + result = array_ops.transpose(matrix_2_norm, perm=perm_after) else: result = math_ops.sqrt( math_ops.reduce_sum( -- GitLab From fda633fb7187da8522ef79555d1267996fa983bc Mon Sep 17 00:00:00 2001 From: Wenhao Hu Date: Sun, 18 Mar 2018 21:29:16 +0900 Subject: [PATCH 041/791] remove test code --- tensorflow/python/ops/linalg_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/ops/linalg_ops.py b/tensorflow/python/ops/linalg_ops.py index d8150d85b9..608b72c574 100644 --- a/tensorflow/python/ops/linalg_ops.py +++ b/tensorflow/python/ops/linalg_ops.py @@ -546,7 +546,7 @@ def norm(tensor, axis = functional_ops.map_fn( lambda i: control_flow_ops.cond(i >= 0, lambda: i, lambda: i + rank), - ops.convert_to_tensor(axis)).eval() + ops.convert_to_tensor(axis)) axes = math_ops.range(rank) perm_before = array_ops.concat( [array_ops.setdiff1d(axes, axis)[0], axis], axis=0) -- GitLab From 07502453382cc007f42818118a592220a8c7d849 Mon Sep 17 00:00:00 2001 From: "wenhao.hu" Date: Wed, 28 Mar 2018 10:25:47 +0900 Subject: [PATCH 042/791] clean the pollution of axis --- tensorflow/python/ops/linalg_ops.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/ops/linalg_ops.py b/tensorflow/python/ops/linalg_ops.py index 608b72c574..86be1e7752 100644 --- a/tensorflow/python/ops/linalg_ops.py +++ b/tensorflow/python/ops/linalg_ops.py @@ -543,13 +543,12 @@ def norm(tensor, if ord in ['fro', 'euclidean', 2, 2.0]: if is_matrix_norm and ord in [2, 2.0]: rank = array_ops.rank(tensor) - axis = functional_ops.map_fn( - lambda i: control_flow_ops.cond(i >= 0, lambda: i, - lambda: i + rank), + positive_axis = functional_ops.map_fn( + lambda i: control_flow_ops.cond(i >= 0, lambda: i, lambda: i + rank), ops.convert_to_tensor(axis)) axes = math_ops.range(rank) perm_before = array_ops.concat( - [array_ops.setdiff1d(axes, axis)[0], axis], axis=0) + [array_ops.setdiff1d(axes, positive_axis)[0], positive_axis], axis=0) perm_after = functional_ops.map_fn( lambda i: math_ops.cast( array_ops.squeeze( @@ -557,8 +556,11 @@ def norm(tensor, dtype=dtypes.int32), axes) permed = array_ops.transpose(tensor, perm=perm_before) matrix_2_norm = array_ops.expand_dims( - math_ops.reduce_max(gen_linalg_ops.svd(permed, compute_uv=False)[0], - axis=-1, keepdims=True), axis=-1) + math_ops.reduce_max( + gen_linalg_ops.svd(permed, compute_uv=False)[0], + axis=-1, + keepdims=True), + axis=-1) result = array_ops.transpose(matrix_2_norm, perm=perm_after) else: result = math_ops.sqrt( -- GitLab From e9ea69058974d9155851c6325362dc3cb188cefb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Wed, 28 Mar 2018 10:22:31 +0800 Subject: [PATCH 043/791] CLN: remove no_oss, notsan tags --- tensorflow/contrib/opt/BUILD | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tensorflow/contrib/opt/BUILD b/tensorflow/contrib/opt/BUILD index a86d150f7a..aaf0012808 100644 --- a/tensorflow/contrib/opt/BUILD +++ b/tensorflow/contrib/opt/BUILD @@ -53,10 +53,6 @@ py_test( name = "adamax_test", srcs = ["python/training/adamax_test.py"], srcs_version = "PY2AND3", - tags = [ - "no_oss", # b/73507407 - "notsan", # b/31055119 - ], deps = [ ":opt_py", "//tensorflow/python:array_ops", -- GitLab From 3a9d5e51bbb7f205a74cbfe5e6bae953d4fc2149 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Wed, 28 Mar 2018 10:28:21 +0800 Subject: [PATCH 044/791] CLN: add comment for variable --- tensorflow/contrib/opt/python/training/adamax.py | 2 +- tensorflow/core/kernels/training_ops.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/opt/python/training/adamax.py b/tensorflow/contrib/opt/python/training/adamax.py index 65918831e9..403fdaa637 100644 --- a/tensorflow/contrib/opt/python/training/adamax.py +++ b/tensorflow/contrib/opt/python/training/adamax.py @@ -48,7 +48,7 @@ class AdaMaxOptimizer(adam.AdamOptimizer): ``` The update rule for `variable` with gradient `g` uses an optimization - described at the end of section7.1 of the paper: + described at the end of section 7.1 of the paper: ``` t <- t + 1 diff --git a/tensorflow/core/kernels/training_ops.cc b/tensorflow/core/kernels/training_ops.cc index 0387e3011e..45c600fd40 100644 --- a/tensorflow/core/kernels/training_ops.cc +++ b/tensorflow/core/kernels/training_ops.cc @@ -343,9 +343,9 @@ struct ApplyAdaMaxNonCuda { LOG(WARNING) << "AdaMax doesn't support use_nesterov yet, ignore it."; } m.device(d) += (grad - m) * (T(1) - beta1()); - // v == u + // Here v is u in section 7.1 v.device(d) = (beta2() * v).cwiseMax(grad.abs()); - // var == θ + // var is θ in section 7.1 var.device(d) -= lr() / (T(1) - beta1_power()) * (m / (v + epsilon())); } }; -- GitLab From c15dbc39505de93770fd89cab4f4ae9a2a72b4e1 Mon Sep 17 00:00:00 2001 From: Wenhao Hu Date: Thu, 29 Mar 2018 02:33:24 +0900 Subject: [PATCH 045/791] fix test --- tensorflow/python/kernel_tests/norm_op_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/kernel_tests/norm_op_test.py b/tensorflow/python/kernel_tests/norm_op_test.py index d6625b69ef..0e7d4fd9b9 100644 --- a/tensorflow/python/kernel_tests/norm_op_test.py +++ b/tensorflow/python/kernel_tests/norm_op_test.py @@ -37,17 +37,17 @@ class NormOpTest(test_lib.TestCase): def testBadOrder(self): matrix = [[0., 1.], [2., 3.]] - for ord_ in "foo", -7, -1.1, 0: + for ord_ in "fro", -7, -1.1, 0: with self.assertRaisesRegexp(ValueError, "'ord' must be a supported vector norm"): - linalg_ops.norm(matrix, ord="fro") + linalg_ops.norm(matrix, ord=ord_) - for ord_ in "foo", -7, -1.1, 0: + for ord_ in "fro", -7, -1.1, 0: with self.assertRaisesRegexp(ValueError, "'ord' must be a supported vector norm"): linalg_ops.norm(matrix, ord=ord_, axis=-1) - for ord_ in 1.1, 2: + for ord_ in "foo", -7, -1.1, 1.1: with self.assertRaisesRegexp(ValueError, "'ord' must be a supported matrix norm"): linalg_ops.norm(matrix, ord=ord_, axis=[-2, -1]) -- GitLab From ab4efde7162445f20c73bdd3419811ab9c324a24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Thu, 29 Mar 2018 06:48:19 +0800 Subject: [PATCH 046/791] DOC: explain difference between adamax and adam --- tensorflow/contrib/opt/python/training/adamax.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/opt/python/training/adamax.py b/tensorflow/contrib/opt/python/training/adamax.py index 403fdaa637..ea08a0931b 100644 --- a/tensorflow/contrib/opt/python/training/adamax.py +++ b/tensorflow/contrib/opt/python/training/adamax.py @@ -31,7 +31,8 @@ from tensorflow.python.training import training_ops class AdaMaxOptimizer(adam.AdamOptimizer): """Optimizer that implements the AdaMax algorithm. - See [Kingma et al., 2014](http://arxiv.org/abs/1412.6980) + Adamax is sometimes superior to adam, specially in models with embeddings, + see [Kingma et al., 2014](http://arxiv.org/abs/1412.6980) ([pdf](http://arxiv.org/pdf/1412.6980.pdf)). """ -- GitLab From ab3b1705bc2c546eb3607876fcdcc45902552346 Mon Sep 17 00:00:00 2001 From: Wenhao Hu Date: Sat, 31 Mar 2018 00:36:25 +0900 Subject: [PATCH 047/791] cast svd output to float32 and use keepdims in test cases --- tensorflow/python/kernel_tests/norm_op_test.py | 4 ++-- tensorflow/python/ops/linalg_ops.py | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/kernel_tests/norm_op_test.py b/tensorflow/python/kernel_tests/norm_op_test.py index 0e7d4fd9b9..dde28007d4 100644 --- a/tensorflow/python/kernel_tests/norm_op_test.py +++ b/tensorflow/python/kernel_tests/norm_op_test.py @@ -69,12 +69,12 @@ def _GetNormOpTest(dtype_, shape_, ord_, axis_, keep_dims_, use_static_shape_): if use_static_shape_: tf_matrix = constant_op.constant(matrix) tf_norm = linalg_ops.norm( - tf_matrix, ord=ord_, axis=axis_, keep_dims=keep_dims_) + tf_matrix, ord=ord_, axis=axis_, keepdims=keep_dims_) tf_norm_val = sess.run(tf_norm) else: tf_matrix = array_ops.placeholder(dtype_) tf_norm = linalg_ops.norm( - tf_matrix, ord=ord_, axis=axis_, keep_dims=keep_dims_) + tf_matrix, ord=ord_, axis=axis_, keepdims=keep_dims_) tf_norm_val = sess.run(tf_norm, feed_dict={tf_matrix: matrix}) self.assertAllClose(np_norm, tf_norm_val) diff --git a/tensorflow/python/ops/linalg_ops.py b/tensorflow/python/ops/linalg_ops.py index 86be1e7752..bbc39f58db 100644 --- a/tensorflow/python/ops/linalg_ops.py +++ b/tensorflow/python/ops/linalg_ops.py @@ -548,7 +548,8 @@ def norm(tensor, ops.convert_to_tensor(axis)) axes = math_ops.range(rank) perm_before = array_ops.concat( - [array_ops.setdiff1d(axes, positive_axis)[0], positive_axis], axis=0) + [array_ops.setdiff1d(axes, positive_axis)[0], positive_axis], + axis=0) perm_after = functional_ops.map_fn( lambda i: math_ops.cast( array_ops.squeeze( @@ -557,7 +558,9 @@ def norm(tensor, permed = array_ops.transpose(tensor, perm=perm_before) matrix_2_norm = array_ops.expand_dims( math_ops.reduce_max( - gen_linalg_ops.svd(permed, compute_uv=False)[0], + math_ops.cast( + gen_linalg_ops.svd(permed, compute_uv=False)[0], + dtype=dtypes.float32), axis=-1, keepdims=True), axis=-1) -- GitLab From 6b1d9e788305c41cf436a1873c59df8d0df87d44 Mon Sep 17 00:00:00 2001 From: Wenhao Hu Date: Sat, 31 Mar 2018 01:27:05 +0900 Subject: [PATCH 048/791] use abs instead of cast --- tensorflow/python/ops/linalg_ops.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tensorflow/python/ops/linalg_ops.py b/tensorflow/python/ops/linalg_ops.py index bbc39f58db..b306042aff 100644 --- a/tensorflow/python/ops/linalg_ops.py +++ b/tensorflow/python/ops/linalg_ops.py @@ -558,9 +558,7 @@ def norm(tensor, permed = array_ops.transpose(tensor, perm=perm_before) matrix_2_norm = array_ops.expand_dims( math_ops.reduce_max( - math_ops.cast( - gen_linalg_ops.svd(permed, compute_uv=False)[0], - dtype=dtypes.float32), + math_ops.abs(gen_linalg_ops.svd(permed, compute_uv=False)[0]), axis=-1, keepdims=True), axis=-1) -- GitLab From 3bf08422a2cdd732e9b00debe3d217d04473902d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Sun, 1 Apr 2018 09:56:48 +0800 Subject: [PATCH 049/791] CLN: remove use_nesterov argument --- .../base_api/api_def_ApplyAdaMax.pbtxt | 6 - .../api_def_ResourceApplyAdaMax.pbtxt | 6 - tensorflow/core/kernels/training_ops.cc | 204 +++++++++++------- tensorflow/core/kernels/training_ops.h | 2 +- .../core/kernels/training_ops_gpu.cu.cc | 2 +- tensorflow/core/ops/training_ops.cc | 2 - 6 files changed, 133 insertions(+), 89 deletions(-) diff --git a/tensorflow/core/api_def/base_api/api_def_ApplyAdaMax.pbtxt b/tensorflow/core/api_def/base_api/api_def_ApplyAdaMax.pbtxt index 106c30ca83..57938b42ae 100644 --- a/tensorflow/core/api_def/base_api/api_def_ApplyAdaMax.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_ApplyAdaMax.pbtxt @@ -72,12 +72,6 @@ END If `True`, updating of the var, m, and v tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. -END - } - attr { - name: "use_nesterov" - description: <::ConstScalar beta1, typename TTypes::ConstScalar beta2, typename TTypes::ConstScalar epsilon, - typename TTypes::ConstFlat grad, bool use_nesterov) { - if (use_nesterov) { - LOG(WARNING) << "AdaMax doesn't support use_nesterov yet, ignore it."; - } + typename TTypes::ConstFlat grad) { m.device(d) += (grad - m) * (T(1) - beta1()); // Here v is u in section 7.1 v.device(d) = (beta2() * v).cwiseMax(grad.abs()); @@ -350,20 +347,6 @@ struct ApplyAdaMaxNonCuda { } }; -#ifdef TENSORFLOW_USE_SYCL -template -struct ApplyAdaMaxSYCL { - void operator()(const SYCLDevice& d, typename TTypes::Flat var, - typename TTypes::Flat m, typename TTypes::Flat v, - T beta1_power, T beta2_power, T lr, T beta1, T beta2, - T epsilon, typename TTypes::ConstFlat grad) { - m.device(d) += (grad - m) * (T(1) - beta1); - v.device(d) = (beta2 * v).cwiseMax(grad.abs()); - var.device(d) -= lr / (T(1) - beta1_power) * (m / (v + epsilon)); - } -}; -#endif // TENSORFLOW_USE_SYCL - template struct ApplyAdaMax : ApplyAdaMaxNonCuda {}; @@ -2516,12 +2499,10 @@ TF_CALL_double(REGISTER_CPU_KERNELS); #undef REGISTER_CPU_KERNELS #undef REGISTER_KERNELS -template - class Functor> -class ApplyAdamBaseOp : public OpKernel { +template +class ApplyAdamOp : public OpKernel { public: - explicit ApplyAdamBaseOp(OpKernelConstruction* ctx) : OpKernel(ctx) { + explicit ApplyAdamOp(OpKernelConstruction* ctx) : OpKernel(ctx) { OP_REQUIRES_OK(ctx, ctx->GetAttr("use_locking", &use_exclusive_lock_)); OP_REQUIRES_OK(ctx, ctx->GetAttr("use_nesterov", &use_nesterov_)); } @@ -2594,11 +2575,11 @@ class ApplyAdamBaseOp : public OpKernel { grad.shape().DebugString())); const Device& device = ctx->template eigen_device(); - auto functor = Functor(); - functor(device, var.flat(), m.flat(), v.flat(), - beta1_power.scalar(), beta2_power.scalar(), lr.scalar(), - beta1.scalar(), beta2.scalar(), epsilon.scalar(), - grad.flat(), use_nesterov_); + functor::ApplyAdam()( + device, var.flat(), m.flat(), v.flat(), + beta1_power.scalar(), beta2_power.scalar(), lr.scalar(), + beta1.scalar(), beta2.scalar(), epsilon.scalar(), + grad.flat(), use_nesterov_); MaybeForwardRefInputToRefOutput(ctx, 0, 0); } @@ -2609,11 +2590,10 @@ class ApplyAdamBaseOp : public OpKernel { }; #ifdef TENSORFLOW_USE_SYCL -template class Functor> -class ApplyAdamBaseOp : public OpKernel { +template +class ApplyAdamOp : public OpKernel { public: - explicit ApplyAdamBaseOp(OpKernelConstruction* ctx) : OpKernel(ctx) { + explicit ApplyAdamOp(OpKernelConstruction* ctx) : OpKernel(ctx) { OP_REQUIRES_OK(ctx, ctx->GetAttr("use_locking", &use_exclusive_lock_)); } @@ -2714,10 +2694,9 @@ class ApplyAdamBaseOp : public OpKernel { var.shape().DebugString(), " ", grad.shape().DebugString())); - auto functor = Functor(); - functor(device, var.flat(), m.flat(), v.flat(), - beta1_power, beta2_power, lr, beta1, beta2, - epsilon, grad.flat()); + functor::ApplyAdamSYCL()(device, var.flat(), m.flat(), v.flat(), + beta1_power, beta2_power, lr, beta1, beta2, + epsilon, grad.flat()); MaybeForwardRefInputToRefOutput(ctx, 0, 0); } @@ -2727,28 +2706,28 @@ class ApplyAdamBaseOp : public OpKernel { }; #endif // TENSORFLOW_USE_SYCL -#define REGISTER_KERNELS(D, T, F) \ +#define REGISTER_KERNELS(D, T) \ REGISTER_KERNEL_BUILDER( \ Name("ApplyAdam").Device(DEVICE_##D).TypeConstraint("T"), \ - ApplyAdamBaseOp); \ + ApplyAdamOp); \ REGISTER_KERNEL_BUILDER(Name("ResourceApplyAdam") \ .HostMemory("var") \ .HostMemory("m") \ .HostMemory("v") \ .Device(DEVICE_##D) \ .TypeConstraint("T"), \ - ApplyAdamBaseOp); -#define REGISTER_CPU_KERNELS(T) REGISTER_KERNELS(CPU, T, functor::ApplyAdam); + ApplyAdamOp); +#define REGISTER_CPU_KERNELS(T) REGISTER_KERNELS(CPU, T); + TF_CALL_half(REGISTER_CPU_KERNELS); TF_CALL_float(REGISTER_CPU_KERNELS); TF_CALL_double(REGISTER_CPU_KERNELS); -#undef REGISTER_CPU_KERNELS #ifdef TENSORFLOW_USE_SYCL -#define REGISTER_SYCL_KERNELS(T) REGISTER_KERNELS(SYCL, T, functor::ApplyAdamSYCL); +#define REGISTER_SYCL_KERNELS(T) REGISTER_KERNELS(SYCL, T); + TF_CALL_float(REGISTER_SYCL_KERNELS); TF_CALL_double(REGISTER_SYCL_KERNELS); -#undef REGISTER_SYCL_KERNELS #endif #if GOOGLE_CUDA @@ -2773,44 +2752,124 @@ DECLARE_GPU_SPEC(double); #undef DECLARE_GPU_SPEC } // namespace functor -#define REGISTER_GPU_KERNELS(T) REGISTER_KERNELS(GPU, T, functor::ApplyAdam); -REGISTER_GPU_KERNELS(Eigen::half); -REGISTER_GPU_KERNELS(float); -REGISTER_GPU_KERNELS(double); -#undef REGISTER_GPU_KERNELS +REGISTER_KERNELS(GPU, Eigen::half); +REGISTER_KERNELS(GPU, float); +REGISTER_KERNELS(GPU, double); #endif +#undef REGISTER_CPU_KERNELS #undef REGISTER_KERNELS -#define REGISTER_KERNELS(D, T, F) \ - REGISTER_KERNEL_BUILDER( \ +template +class ApplyAdaMaxOp : public OpKernel { + public: + explicit ApplyAdaMaxOp(OpKernelConstruction* ctx) : OpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("use_locking", &use_exclusive_lock_)); + } + + void Compute(OpKernelContext* ctx) override { + auto locks = MaybeLockVariableInputMutexesInOrder(ctx, use_exclusive_lock_, + {0, 1, 2}); + + Tensor var; + OP_REQUIRES_OK(ctx, GetInputTensorFromVariable( + ctx, 0, use_exclusive_lock_, false, &var)); + Tensor m; + OP_REQUIRES_OK(ctx, GetInputTensorFromVariable( + ctx, 1, use_exclusive_lock_, false, &m)); + Tensor v; + OP_REQUIRES_OK(ctx, GetInputTensorFromVariable( + ctx, 2, use_exclusive_lock_, false, &v)); + OP_REQUIRES( + ctx, var.IsInitialized(), + errors::FailedPrecondition( + "Attempting to use uninitialized variables: ", requested_input(0))); + OP_REQUIRES( + ctx, m.IsInitialized(), + errors::FailedPrecondition( + "Attempting to use uninitialized variables: ", requested_input(1))); + OP_REQUIRES( + ctx, v.IsInitialized(), + errors::FailedPrecondition( + "Attempting to use uninitialized variables: ", requested_input(2))); + + const Tensor& beta1_power = ctx->input(3); + const Tensor& beta2_power = ctx->input(4); + const Tensor& lr = ctx->input(5); + const Tensor& beta1 = ctx->input(6); + const Tensor& beta2 = ctx->input(7); + const Tensor& epsilon = ctx->input(8); + + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(beta1_power.shape()), + errors::InvalidArgument("beta1_power is not a scalar: ", + beta1_power.shape().DebugString())); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(beta2_power.shape()), + errors::InvalidArgument("beta2_power is not a scalar: ", + beta2_power.shape().DebugString())); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(lr.shape()), + errors::InvalidArgument("lr is not a scalar : ", + lr.shape().DebugString())); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(beta1.shape()), + errors::InvalidArgument("beta1 is not a scalar: ", + beta1.shape().DebugString())); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(beta2.shape()), + errors::InvalidArgument("beta2 is not a scalar: ", + beta2.shape().DebugString())); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(epsilon.shape()), + errors::InvalidArgument("epsilon is not a scalar: ", + epsilon.shape().DebugString())); + + const Tensor& grad = ctx->input(9); + OP_REQUIRES(ctx, var.shape().IsSameSize(m.shape()), + errors::InvalidArgument("var and m do not have the same shape", + var.shape().DebugString(), " ", + m.shape().DebugString())); + OP_REQUIRES(ctx, var.shape().IsSameSize(v.shape()), + errors::InvalidArgument("var and v do not have the same shape", + var.shape().DebugString(), " ", + v.shape().DebugString())); + OP_REQUIRES( + ctx, var.shape().IsSameSize(grad.shape()), + errors::InvalidArgument("var and grad do not have the same shape", + var.shape().DebugString(), " ", + grad.shape().DebugString())); + + const Device& device = ctx->template eigen_device(); + functor::ApplyAdaMax()( + device, var.flat(), m.flat(), v.flat(), + beta1_power.scalar(), beta2_power.scalar(), lr.scalar(), + beta1.scalar(), beta2.scalar(), epsilon.scalar(), + grad.flat()); + + MaybeForwardRefInputToRefOutput(ctx, 0, 0); + } + + private: + bool use_exclusive_lock_; +}; + +#define REGISTER_KERNELS(D, T) \ + REGISTER_KERNEL_BUILDER( \ Name("ApplyAdaMax").Device(DEVICE_##D).TypeConstraint("T"), \ - ApplyAdamBaseOp); \ + ApplyAdaMaxOp); \ REGISTER_KERNEL_BUILDER(Name("ResourceApplyAdaMax") \ - .HostMemory("var") \ - .HostMemory("m") \ - .HostMemory("v") \ - .Device(DEVICE_##D) \ - .TypeConstraint("T"), \ - ApplyAdamBaseOp); -#define REGISTER_CPU_KERNELS(T) REGISTER_KERNELS(CPU, T, functor::ApplyAdaMax); + .HostMemory("var") \ + .HostMemory("m") \ + .HostMemory("v") \ + .Device(DEVICE_##D) \ + .TypeConstraint("T"), \ + ApplyAdaMaxOp); +#define REGISTER_CPU_KERNELS(T) REGISTER_KERNELS(CPU, T); + TF_CALL_half(REGISTER_CPU_KERNELS); TF_CALL_float(REGISTER_CPU_KERNELS); TF_CALL_double(REGISTER_CPU_KERNELS); -#undef REGISTER_CPU_KERNELS - -#ifdef TENSORFLOW_USE_SYCL -#define REGISTER_SYCL_KERNELS(T) REGISTER_KERNELS(SYCL, T, functor::ApplyAdaMaxSYCL); -TF_CALL_float(REGISTER_SYCL_KERNELS); -TF_CALL_double(REGISTER_SYCL_KERNELS); -#undef REGISTER_SYCL_KERNELS -#endif #if GOOGLE_CUDA // Forward declarations of the functor specializations for GPU. namespace functor { #define DECLARE_GPU_SPEC(T) \ template <> \ - void ApplyAdaMax::operator()( \ + void ApplyAdaMax::operator()( \ const GPUDevice& d, typename TTypes::Flat var, \ typename TTypes::Flat m, typename TTypes::Flat v, \ typename TTypes::ConstScalar beta1_power, \ @@ -2819,7 +2878,7 @@ namespace functor { typename TTypes::ConstScalar beta1, \ typename TTypes::ConstScalar beta2, \ typename TTypes::ConstScalar epsilon, \ - typename TTypes::ConstFlat grad, bool use_nesterov); \ + typename TTypes::ConstFlat grad); \ extern template struct ApplyAdaMax; DECLARE_GPU_SPEC(Eigen::half); DECLARE_GPU_SPEC(float); @@ -2827,12 +2886,11 @@ DECLARE_GPU_SPEC(double); #undef DECLARE_GPU_SPEC } // namespace functor -#define REGISTER_GPU_KERNELS(T) REGISTER_KERNELS(GPU, T, functor::ApplyAdaMax); -REGISTER_GPU_KERNELS(Eigen::half); -REGISTER_GPU_KERNELS(float); -REGISTER_GPU_KERNELS(double); -#undef REGISTER_GPU_KERNELS +REGISTER_KERNELS(GPU, Eigen::half); +REGISTER_KERNELS(GPU, float); +REGISTER_KERNELS(GPU, double); #endif +#undef REGISTER_CPU_KERNELS #undef REGISTER_KERNELS template diff --git a/tensorflow/core/kernels/training_ops.h b/tensorflow/core/kernels/training_ops.h index 46a5290210..74acc12d50 100644 --- a/tensorflow/core/kernels/training_ops.h +++ b/tensorflow/core/kernels/training_ops.h @@ -149,7 +149,7 @@ struct ApplyAdaMax { typename TTypes::ConstScalar beta1, typename TTypes::ConstScalar beta2, typename TTypes::ConstScalar epsilon, - typename TTypes::ConstFlat grad, bool use_nesterov); + typename TTypes::ConstFlat grad); }; template diff --git a/tensorflow/core/kernels/training_ops_gpu.cu.cc b/tensorflow/core/kernels/training_ops_gpu.cu.cc index 54c06b130c..1a6fc26422 100644 --- a/tensorflow/core/kernels/training_ops_gpu.cu.cc +++ b/tensorflow/core/kernels/training_ops_gpu.cu.cc @@ -152,7 +152,7 @@ struct ApplyAdaMax { typename TTypes::ConstScalar beta1, typename TTypes::ConstScalar beta2, typename TTypes::ConstScalar epsilon, - typename TTypes::ConstFlat grad, bool use_nesterov) { + typename TTypes::ConstFlat grad) { Eigen::array::Tensor::Index, 1> bcast; bcast[0] = grad.dimension(0); Eigen::Sizes<1> single; diff --git a/tensorflow/core/ops/training_ops.cc b/tensorflow/core/ops/training_ops.cc index 6f107db3ea..99176cec55 100644 --- a/tensorflow/core/ops/training_ops.cc +++ b/tensorflow/core/ops/training_ops.cc @@ -751,7 +751,6 @@ REGISTER_OP("ApplyAdaMax") .Output("out: Ref(T)") .Attr("T: numbertype") .Attr("use_locking: bool = false") - .Attr("use_nesterov: bool = false") .SetShapeFn([](InferenceContext* c) { return ApplyAdamShapeFn(c, false /* sparse */); }); @@ -769,7 +768,6 @@ REGISTER_OP("ResourceApplyAdaMax") .Input("grad: T") .Attr("T: numbertype") .Attr("use_locking: bool = false") - .Attr("use_nesterov: bool = false") .SetShapeFn([](InferenceContext* c) { return ApplyAdamShapeFn(c, false /* sparse */); }); -- GitLab From f4850641530017a3b2b294974298ae13028b8583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Sun, 1 Apr 2018 10:21:46 +0800 Subject: [PATCH 050/791] CLN: code style --- tensorflow/core/kernels/training_ops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/kernels/training_ops.cc b/tensorflow/core/kernels/training_ops.cc index 1a8d08288b..aedca80c31 100644 --- a/tensorflow/core/kernels/training_ops.cc +++ b/tensorflow/core/kernels/training_ops.cc @@ -342,7 +342,7 @@ struct ApplyAdaMaxNonCuda { m.device(d) += (grad - m) * (T(1) - beta1()); // Here v is u in section 7.1 v.device(d) = (beta2() * v).cwiseMax(grad.abs()); - // var is θ in section 7.1 + // var is θ in section 7.1 var.device(d) -= lr() / (T(1) - beta1_power()) * (m / (v + epsilon())); } }; -- GitLab From 0d343fbb0e8c66622bc21aab39e225c6d895a78b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Sun, 1 Apr 2018 10:42:10 +0800 Subject: [PATCH 051/791] CLN: remove unused argument beta2_power --- .../contrib/opt/python/training/adamax.py | 42 ++++++++++++++++--- .../opt/python/training/adamax_test.py | 17 +++----- .../base_api/api_def_ApplyAdaMax.pbtxt | 6 --- .../api_def_ResourceApplyAdaMax.pbtxt | 6 --- tensorflow/core/kernels/training_ops.cc | 18 +++----- tensorflow/core/kernels/training_ops.h | 1 - .../core/kernels/training_ops_gpu.cu.cc | 1 - tensorflow/core/ops/training_ops.cc | 24 +++++++++-- 8 files changed, 67 insertions(+), 48 deletions(-) diff --git a/tensorflow/contrib/opt/python/training/adamax.py b/tensorflow/contrib/opt/python/training/adamax.py index ea08a0931b..ba9e79be99 100644 --- a/tensorflow/contrib/opt/python/training/adamax.py +++ b/tensorflow/contrib/opt/python/training/adamax.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.eager import context from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops @@ -85,14 +86,35 @@ class AdaMaxOptimizer(adam.AdamOptimizer): super(AdaMaxOptimizer, self).__init__(learning_rate, beta1, beta2, epsilon, use_locking, name) + def _get_beta_accumulators(self): + if context.in_graph_mode(): + graph = ops.get_default_graph() + else: + graph = None + return self._get_non_slot_variable("beta1_power", graph=graph) + + def _create_slots(self, var_list): + # Create the beta1 accumulators on the same device as the first + # variable. Sort the var_list to make sure this device is consistent across + # workers (these need to go on the same PS, otherwise some updates are + # silently ignored). + first_var = min(var_list, key=lambda x: x.name) + self._create_non_slot_variable(initial_value=self._beta1, + name="beta1_power", + colocate_with=first_var) + + # Create slots for the first and second moments. + for v in var_list: + self._zeros_slot(v, "m", self._name) + self._zeros_slot(v, "v", self._name) + def _apply_dense(self, grad, var): m = self.get_slot(var, "m") v = self.get_slot(var, "v") - beta1_power, beta2_power = self._get_beta_accumulators() + beta1_power = self._get_beta_accumulators() return training_ops.apply_ada_max( var, m, v, math_ops.cast(beta1_power, var.dtype.base_dtype), - math_ops.cast(beta2_power, var.dtype.base_dtype), math_ops.cast(self._lr_t, var.dtype.base_dtype), math_ops.cast(self._beta1_t, var.dtype.base_dtype), math_ops.cast(self._beta2_t, var.dtype.base_dtype), @@ -102,11 +124,10 @@ class AdaMaxOptimizer(adam.AdamOptimizer): def _resource_apply_dense(self, grad, var): m = self.get_slot(var, "m") v = self.get_slot(var, "v") - beta1_power, beta2_power = self._get_beta_accumulators() + beta1_power = self._get_beta_accumulators() return training_ops.resource_apply_ada_max( var.handle, m.handle, v.handle, math_ops.cast(beta1_power, grad.dtype.base_dtype), - math_ops.cast(beta2_power, grad.dtype.base_dtype), math_ops.cast(self._lr_t, grad.dtype.base_dtype), math_ops.cast(self._beta1_t, grad.dtype.base_dtype), math_ops.cast(self._beta2_t, grad.dtype.base_dtype), @@ -115,9 +136,8 @@ class AdaMaxOptimizer(adam.AdamOptimizer): def _apply_sparse_shared(self, grad, var, indices, scatter_add, scatter_update): - beta1_power, beta2_power = self._get_beta_accumulators() + beta1_power = self._get_beta_accumulators() beta1_power = math_ops.cast(beta1_power, var.dtype.base_dtype) - beta2_power = math_ops.cast(beta2_power, var.dtype.base_dtype) lr_t = math_ops.cast(self._lr_t, var.dtype.base_dtype) beta1_t = math_ops.cast(self._beta1_t, var.dtype.base_dtype) beta2_t = math_ops.cast(self._beta2_t, var.dtype.base_dtype) @@ -159,3 +179,13 @@ class AdaMaxOptimizer(adam.AdamOptimizer): return self._apply_sparse_shared( grad, var, indices, self._resource_scatter_add, self._resource_scatter_update) + + def _finish(self, update_ops, name_scope): + # Update the power accumulators. + with ops.control_dependencies(update_ops): + beta1_power = self._get_beta_accumulators() + with ops.colocate_with(beta1_power): + update_beta1 = beta1_power.assign( + beta1_power * self._beta1_t, use_locking=self._use_locking) + return control_flow_ops.group(*update_ops + [update_beta1], + name=name_scope) diff --git a/tensorflow/contrib/opt/python/training/adamax_test.py b/tensorflow/contrib/opt/python/training/adamax_test.py index e91e5cb96a..ccd08c0934 100644 --- a/tensorflow/contrib/opt/python/training/adamax_test.py +++ b/tensorflow/contrib/opt/python/training/adamax_test.py @@ -105,12 +105,11 @@ class AdaMaxOptimizerTest(test.TestCase): self.assertAllClose([1.0, 2.0, 3.0], var0.eval()) self.assertAllClose([4.0, 5.0, 6.0], var1.eval()) - beta1_power, beta2_power = opt._get_beta_accumulators() + beta1_power = opt._get_beta_accumulators() # Run 3 steps of AdaMax for t in range(1, 4): self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) - self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval()) update.run() var0_np, m0, v0 = adamax_sparse_update_numpy( @@ -195,11 +194,9 @@ class AdaMaxOptimizerTest(test.TestCase): opt = adamax.AdaMaxOptimizer() update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) opt_variables = opt.variables() - beta1_power, beta2_power = opt._get_beta_accumulators() + beta1_power = opt._get_beta_accumulators() self.assertTrue(beta1_power is not None) - self.assertTrue(beta2_power is not None) self.assertIn(beta1_power, opt_variables) - self.assertIn(beta2_power, opt_variables) with ops.Graph().as_default(): # Shouldn't return non-slot variables from other graphs. @@ -211,7 +208,7 @@ class AdaMaxOptimizerTest(test.TestCase): self.assertAllClose([1.0, 2.0], self.evaluate(var0)) self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - beta1_power, beta2_power = opt._get_beta_accumulators() + beta1_power = opt._get_beta_accumulators() # Run 3 steps of AdaMax for t in range(1, 4): @@ -222,8 +219,6 @@ class AdaMaxOptimizerTest(test.TestCase): self.assertAllCloseAccordingToType(0.9**(t + 1), self.evaluate(beta1_power)) - self.assertAllCloseAccordingToType(0.999**(t + 1), - self.evaluate(beta2_power)) var0_np, m0, v0 = adamax_update_numpy(var0_np, grads0_np, t, m0, v0) var1_np, m1, v1 = adamax_update_numpy(var1_np, grads1_np, t, m1, v1) @@ -265,12 +260,11 @@ class AdaMaxOptimizerTest(test.TestCase): self.assertAllClose([1.0, 2.0], var0.eval()) self.assertAllClose([3.0, 4.0], var1.eval()) - beta1_power, beta2_power = opt._get_beta_accumulators() + beta1_power = opt._get_beta_accumulators() # Run 3 steps of AdaMax for t in range(1, 4): self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) - self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval()) update.run() var0_np, m0, v0 = adamax_update_numpy(var0_np, grads0_np, t, m0, v0) @@ -299,7 +293,7 @@ class AdaMaxOptimizerTest(test.TestCase): update2 = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() - beta1_power, beta2_power = opt._get_beta_accumulators() + beta1_power = opt._get_beta_accumulators() # Fetch params to validate initial values self.assertAllClose([1.0, 2.0], var0.eval()) @@ -308,7 +302,6 @@ class AdaMaxOptimizerTest(test.TestCase): # Run 3 steps of intertwined AdaMax1 and AdaMax2. for t in range(1, 4): self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) - self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval()) if t % 2 == 0: update1.run() else: diff --git a/tensorflow/core/api_def/base_api/api_def_ApplyAdaMax.pbtxt b/tensorflow/core/api_def/base_api/api_def_ApplyAdaMax.pbtxt index 57938b42ae..5e705c009c 100644 --- a/tensorflow/core/api_def/base_api/api_def_ApplyAdaMax.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_ApplyAdaMax.pbtxt @@ -22,12 +22,6 @@ END name: "beta1_power" description: <::Flat var, typename TTypes::Flat m, typename TTypes::Flat v, typename TTypes::ConstScalar beta1_power, - typename TTypes::ConstScalar beta2_power, typename TTypes::ConstScalar lr, typename TTypes::ConstScalar beta1, typename TTypes::ConstScalar beta2, @@ -2793,18 +2792,14 @@ class ApplyAdaMaxOp : public OpKernel { "Attempting to use uninitialized variables: ", requested_input(2))); const Tensor& beta1_power = ctx->input(3); - const Tensor& beta2_power = ctx->input(4); - const Tensor& lr = ctx->input(5); - const Tensor& beta1 = ctx->input(6); - const Tensor& beta2 = ctx->input(7); - const Tensor& epsilon = ctx->input(8); + const Tensor& lr = ctx->input(4); + const Tensor& beta1 = ctx->input(5); + const Tensor& beta2 = ctx->input(6); + const Tensor& epsilon = ctx->input(7); OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(beta1_power.shape()), errors::InvalidArgument("beta1_power is not a scalar: ", beta1_power.shape().DebugString())); - OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(beta2_power.shape()), - errors::InvalidArgument("beta2_power is not a scalar: ", - beta2_power.shape().DebugString())); OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(lr.shape()), errors::InvalidArgument("lr is not a scalar : ", lr.shape().DebugString())); @@ -2818,7 +2813,7 @@ class ApplyAdaMaxOp : public OpKernel { errors::InvalidArgument("epsilon is not a scalar: ", epsilon.shape().DebugString())); - const Tensor& grad = ctx->input(9); + const Tensor& grad = ctx->input(8); OP_REQUIRES(ctx, var.shape().IsSameSize(m.shape()), errors::InvalidArgument("var and m do not have the same shape", var.shape().DebugString(), " ", @@ -2836,7 +2831,7 @@ class ApplyAdaMaxOp : public OpKernel { const Device& device = ctx->template eigen_device(); functor::ApplyAdaMax()( device, var.flat(), m.flat(), v.flat(), - beta1_power.scalar(), beta2_power.scalar(), lr.scalar(), + beta1_power.scalar(), lr.scalar(), beta1.scalar(), beta2.scalar(), epsilon.scalar(), grad.flat()); @@ -2873,7 +2868,6 @@ namespace functor { const GPUDevice& d, typename TTypes::Flat var, \ typename TTypes::Flat m, typename TTypes::Flat v, \ typename TTypes::ConstScalar beta1_power, \ - typename TTypes::ConstScalar beta2_power, \ typename TTypes::ConstScalar lr, \ typename TTypes::ConstScalar beta1, \ typename TTypes::ConstScalar beta2, \ diff --git a/tensorflow/core/kernels/training_ops.h b/tensorflow/core/kernels/training_ops.h index 74acc12d50..f536a61eb0 100644 --- a/tensorflow/core/kernels/training_ops.h +++ b/tensorflow/core/kernels/training_ops.h @@ -144,7 +144,6 @@ struct ApplyAdaMax { void operator()(const Device& d, typename TTypes::Flat var, typename TTypes::Flat m, typename TTypes::Flat v, typename TTypes::ConstScalar beta1_power, - typename TTypes::ConstScalar beta2_power, typename TTypes::ConstScalar lr, typename TTypes::ConstScalar beta1, typename TTypes::ConstScalar beta2, diff --git a/tensorflow/core/kernels/training_ops_gpu.cu.cc b/tensorflow/core/kernels/training_ops_gpu.cu.cc index 1a6fc26422..2aa17f2a0f 100644 --- a/tensorflow/core/kernels/training_ops_gpu.cu.cc +++ b/tensorflow/core/kernels/training_ops_gpu.cu.cc @@ -147,7 +147,6 @@ struct ApplyAdaMax { void operator()(const GPUDevice& d, typename TTypes::Flat var, typename TTypes::Flat m, typename TTypes::Flat v, typename TTypes::ConstScalar beta1_power, - typename TTypes::ConstScalar beta2_power, typename TTypes::ConstScalar lr, typename TTypes::ConstScalar beta1, typename TTypes::ConstScalar beta2, diff --git a/tensorflow/core/ops/training_ops.cc b/tensorflow/core/ops/training_ops.cc index 99176cec55..dc7b588898 100644 --- a/tensorflow/core/ops/training_ops.cc +++ b/tensorflow/core/ops/training_ops.cc @@ -737,12 +737,29 @@ REGISTER_OP("ResourceApplyAdam") return ApplyAdamShapeFn(c, false /* sparse */); }); +static Status ApplyAdaMaxShapeFn(InferenceContext* c, bool sparse) { + ShapeHandle unused; + ShapeHandle s = ShapeOrHandleShape(c, 0); // var + TF_RETURN_IF_ERROR(c->Merge(s, ShapeOrHandleShape(c, 1), &s)); // m + TF_RETURN_IF_ERROR(c->Merge(s, ShapeOrHandleShape(c, 2), &s)); // v + TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &unused)); // beta1_power + TF_RETURN_IF_ERROR(c->WithRank(c->input(4), 0, &unused)); // lr + TF_RETURN_IF_ERROR(c->WithRank(c->input(5), 0, &unused)); // beta1 + TF_RETURN_IF_ERROR(c->WithRank(c->input(6), 0, &unused)); // beta2 + TF_RETURN_IF_ERROR(c->WithRank(c->input(7), 0, &unused)); // epsilon + TF_RETURN_IF_ERROR( + HandleGradAndIndicesInputs(c, sparse, 8 /* grad_idx */, &s)); + if (c->num_outputs() > 0) { + c->set_output(0, s); + } + return Status::OK(); +} + REGISTER_OP("ApplyAdaMax") .Input("var: Ref(T)") .Input("m: Ref(T)") .Input("v: Ref(T)") .Input("beta1_power: T") - .Input("beta2_power: T") .Input("lr: T") .Input("beta1: T") .Input("beta2: T") @@ -752,7 +769,7 @@ REGISTER_OP("ApplyAdaMax") .Attr("T: numbertype") .Attr("use_locking: bool = false") .SetShapeFn([](InferenceContext* c) { - return ApplyAdamShapeFn(c, false /* sparse */); + return ApplyAdaMaxShapeFn(c, false /* sparse */); }); REGISTER_OP("ResourceApplyAdaMax") @@ -760,7 +777,6 @@ REGISTER_OP("ResourceApplyAdaMax") .Input("m: resource") .Input("v: resource") .Input("beta1_power: T") - .Input("beta2_power: T") .Input("lr: T") .Input("beta1: T") .Input("beta2: T") @@ -769,7 +785,7 @@ REGISTER_OP("ResourceApplyAdaMax") .Attr("T: numbertype") .Attr("use_locking: bool = false") .SetShapeFn([](InferenceContext* c) { - return ApplyAdamShapeFn(c, false /* sparse */); + return ApplyAdaMaxShapeFn(c, false /* sparse */); }); static Status ApplyRMSPropShapeFn(InferenceContext* c, bool sparse) { -- GitLab From 5ca9fedc6b3f9619a3bcf7a5a4a523668055f57d Mon Sep 17 00:00:00 2001 From: imsheridan Date: Mon, 2 Apr 2018 13:02:01 +0800 Subject: [PATCH 052/791] Fix adam optimizer related math equation rendering format --- .../opt/python/training/lazy_adam_optimizer.py | 6 +++--- tensorflow/contrib/optimizer_v2/adam.py | 16 ++++++++-------- .../api_def/base_api/api_def_ApplyAdam.pbtxt | 8 ++++---- .../base_api/api_def_ResourceApplyAdam.pbtxt | 8 ++++---- tensorflow/python/training/adam.py | 16 ++++++++-------- 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/tensorflow/contrib/opt/python/training/lazy_adam_optimizer.py b/tensorflow/contrib/opt/python/training/lazy_adam_optimizer.py index aeca900bc8..72117c1e81 100644 --- a/tensorflow/contrib/opt/python/training/lazy_adam_optimizer.py +++ b/tensorflow/contrib/opt/python/training/lazy_adam_optimizer.py @@ -56,21 +56,21 @@ class LazyAdamOptimizer(adam.AdamOptimizer): epsilon_t = math_ops.cast(self._epsilon_t, var.dtype.base_dtype) lr = (lr_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power)) - # m := beta1 * m + (1 - beta1) * g_t + # \\(m := beta1 * m + (1 - beta1) * g_t\\) m = self.get_slot(var, "m") m_t = state_ops.scatter_update(m, grad.indices, beta1_t * array_ops.gather(m, grad.indices) + (1 - beta1_t) * grad.values, use_locking=self._use_locking) - # v := beta2 * v + (1 - beta2) * (g_t * g_t) + # \\(v := beta2 * v + (1 - beta2) * (g_t * g_t)\\) v = self.get_slot(var, "v") v_t = state_ops.scatter_update(v, grad.indices, beta2_t * array_ops.gather(v, grad.indices) + (1 - beta2_t) * math_ops.square(grad.values), use_locking=self._use_locking) - # variable -= learning_rate * m_t / (epsilon_t + sqrt(v_t)) + # \\(variable -= learning_rate * m_t / (epsilon_t + sqrt(v_t))\\) m_t_slice = array_ops.gather(m_t, grad.indices) v_t_slice = array_ops.gather(v_t, grad.indices) denominator_slice = math_ops.sqrt(v_t_slice) + epsilon_t diff --git a/tensorflow/contrib/optimizer_v2/adam.py b/tensorflow/contrib/optimizer_v2/adam.py index 42b7f92a76..e863ca1244 100644 --- a/tensorflow/contrib/optimizer_v2/adam.py +++ b/tensorflow/contrib/optimizer_v2/adam.py @@ -41,21 +41,21 @@ class AdamOptimizer(optimizer_v2.OptimizerV2): Initialization: ``` - m_0 <- 0 (Initialize initial 1st moment vector) - v_0 <- 0 (Initialize initial 2nd moment vector) - t <- 0 (Initialize timestep) + \\(m_0 <- 0\\) (Initialize initial 1st moment vector) + \\(v_0 <- 0\\) (Initialize initial 2nd moment vector) + \\(t <- 0\\) (Initialize timestep) ``` The update rule for `variable` with gradient `g` uses an optimization described at the end of section2 of the paper: ``` - t <- t + 1 - lr_t <- learning_rate * sqrt(1 - beta2^t) / (1 - beta1^t) + $$t <- t + 1$$ + $$lr_t <- learning_rate * sqrt(1 - beta2^t) / (1 - beta1^t)$$ - m_t <- beta1 * m_{t-1} + (1 - beta1) * g - v_t <- beta2 * v_{t-1} + (1 - beta2) * g * g - variable <- variable - lr_t * m_t / (sqrt(v_t) + epsilon) + $$m_t <- beta1 * m_{t-1} + (1 - beta1) * g$$ + $$v_t <- beta2 * v_{t-1} + (1 - beta2) * g * g$$ + $$variable <- variable - lr_t * m_t / (sqrt(v_t) + epsilon)$$ ``` The default value of 1e-8 for epsilon might not be a good default in diff --git a/tensorflow/core/api_def/base_api/api_def_ApplyAdam.pbtxt b/tensorflow/core/api_def/base_api/api_def_ApplyAdam.pbtxt index c2858a1bfb..9bffaa79f5 100644 --- a/tensorflow/core/api_def/base_api/api_def_ApplyAdam.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_ApplyAdam.pbtxt @@ -82,9 +82,9 @@ END } summary: "Update \'*var\' according to the Adam algorithm." description: < Date: Mon, 2 Apr 2018 13:11:26 +0800 Subject: [PATCH 053/791] Fix minor typo --- tensorflow/contrib/optimizer_v2/adam.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/contrib/optimizer_v2/adam.py b/tensorflow/contrib/optimizer_v2/adam.py index e863ca1244..9bc160c0b9 100644 --- a/tensorflow/contrib/optimizer_v2/adam.py +++ b/tensorflow/contrib/optimizer_v2/adam.py @@ -51,11 +51,11 @@ class AdamOptimizer(optimizer_v2.OptimizerV2): ``` $$t <- t + 1$$ - $$lr_t <- learning_rate * sqrt(1 - beta2^t) / (1 - beta1^t)$$ + $$lr_t <- \text{learning_rate} * \sqrt{(1 - beta_2^t) / (1 - beta_1^t)}$$ - $$m_t <- beta1 * m_{t-1} + (1 - beta1) * g$$ - $$v_t <- beta2 * v_{t-1} + (1 - beta2) * g * g$$ - $$variable <- variable - lr_t * m_t / (sqrt(v_t) + epsilon)$$ + $$m_t <- beta_1 * m_{t-1} + (1 - beta_1) * g$$ + $$v_t <- beta_2 * v_{t-1} + (1 - beta_2) * g * g$$ + $$variable <- variable - lr_t * m_t / (\sqrt{v_t} + \epsilon)$$ ``` The default value of 1e-8 for epsilon might not be a good default in -- GitLab From 41074cd435a5d8b3831db8333b3669877b15a2c9 Mon Sep 17 00:00:00 2001 From: imsheridan Date: Mon, 2 Apr 2018 13:14:48 +0800 Subject: [PATCH 054/791] Fix minor typo --- tensorflow/core/api_def/base_api/api_def_ApplyAdam.pbtxt | 8 ++++---- tensorflow/python/training/adam.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tensorflow/core/api_def/base_api/api_def_ApplyAdam.pbtxt b/tensorflow/core/api_def/base_api/api_def_ApplyAdam.pbtxt index 9bffaa79f5..fc2cb09471 100644 --- a/tensorflow/core/api_def/base_api/api_def_ApplyAdam.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_ApplyAdam.pbtxt @@ -82,9 +82,9 @@ END } summary: "Update \'*var\' according to the Adam algorithm." description: < Date: Tue, 3 Apr 2018 00:18:32 +0800 Subject: [PATCH 055/791] Fix minor typo --- .../api_def/base_api/api_def_ResourceApplyAdam.pbtxt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tensorflow/core/api_def/base_api/api_def_ResourceApplyAdam.pbtxt b/tensorflow/core/api_def/base_api/api_def_ResourceApplyAdam.pbtxt index 109b68e472..5c60fa3aa1 100644 --- a/tensorflow/core/api_def/base_api/api_def_ResourceApplyAdam.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_ResourceApplyAdam.pbtxt @@ -76,9 +76,8 @@ END } summary: "Update \'*var\' according to the Adam algorithm." description: < Date: Tue, 3 Apr 2018 18:09:46 +0800 Subject: [PATCH 056/791] CLN: fix wrong hanging indentation --- tensorflow/contrib/opt/python/training/adamax.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/opt/python/training/adamax.py b/tensorflow/contrib/opt/python/training/adamax.py index ba9e79be99..4692f88349 100644 --- a/tensorflow/contrib/opt/python/training/adamax.py +++ b/tensorflow/contrib/opt/python/training/adamax.py @@ -186,6 +186,6 @@ class AdaMaxOptimizer(adam.AdamOptimizer): beta1_power = self._get_beta_accumulators() with ops.colocate_with(beta1_power): update_beta1 = beta1_power.assign( - beta1_power * self._beta1_t, use_locking=self._use_locking) + beta1_power * self._beta1_t, use_locking=self._use_locking) return control_flow_ops.group(*update_ops + [update_beta1], name=name_scope) -- GitLab From c3c3fb62f34213f96a6c9bb4174240168d8b5873 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Tue, 3 Apr 2018 18:10:18 +0800 Subject: [PATCH 057/791] CLN: add deps: egaer:context --- tensorflow/contrib/opt/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/contrib/opt/BUILD b/tensorflow/contrib/opt/BUILD index aaf0012808..39a86dbd71 100644 --- a/tensorflow/contrib/opt/BUILD +++ b/tensorflow/contrib/opt/BUILD @@ -44,6 +44,7 @@ py_library( "//tensorflow/python:util", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", + "//tensorflow/python/eager:context", "//third_party/py/numpy", "@six_archive//:six", ], -- GitLab From 9e1be727f1427284df4dda77f47a686cac07d098 Mon Sep 17 00:00:00 2001 From: Wenhao Hu Date: Wed, 4 Apr 2018 01:33:08 +0900 Subject: [PATCH 058/791] add functional_ops to BUILD --- tensorflow/python/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 3cbeb34c54..8b65b3f057 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -1916,6 +1916,7 @@ py_library( ":array_ops", ":dtypes", ":framework_ops", + ":functional_ops", ":linalg_ops_gen", ":math_ops", "//third_party/py/numpy", -- GitLab From 0b9eedd684b4085ab65d60627efa8594a92a0b98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Sat, 7 Apr 2018 11:47:03 +0800 Subject: [PATCH 059/791] TST: add test case for duplicate indices --- .../kernel_tests/scatter_nd_ops_test.py | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py index 03b2f892c6..dfe9600dbb 100644 --- a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py +++ b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py @@ -366,13 +366,35 @@ class ScatterNdTest(test.TestCase): def testString(self): indices = constant_op.constant([[4], [3], [1], [7]], dtype=dtypes.int32) - updates = constant_op.constant(["four", "three", "one", "seven"], dtype=dtypes.string) + updates = constant_op.constant(["four", "three", "one", "seven"], + dtype=dtypes.string) expected = np.array(["", "one", "", "three", "four", "", "", "seven"]) scatter = self.scatter_nd(indices, updates, shape=(8,)) + with self.test_session() as sess: + result = sess.run(scatter) + self.assertAllEqual(expected, result) + # Same indice is updated twice by same value. + indices = constant_op.constant([[4], [3], [3], [7]], dtype=dtypes.int32) + updates = constant_op.constant(["a", "b", "b", "c"], + dtype=dtypes.string) + expected = np.array(["", "", "", "bb", "a", "", "", "c"]) + scatter = self.scatter_nd(indices, updates, shape=(8,)) + with self.test_session() as sess: + result = sess.run(scatter) + self.assertAllEqual(expected, result) + + # Same indice is updated twice by different value. + indices = constant_op.constant([[4], [3], [3], [7]], dtype=dtypes.int32) + updates = constant_op.constant(["a", "b", "c", "d"], + dtype=dtypes.string) + expected = [np.array(["", "", "", "bc", "a", "", "", "d"]), + np.array(["", "", "", "cb", "a", "", "", "d"])] + scatter = self.scatter_nd(indices, updates, shape=(8,)) with self.test_session() as sess: result = sess.run(scatter) - self.assertTrue(np.array_equal(result, expected)) + self.assertTrue(np.array_equal(result, expected[0]) or + np.array_equal(result, expected[1])) def testRank3ValidShape(self): indices = array_ops.zeros([2, 2, 2], dtypes.int32) -- GitLab From 9e1bbbc0fb770f077d9de295b53181e3592f1d24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Sat, 7 Apr 2018 12:07:11 +0800 Subject: [PATCH 060/791] DOC: remove the misleading 'empty tensor' --- tensorflow/core/api_def/base_api/api_def_ScatterNd.pbtxt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/api_def/base_api/api_def_ScatterNd.pbtxt b/tensorflow/core/api_def/base_api/api_def_ScatterNd.pbtxt index 4e95895f54..58753a651a 100644 --- a/tensorflow/core/api_def/base_api/api_def_ScatterNd.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_ScatterNd.pbtxt @@ -25,7 +25,7 @@ A new tensor with the given shape and updates applied according to the indices. END } - summary: "Scatter `updates` into a new empty tensor according to `indices`." + summary: "Scatter `updates` into a new tensor according to `indices`." description: < Date: Sat, 7 Apr 2018 22:42:10 +0900 Subject: [PATCH 061/791] move dependency --- tensorflow/python/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 9dad747ac0..7d40c133c4 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -1970,6 +1970,7 @@ py_library( ":array_ops", ":control_flow_ops", ":framework_for_generated_wrappers", + ":functional_ops", ":linalg_ops", ":math_ops", "//tensorflow/python/ops/linalg:linalg_impl", @@ -1984,7 +1985,6 @@ py_library( ":array_ops", ":dtypes", ":framework_ops", - ":functional_ops", ":linalg_ops_gen", ":math_ops", "//third_party/py/numpy", -- GitLab From 7c95ee3ca48f4e50818f12daf749cbe050a8643f Mon Sep 17 00:00:00 2001 From: Brett Koonce Date: Sun, 18 Mar 2018 13:41:12 -0700 Subject: [PATCH 062/791] contrib: minor spelling tweaks packages: data training tensor_forest --- .../python/kernel_tests/dataset_serialization_test_base.py | 2 +- .../data/python/kernel_tests/interleave_dataset_op_test.py | 4 ++-- tensorflow/contrib/data/python/ops/scan_ops.py | 2 +- tensorflow/contrib/tensor_forest/client/random_forest.py | 2 +- .../hybrid/core/ops/hard_routing_function_op.cc | 2 +- .../hybrid/core/ops/stochastic_hard_routing_function_op.cc | 2 +- .../hybrid/core/ops/stochastic_hard_routing_gradient_op.cc | 2 +- tensorflow/contrib/tensor_forest/kernels/tree_utils.cc | 4 ++-- tensorflow/contrib/tensor_forest/kernels/tree_utils.h | 2 +- .../tensor_forest/kernels/v4/decision-tree-resource.h | 2 +- .../tensor_forest/kernels/v4/decision_node_evaluator.h | 2 +- tensorflow/contrib/tensor_forest/ops/model_ops.cc | 2 +- tensorflow/contrib/tensor_forest/ops/stats_ops.cc | 4 ++-- tensorflow/contrib/tensor_forest/python/tensor_forest.py | 2 +- tensorflow/contrib/training/python/training/resample.py | 2 +- tensorflow/contrib/training/python/training/sampling_ops.py | 6 +++--- .../python/training/sequence_queueing_state_saver.py | 4 ++-- 17 files changed, 23 insertions(+), 23 deletions(-) diff --git a/tensorflow/contrib/data/python/kernel_tests/dataset_serialization_test_base.py b/tensorflow/contrib/data/python/kernel_tests/dataset_serialization_test_base.py index dbc35097dd..78ecce8f7d 100644 --- a/tensorflow/contrib/data/python/kernel_tests/dataset_serialization_test_base.py +++ b/tensorflow/contrib/data/python/kernel_tests/dataset_serialization_test_base.py @@ -163,7 +163,7 @@ class DatasetSerializationTestBase(test.TestCase): num_outputs, sparse_tensors=False, verify_exhausted=True): - """Verifies that restoring into an already initilized iterator works. + """Verifies that restoring into an already initialized iterator works. Args: ds_fn: See `run_core_tests`. diff --git a/tensorflow/contrib/data/python/kernel_tests/interleave_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/interleave_dataset_op_test.py index 256ad8d94d..6a88a7caf6 100644 --- a/tensorflow/contrib/data/python/kernel_tests/interleave_dataset_op_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/interleave_dataset_op_test.py @@ -338,7 +338,7 @@ class ParallelInterleaveDatasetTest(test.TestCase): def _testTwoThreadsNoContentionWithRaces(self, sloppy=False): """Tests where all the workers race in producing elements. - Note: this is in contrast with the prevous test which carefully sequences + Note: this is in contrast with the previous test which carefully sequences the execution of the map functions. Args: @@ -424,7 +424,7 @@ class ParallelInterleaveDatasetTest(test.TestCase): def _testTwoThreadsNoContentionWithRacesAndBlocking(self, sloppy=False): """Tests where all the workers race in producing elements. - Note: this is in contrast with the prevous test which carefully sequences + Note: this is in contrast with the previous test which carefully sequences the execution of the map functions. diff --git a/tensorflow/contrib/data/python/ops/scan_ops.py b/tensorflow/contrib/data/python/ops/scan_ops.py index 1c88366273..fe49ee8b19 100644 --- a/tensorflow/contrib/data/python/ops/scan_ops.py +++ b/tensorflow/contrib/data/python/ops/scan_ops.py @@ -57,7 +57,7 @@ class _ScanDataset(dataset_ops.Dataset): self._output_shapes = None self._output_types = None - # Iteratively rerun the scan function until reaching a fixed pont on + # Iteratively rerun the scan function until reaching a fixed point on # `self._state_shapes`. need_to_rerun = True while need_to_rerun: diff --git a/tensorflow/contrib/tensor_forest/client/random_forest.py b/tensorflow/contrib/tensor_forest/client/random_forest.py index 4abcc20ed3..35e8c92aba 100644 --- a/tensorflow/contrib/tensor_forest/client/random_forest.py +++ b/tensorflow/contrib/tensor_forest/client/random_forest.py @@ -399,7 +399,7 @@ def get_combined_model_fn(model_fns): training ops: tf.group them. loss: average them. predictions: concat probabilities such that predictions[*][0-C1] are the - probablities for output 1 (where C1 is the number of classes in output 1), + probabilities for output 1 (where C1 is the number of classes in output 1), predictions[*][C1-(C1+C2)] are the probabilities for output 2 (where C2 is the number of classes in output 2), etc. Also stack predictions such that predictions[i][j] is the class prediction for example i and output j. diff --git a/tensorflow/contrib/tensor_forest/hybrid/core/ops/hard_routing_function_op.cc b/tensorflow/contrib/tensor_forest/hybrid/core/ops/hard_routing_function_op.cc index cf0db788a4..06bfe871fd 100644 --- a/tensorflow/contrib/tensor_forest/hybrid/core/ops/hard_routing_function_op.cc +++ b/tensorflow/contrib/tensor_forest/hybrid/core/ops/hard_routing_function_op.cc @@ -80,7 +80,7 @@ REGISTER_OP("HardRoutingFunction") regression model that translates from node features to probabilities. - path_probility: `path_probability[i]` gives the probability of reaching each + path_probability: `path_probability[i]` gives the probability of reaching each node in `path[i]`. path: `path[i][j]` gives the jth node in the path taken by the ith data instance. diff --git a/tensorflow/contrib/tensor_forest/hybrid/core/ops/stochastic_hard_routing_function_op.cc b/tensorflow/contrib/tensor_forest/hybrid/core/ops/stochastic_hard_routing_function_op.cc index c9df09bfda..1a055756c0 100644 --- a/tensorflow/contrib/tensor_forest/hybrid/core/ops/stochastic_hard_routing_function_op.cc +++ b/tensorflow/contrib/tensor_forest/hybrid/core/ops/stochastic_hard_routing_function_op.cc @@ -85,7 +85,7 @@ REGISTER_OP("StochasticHardRoutingFunction") regression model that translates from node features to probabilities. - path_probility: `path_probability[i]` gives the probability of reaching each + path_probability: `path_probability[i]` gives the probability of reaching each node in `path[i]`. path: `path[i][j]` gives the jth node in the path taken by the ith data instance. diff --git a/tensorflow/contrib/tensor_forest/hybrid/core/ops/stochastic_hard_routing_gradient_op.cc b/tensorflow/contrib/tensor_forest/hybrid/core/ops/stochastic_hard_routing_gradient_op.cc index b0d8b832b5..7d092bbc24 100644 --- a/tensorflow/contrib/tensor_forest/hybrid/core/ops/stochastic_hard_routing_gradient_op.cc +++ b/tensorflow/contrib/tensor_forest/hybrid/core/ops/stochastic_hard_routing_gradient_op.cc @@ -81,7 +81,7 @@ REGISTER_OP("StochasticHardRoutingGradient") tree_biases: `tree_biases[i]` gives the bias of the logistic regression model that translates from node features to probabilities. - path_probility: `path_probability[i]` gives the probability of reaching each + path_probability: `path_probability[i]` gives the probability of reaching each node in `path[i]`. path: `path[i][j]` gives the jth node in the path taken by the ith data instance. diff --git a/tensorflow/contrib/tensor_forest/kernels/tree_utils.cc b/tensorflow/contrib/tensor_forest/kernels/tree_utils.cc index 44997ec5d6..cefcc96051 100644 --- a/tensorflow/contrib/tensor_forest/kernels/tree_utils.cc +++ b/tensorflow/contrib/tensor_forest/kernels/tree_utils.cc @@ -421,7 +421,7 @@ double getChebyshevEpsilon(const std::vector& mu1, const std::vector& mu2) { // Math time!! // We are trying to minimize d = |mu1 - x|^2 + |mu2 - y|^2 over the surface. - // Using Langrange multipliers, we get + // Using Lagrange multipliers, we get // partial d / partial x = -2 mu1 + 2 x = lambda_1 1 + 2 lambda_3 x // partial d / partial y = -2 mu2 + 2 y = lambda_2 1 - 2 lambda_3 y // or @@ -485,7 +485,7 @@ double getChebyshevEpsilon(const std::vector& mu1, } double sdiscrim = sqrt(discrim); - // TODO(thomaswc): Analyze whetever one of these is always closer. + // TODO(thomaswc): Analyze whatever one of these is always closer. double v1 = (-b + sdiscrim) / (2 * a); double v2 = (-b - sdiscrim) / (2 * a); double dist1 = getDistanceFromLambda3(v1, mu1, mu2); diff --git a/tensorflow/contrib/tensor_forest/kernels/tree_utils.h b/tensorflow/contrib/tensor_forest/kernels/tree_utils.h index edbac67006..03aab1b61e 100644 --- a/tensorflow/contrib/tensor_forest/kernels/tree_utils.h +++ b/tensorflow/contrib/tensor_forest/kernels/tree_utils.h @@ -123,7 +123,7 @@ bool BestSplitDominatesRegression(const Tensor& total_sums, const Tensor& split_squares, int32 accumulator); -// Performs booststrap_samples bootstrap samples of the best split's class +// Performs bootstrap_samples bootstrap samples of the best split's class // counts and the second best splits's class counts, and returns true if at // least dominate_fraction of the time, the former has a better (lower) // Gini impurity. Does not take over ownership of *rand. diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/decision-tree-resource.h b/tensorflow/contrib/tensor_forest/kernels/v4/decision-tree-resource.h index 328af28725..d3edb43733 100644 --- a/tensorflow/contrib/tensor_forest/kernels/v4/decision-tree-resource.h +++ b/tensorflow/contrib/tensor_forest/kernels/v4/decision-tree-resource.h @@ -60,7 +60,7 @@ class DecisionTreeResource : public ResourceBase { mutex* get_mutex() { return &mu_; } // Return the TreeNode for the leaf that the example ends up at according - // to decsion_tree_. Also fill in that leaf's depth if it isn't nullptr. + // to decision_tree_. Also fill in that leaf's depth if it isn't nullptr. int32 TraverseTree(const std::unique_ptr& input_data, int example, int32* depth, TreePath* path) const; diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/decision_node_evaluator.h b/tensorflow/contrib/tensor_forest/kernels/v4/decision_node_evaluator.h index bf2b2aaa3c..3db351c328 100644 --- a/tensorflow/contrib/tensor_forest/kernels/v4/decision_node_evaluator.h +++ b/tensorflow/contrib/tensor_forest/kernels/v4/decision_node_evaluator.h @@ -60,7 +60,7 @@ class InequalityDecisionNodeEvaluator : public BinaryDecisionNodeEvaluator { bool include_equals_; }; -// Evalutor for splits with multiple weighted features. +// Evaluator for splits with multiple weighted features. class ObliqueInequalityDecisionNodeEvaluator : public BinaryDecisionNodeEvaluator { public: diff --git a/tensorflow/contrib/tensor_forest/ops/model_ops.cc b/tensorflow/contrib/tensor_forest/ops/model_ops.cc index 3099cccdf8..98124d519c 100644 --- a/tensorflow/contrib/tensor_forest/ops/model_ops.cc +++ b/tensorflow/contrib/tensor_forest/ops/model_ops.cc @@ -165,7 +165,7 @@ tree_handle: The handle to the tree. leaf_ids: `leaf_ids[i]` is the leaf id for input i. input_labels: The training batch's labels as a 1 or 2-d tensor. 'input_labels[i][j]' gives the j-th label/target for the i-th input. -input_weights: The training batch's eample weights as a 1-d tensor. +input_weights: The training batch's weights as a 1-d tensor. 'input_weights[i]' gives the weight for the i-th input. )doc"); diff --git a/tensorflow/contrib/tensor_forest/ops/stats_ops.cc b/tensorflow/contrib/tensor_forest/ops/stats_ops.cc index e8b5c5d8a6..be0a11546d 100644 --- a/tensorflow/contrib/tensor_forest/ops/stats_ops.cc +++ b/tensorflow/contrib/tensor_forest/ops/stats_ops.cc @@ -83,7 +83,7 @@ Grows the tree for finished nodes and allocates waiting nodes. params: A serialized TensorForestParams proto. tree_handle: The handle to the tree. stats_handle: The handle to the stats. -finshed_nodes: A 1-d Tensor of finished node ids from ProcessInput. +finished_nodes: A 1-d Tensor of finished node ids from ProcessInput. )doc"); REGISTER_OP("ProcessInputV4") @@ -119,7 +119,7 @@ sparse_input_values: The values tensor from the SparseTensor input. sparse_input_shape: The shape tensor from the SparseTensor input. input_labels: The training batch's labels as a 1 or 2-d tensor. 'input_labels[i][j]' gives the j-th label/target for the i-th input. -input_weights: The training batch's eample weights as a 1-d tensor. +input_weights: The training batch's weights as a 1-d tensor. 'input_weights[i]' gives the weight for the i-th input. finished_nodes: A 1-d tensor of node ids that have finished and are ready to grow. diff --git a/tensorflow/contrib/tensor_forest/python/tensor_forest.py b/tensorflow/contrib/tensor_forest/python/tensor_forest.py index 3650b5d52f..b9bcbb170b 100644 --- a/tensorflow/contrib/tensor_forest/python/tensor_forest.py +++ b/tensorflow/contrib/tensor_forest/python/tensor_forest.py @@ -212,7 +212,7 @@ class ForestHParams(object): self.regression = getattr(self, 'regression', False) # Num_outputs is the actual number of outputs (a single prediction for - # classification, a N-dimenensional point for regression). + # classification, a N-dimensional point for regression). self.num_outputs = self.num_classes if self.regression else 1 # Add an extra column to classes for storing counts, which is needed for diff --git a/tensorflow/contrib/training/python/training/resample.py b/tensorflow/contrib/training/python/training/resample.py index b16159bc16..7b8332b1d6 100644 --- a/tensorflow/contrib/training/python/training/resample.py +++ b/tensorflow/contrib/training/python/training/resample.py @@ -77,7 +77,7 @@ def resample_at_rate(inputs, rates, scope=None, seed=None, back_prop=False): Args: inputs: A list of tensors, each of which has a shape of `[batch_size, ...]` - rates: A tensor of shape `[batch_size]` contiaining the resampling rates + rates: A tensor of shape `[batch_size]` containing the resampling rates for each input. scope: Scope for the op. seed: Random seed to use. diff --git a/tensorflow/contrib/training/python/training/sampling_ops.py b/tensorflow/contrib/training/python/training/sampling_ops.py index ba888f87dc..7140f2a46d 100644 --- a/tensorflow/contrib/training/python/training/sampling_ops.py +++ b/tensorflow/contrib/training/python/training/sampling_ops.py @@ -123,7 +123,7 @@ def rejection_sample(tensors, batch_size=batch_size, num_threads=queue_threads) - # Queues return a single tensor if the list of enqued tensors is one. Since + # Queues return a single tensor if the list of enqueued tensors is one. Since # we want the type to always be the same, always return a list. if isinstance(minibatch, ops.Tensor): minibatch = [minibatch] @@ -312,7 +312,7 @@ def _verify_input(tensor_list, labels, probs_list): """Verify that batched inputs are well-formed.""" checked_probs_list = [] for probs in probs_list: - # Since number of classes shouldn't change at runtime, probalities shape + # Since number of classes shouldn't change at runtime, probabilities shape # should be fully defined. probs.get_shape().assert_is_fully_defined() @@ -407,7 +407,7 @@ def _calculate_acceptance_probabilities(init_probs, target_probs): ``` - A solution for a_i in terms of the other variabes is the following: + A solution for a_i in terms of the other variables is the following: ```a_i = (t_i / p_i) / max_i[t_i / p_i]``` """ # Make list of t_i / p_i. diff --git a/tensorflow/contrib/training/python/training/sequence_queueing_state_saver.py b/tensorflow/contrib/training/python/training/sequence_queueing_state_saver.py index 99d486b183..39d75a0806 100644 --- a/tensorflow/contrib/training/python/training/sequence_queueing_state_saver.py +++ b/tensorflow/contrib/training/python/training/sequence_queueing_state_saver.py @@ -876,7 +876,7 @@ class SequenceQueueingStateSaver(object): ]): self._length = array_ops.identity(self._length) - # Only create barrier; enqueu and dequeue operations happen when you + # Only create barrier; enqueue and dequeue operations happen when you # access prefetch_op and next_batch. self._create_barrier() self._scope = scope @@ -1637,7 +1637,7 @@ def _move_sparse_tensor_out_context(input_context, input_sequences, num_unroll): For `key, value` pairs in `input_context` with `SparseTensor` `value` removes them from `input_context` and transforms the `value` into a sequence and - then adding `key`, transformed `value` into `input_seuqences`. + then adding `key`, transformed `value` into `input_sequences`. The transformation is done by adding a new first dimension of `value_length` equal to that of the other values in input_sequences` and tiling the `value` every `num_unroll` steps. -- GitLab From bbc1ce5b1397041d12d90502c08997de03f798b3 Mon Sep 17 00:00:00 2001 From: DosLin Date: Mon, 9 Apr 2018 13:14:56 +0800 Subject: [PATCH 063/791] Docs: Fix 'Unable to find source java class' --- tensorflow/docs_src/mobile/android_build.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/docs_src/mobile/android_build.md b/tensorflow/docs_src/mobile/android_build.md index 08a5fbe41c..0cd0a98be4 100644 --- a/tensorflow/docs_src/mobile/android_build.md +++ b/tensorflow/docs_src/mobile/android_build.md @@ -51,7 +51,8 @@ If you haven't already, do the following two things: // set to 'bazel', 'cmake', 'makefile', 'none' def nativeBuildSystem = 'none' -4. Click the Run button (the green arrow) or use **Run -> Run 'android'** from the top menu. +4. Running "Build -> Rebuild Project" from Android Studio menu and click the + Run button (the green arrow) or use **Run -> Run 'android'** from the top menu. If it asks you to use Instant Run, click **Proceed Without Instant Run**. -- GitLab From 0a0312473c2c5179d05bb716586e796daa3a0252 Mon Sep 17 00:00:00 2001 From: Benoit Steiner Date: Mon, 9 Apr 2018 17:30:27 -0700 Subject: [PATCH 064/791] Don't fail when running shape inference on a graph that contains functions. PiperOrigin-RevId: 192215493 --- tensorflow/core/graph/graph_constructor.cc | 7 +---- .../core/grappler/costs/graph_properties.cc | 4 +-- .../grappler/costs/graph_properties_test.cc | 29 +++++++++---------- 3 files changed, 16 insertions(+), 24 deletions(-) diff --git a/tensorflow/core/graph/graph_constructor.cc b/tensorflow/core/graph/graph_constructor.cc index f15e2ce9fa..250992fb7a 100644 --- a/tensorflow/core/graph/graph_constructor.cc +++ b/tensorflow/core/graph/graph_constructor.cc @@ -1019,12 +1019,7 @@ Status GraphConstructor::Convert() { } } - // Function shape inference is supported on an opt-in basis per - // ShapeRefiner. - if (refiner_->function_shape_inference_supported() || - g_->flib_def().Find(node_def->name()) == nullptr) { - TF_RETURN_IF_ERROR(ValidateShape(node)); - } + TF_RETURN_IF_ERROR(ValidateShape(node)); // Update pending_count_ for outputs. UpdatePendingCountAndReady(outputs_, o, &pending_count_, &ready_); diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc index 8fe154dbf3..9fa2b7a259 100644 --- a/tensorflow/core/grappler/costs/graph_properties.cc +++ b/tensorflow/core/grappler/costs/graph_properties.cc @@ -920,9 +920,9 @@ Status GraphProperties::UpdateResource( } Status GraphProperties::InferStatically(bool assume_valid_feeds) { - Graph graph(OpRegistry::Global()); - FunctionLibraryDefinition function_library(graph.op_registry(), + FunctionLibraryDefinition function_library(OpRegistry::Global(), item_.graph.library()); + Graph graph(function_library); ShapeRefiner shape_refiner(graph.versions(), graph.op_registry()); shape_refiner.set_require_shape_inference_fns(false); shape_refiner.set_disable_constant_propagation(true); diff --git a/tensorflow/core/grappler/costs/graph_properties_test.cc b/tensorflow/core/grappler/costs/graph_properties_test.cc index db4dae96de..d3d89b59af 100644 --- a/tensorflow/core/grappler/costs/graph_properties_test.cc +++ b/tensorflow/core/grappler/costs/graph_properties_test.cc @@ -742,8 +742,6 @@ TEST_F(GraphPropertiesTest, InferRestoreOpShape_WithTwoNodesShareSameOutput) { EXPECT_EQ("float: [128,256]", PropToString(prop)); } -#if 0 -// Disabled for now since this doesnt' seem to work when functions are instantiated inside while loops. It's also unclear whether it's correct when the same function is instantiated twice. TEST_F(GraphPropertiesTest, FunctionStaticShapeInference) { // Test graph produced in python using: /* @@ -757,27 +755,26 @@ TEST_F(GraphPropertiesTest, FunctionStaticShapeInference) { z = MyAdd(x, y) z = MyAdd(x, z) */ - // Check that the shape of the second MyAdd node propagates - // correctly. + // Check that the shape inference code infers what it can. GrapplerItem item; string filename = io::JoinPath(testing::TensorFlowSrcRoot(), kTestDataPath, "simple_function.pbtxt"); TF_CHECK_OK(ReadGraphDefFromFile(filename, &item.graph)); GraphProperties properties(item); TF_CHECK_OK(properties.InferStatically(false)); - const auto props = properties.GetOutputProperties("MyAdd_55e046a8_1"); - const OpInfo::TensorProperties& prop = props[0]; - EXPECT_EQ(DT_FLOAT, prop.dtype()); - EXPECT_FALSE(prop.shape().unknown_rank()); - EXPECT_EQ(2, prop.shape().dim_size()); - EXPECT_EQ(1, prop.shape().dim(0).size()); - EXPECT_EQ(2, prop.shape().dim(1).size()); - - PartialTensorShape shape(prop.shape()); - EXPECT_TRUE(shape.IsFullyDefined()); - EXPECT_FALSE(shape.unknown_rank()); + const auto out_props = properties.GetOutputProperties("MyAdd_55e046a8"); + const OpInfo::TensorProperties& out_prop = out_props[0]; + EXPECT_EQ(DT_FLOAT, out_prop.dtype()); + EXPECT_TRUE(out_prop.shape().unknown_rank()); + + const auto in_props = properties.GetInputProperties("MyAdd_55e046a8"); + const OpInfo::TensorProperties& in_prop = in_props[0]; + EXPECT_EQ(DT_FLOAT, in_prop.dtype()); + EXPECT_FALSE(in_prop.shape().unknown_rank()); + EXPECT_EQ(2, in_prop.shape().dim_size()); + EXPECT_EQ(1, in_prop.shape().dim(0).size()); + EXPECT_EQ(2, in_prop.shape().dim(1).size()); } -#endif TEST_F(GraphPropertiesTest, SymbolicShapes) { // Build a simple graph with placeholders of unknown dimensions. These -- GitLab From fccc96cbf48537fca49b61f94147d1c8e299fea4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 9 Apr 2018 17:36:41 -0700 Subject: [PATCH 065/791] [slim] Allow passing timeout_fn to 'evaluation_loop' so that caller can have more control over what to do if checkpoints_iterator times out. PiperOrigin-RevId: 192216302 --- tensorflow/contrib/slim/python/slim/evaluation.py | 7 ++++++- .../contrib/slim/python/slim/evaluation_test.py | 11 +++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/slim/python/slim/evaluation.py b/tensorflow/contrib/slim/python/slim/evaluation.py index 3caf4e02da..5cfd5ee82e 100644 --- a/tensorflow/contrib/slim/python/slim/evaluation.py +++ b/tensorflow/contrib/slim/python/slim/evaluation.py @@ -230,6 +230,7 @@ def evaluation_loop(master, max_number_of_evaluations=None, session_config=None, timeout=None, + timeout_fn=None, hooks=None): """Runs TF-Slim's Evaluation Loop. @@ -261,6 +262,9 @@ def evaluation_loop(master, configure the `Session`. If left as `None`, the default will be used. timeout: The maximum amount of time to wait between checkpoints. If left as `None`, then the process will wait indefinitely. + timeout_fn: Optional function to call after a timeout. If the function + returns True, then it means that no new checkpoints will be generated and + the iterator will exit. The function is called with no arguments. hooks: A list of additional `SessionRunHook` objects to pass during repeated evaluations. @@ -298,4 +302,5 @@ def evaluation_loop(master, hooks=all_hooks, config=session_config, max_number_of_evaluations=max_number_of_evaluations, - timeout=timeout) + timeout=timeout, + timeout_fn=timeout_fn) diff --git a/tensorflow/contrib/slim/python/slim/evaluation_test.py b/tensorflow/contrib/slim/python/slim/evaluation_test.py index c24bd04851..94fc12ca81 100644 --- a/tensorflow/contrib/slim/python/slim/evaluation_test.py +++ b/tensorflow/contrib/slim/python/slim/evaluation_test.py @@ -177,6 +177,17 @@ class EvaluationTest(test.TestCase): # The timeout kicked in. self.assertLess(end, start + 1.1) + def testTimeoutFnOnEvaluationLoop(self): + # We require a mutable object (e.g. list but not an int) to maintain state + # across calls of a nested function. + timeout_fn_calls = [0] + def _TimeoutFn(): + timeout_fn_calls[0] += 1 + return timeout_fn_calls[0] >= 3 + # Need not do any evaluation, but should just call timeout_fn repeatedly. + evaluation.evaluation_loop('', '', '', timeout=0, timeout_fn=_TimeoutFn) + self.assertEqual(timeout_fn_calls[0], 3) + def testMonitorCheckpointsLoopTimeout(self): ret = list( evaluation_lib.checkpoints_iterator( -- GitLab From 01417f6fe7c28ec530154f63efac333d19ba7632 Mon Sep 17 00:00:00 2001 From: Skye Wanderman-Milne Date: Mon, 9 Apr 2018 17:40:46 -0700 Subject: [PATCH 066/791] Remove extra #define EIGEN_USE_THREADS. It breaks some builds with error: "EIGEN_USE_THREADS" redefined [-Werror] PiperOrigin-RevId: 192216796 --- tensorflow/core/common_runtime/eigen_thread_pool.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/tensorflow/core/common_runtime/eigen_thread_pool.h b/tensorflow/core/common_runtime/eigen_thread_pool.h index ddd627fb20..c6f13c6a11 100644 --- a/tensorflow/core/common_runtime/eigen_thread_pool.h +++ b/tensorflow/core/common_runtime/eigen_thread_pool.h @@ -16,8 +16,6 @@ limitations under the License. #ifndef TENSORFLOW_COMMON_RUNTIME_EIGEN_THREAD_POOL_H_ #define TENSORFLOW_COMMON_RUNTIME_EIGEN_THREAD_POOL_H_ -#define EIGEN_USE_THREADS - #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/lib/core/threadpool.h" -- GitLab From 0295c3c212546147e57609a791f126aaca44ba44 Mon Sep 17 00:00:00 2001 From: Younghee Kwon Date: Mon, 9 Apr 2018 17:45:13 -0700 Subject: [PATCH 067/791] boosted_trees: early stop hooks are fixed to stop at the right moment by reading tensor values in a separate session after train_op run. PiperOrigin-RevId: 192217338 --- .../python/estimator/boosted_trees_test.py | 97 +++++++------------ .../python/estimator/canned/boosted_trees.py | 33 +++---- .../estimator/canned/boosted_trees_test.py | 63 +++++------- 3 files changed, 71 insertions(+), 122 deletions(-) diff --git a/tensorflow/contrib/estimator/python/estimator/boosted_trees_test.py b/tensorflow/contrib/estimator/python/estimator/boosted_trees_test.py index e99a87f3b3..eee5910687 100644 --- a/tensorflow/contrib/estimator/python/estimator/boosted_trees_test.py +++ b/tensorflow/contrib/estimator/python/estimator/boosted_trees_test.py @@ -20,6 +20,7 @@ from __future__ import print_function import numpy as np from tensorflow.contrib.estimator.python.estimator import boosted_trees +from tensorflow.core.kernels.boosted_trees import boosted_trees_pb2 from tensorflow.python.estimator.canned import boosted_trees as canned_boosted_trees from tensorflow.python.estimator.inputs import numpy_io from tensorflow.python.feature_column import feature_column @@ -69,10 +70,18 @@ class BoostedTreesEstimatorTest(test_util.TensorFlowTestCase): for i in range(NUM_FEATURES) } - def _assert_checkpoint(self, model_dir, expected_global_step): - self.assertEqual(expected_global_step, - checkpoint_utils.load_variable(model_dir, - ops.GraphKeys.GLOBAL_STEP)) + def _assert_checkpoint(self, model_dir, global_step, finalized_trees, + attempted_layers): + reader = checkpoint_utils.load_checkpoint(model_dir) + self.assertEqual(global_step, reader.get_tensor(ops.GraphKeys.GLOBAL_STEP)) + serialized = reader.get_tensor('boosted_trees:0_serialized') + ensemble_proto = boosted_trees_pb2.TreeEnsemble() + ensemble_proto.ParseFromString(serialized) + self.assertEqual( + finalized_trees, + sum([1 for t in ensemble_proto.tree_metadata if t.is_finalized])) + self.assertEqual(attempted_layers, + ensemble_proto.growing_metadata.num_layers_attempted) def testTrainAndEvaluateEstimator(self): input_fn = _make_train_input_fn(is_classification=False) @@ -88,9 +97,10 @@ class BoostedTreesEstimatorTest(test_util.TensorFlowTestCase): num_steps = 100 # Train for a few steps, and validate final checkpoint. est.train(input_fn, steps=num_steps) - self._assert_checkpoint(est.model_dir, 11) + self._assert_checkpoint( + est.model_dir, global_step=10, finalized_trees=2, attempted_layers=10) eval_res = est.evaluate(input_fn=input_fn, steps=1) - self.assertAllClose(eval_res['average_loss'], 0.913176) + self.assertAllClose(eval_res['average_loss'], 1.008551) def testInferEstimator(self): train_input_fn = _make_train_input_fn(is_classification=False) @@ -108,31 +118,13 @@ class BoostedTreesEstimatorTest(test_util.TensorFlowTestCase): num_steps = 100 # Train for a few steps, and validate final checkpoint. est.train(train_input_fn, steps=num_steps) - self._assert_checkpoint(est.model_dir, 6) - + self._assert_checkpoint( + est.model_dir, global_step=5, finalized_trees=1, attempted_layers=5) + # Validate predictions. predictions = list(est.predict(input_fn=predict_input_fn)) - self.assertEquals(5, len(predictions)) - self.assertAllClose([0.703549], predictions[0]['predictions']) - self.assertAllClose([0.266539], predictions[1]['predictions']) - self.assertAllClose([0.256479], predictions[2]['predictions']) - self.assertAllClose([1.088732], predictions[3]['predictions']) - self.assertAllClose([1.901732], predictions[4]['predictions']) - - -class BoostedTreesClassifierTrainInMemoryTest(test_util.TensorFlowTestCase): - - def setUp(self): - self._feature_columns = { - feature_column.bucketized_column( - feature_column.numeric_column('f_%d' % i, dtype=dtypes.float32), - BUCKET_BOUNDARIES) - for i in range(NUM_FEATURES) - } - - def _assert_checkpoint(self, model_dir, expected_global_step): - self.assertEqual(expected_global_step, - checkpoint_utils.load_variable(model_dir, - ops.GraphKeys.GLOBAL_STEP)) + self.assertAllClose( + [[0.571619], [0.262821], [0.124549], [0.956801], [1.769801]], + [pred['predictions'] for pred in predictions]) def testBinaryClassifierTrainInMemoryAndEvalAndInfer(self): train_input_fn = _make_train_input_fn(is_classification=True) @@ -145,36 +137,16 @@ class BoostedTreesClassifierTrainInMemoryTest(test_util.TensorFlowTestCase): n_trees=1, max_depth=5) # It will stop after 5 steps because of the max depth and num trees. - self._assert_checkpoint(est.model_dir, 6) + self._assert_checkpoint( + est.model_dir, global_step=5, finalized_trees=1, attempted_layers=5) # Check eval. eval_res = est.evaluate(input_fn=train_input_fn, steps=1) self.assertAllClose(eval_res['accuracy'], 1.0) - - # Check predict that all labels are correct. + # Validate predictions. predictions = list(est.predict(input_fn=predict_input_fn)) - self.assertEquals(5, len(predictions)) - self.assertAllClose([0], predictions[0]['class_ids']) - self.assertAllClose([1], predictions[1]['class_ids']) - self.assertAllClose([1], predictions[2]['class_ids']) - self.assertAllClose([0], predictions[3]['class_ids']) - self.assertAllClose([0], predictions[4]['class_ids']) - - -class BoostedTreesRegressorTrainInMemoryTest(test_util.TensorFlowTestCase): - - def setUp(self): - self._feature_columns = { - feature_column.bucketized_column( - feature_column.numeric_column('f_%d' % i, dtype=dtypes.float32), - BUCKET_BOUNDARIES) - for i in range(NUM_FEATURES) - } - - def _assert_checkpoint(self, model_dir, expected_global_step): - self.assertEqual(expected_global_step, - checkpoint_utils.load_variable(model_dir, - ops.GraphKeys.GLOBAL_STEP)) + self.assertAllClose([[0], [1], [1], [0], [0]], + [pred['class_ids'] for pred in predictions]) def testRegressorTrainInMemoryAndEvalAndInfer(self): train_input_fn = _make_train_input_fn(is_classification=False) @@ -187,20 +159,17 @@ class BoostedTreesRegressorTrainInMemoryTest(test_util.TensorFlowTestCase): n_trees=1, max_depth=5) # It will stop after 5 steps because of the max depth and num trees. - self._assert_checkpoint(est.model_dir, 6) + self._assert_checkpoint( + est.model_dir, global_step=5, finalized_trees=1, attempted_layers=5) # Check eval. eval_res = est.evaluate(input_fn=train_input_fn, steps=1) - self.assertAllClose(eval_res['average_loss'], 2.2136638) - + self.assertAllClose(eval_res['average_loss'], 2.478283) # Validate predictions. predictions = list(est.predict(input_fn=predict_input_fn)) - self.assertEquals(5, len(predictions)) - self.assertAllClose([0.703549], predictions[0]['predictions']) - self.assertAllClose([0.266539], predictions[1]['predictions']) - self.assertAllClose([0.256479], predictions[2]['predictions']) - self.assertAllClose([1.088732], predictions[3]['predictions']) - self.assertAllClose([1.901732], predictions[4]['predictions']) + self.assertAllClose( + [[0.571619], [0.262821], [0.124549], [0.956801], [1.769801]], + [pred['predictions'] for pred in predictions]) if __name__ == '__main__': diff --git a/tensorflow/python/estimator/canned/boosted_trees.py b/tensorflow/python/estimator/canned/boosted_trees.py index 500ea03ea7..c5d5455b1a 100644 --- a/tensorflow/python/estimator/canned/boosted_trees.py +++ b/tensorflow/python/estimator/canned/boosted_trees.py @@ -209,8 +209,8 @@ class _CacheTrainingStatesUsingVariables(object): name='cache_insert') -class StopAtAttemptsHook(session_run_hook.SessionRunHook): - """Hook that requests stop at the number of trees.""" +class _StopAtAttemptsHook(session_run_hook.SessionRunHook): + """Hook that requests stop at the number of attempts.""" def __init__(self, num_finalized_trees_tensor, num_attempted_layers_tensor, max_trees, max_depth): @@ -224,25 +224,17 @@ class StopAtAttemptsHook(session_run_hook.SessionRunHook): [self._num_finalized_trees_tensor, self._num_attempted_layers_tensor]) def after_run(self, run_context, run_values): + # num_* tensors should be retrieved by a separate session than the training + # one, in order to read the values after growing. + # So, if it's approaching to the limit, get the actual value by additional + # session. num_finalized_trees, num_attempted_layers = run_values.results + if (num_finalized_trees >= self._max_trees - 1 or + num_attempted_layers > 2 * self._max_trees * self._max_depth - 1): + num_finalized_trees, num_attempted_layers = run_context.session.run( + [self._num_finalized_trees_tensor, self._num_attempted_layers_tensor]) if (num_finalized_trees >= self._max_trees or - 1.0 * num_attempted_layers / self._max_depth > 2 * self._max_trees): - run_context.request_stop() - - -class StopAtNumTreesHook(session_run_hook.SessionRunHook): - """Hook that requests stop at the number of trees.""" - - def __init__(self, num_trees_tensor, max_trees): - self._num_trees_tensor = num_trees_tensor - self._max_trees = max_trees - - def before_run(self, run_context): - return session_run_hook.SessionRunArgs(self._num_trees_tensor) - - def after_run(self, run_context, run_values): - num_trees = run_values.results - if num_trees > self._max_trees: + num_attempted_layers > 2 * self._max_trees * self._max_depth): run_context.request_stop() @@ -468,7 +460,8 @@ def _bt_model_fn( # Add an early stop hook. estimator_spec = estimator_spec._replace( training_hooks=estimator_spec.training_hooks + - (StopAtNumTreesHook(num_trees, tree_hparams.n_trees),)) + (_StopAtAttemptsHook(num_finalized_trees, num_attempted_layers, + tree_hparams.n_trees, tree_hparams.max_depth),)) return estimator_spec diff --git a/tensorflow/python/estimator/canned/boosted_trees_test.py b/tensorflow/python/estimator/canned/boosted_trees_test.py index 01e5cc7a5d..625745a3f9 100644 --- a/tensorflow/python/estimator/canned/boosted_trees_test.py +++ b/tensorflow/python/estimator/canned/boosted_trees_test.py @@ -69,7 +69,7 @@ def _make_train_input_fn(is_classification): return _input_fn -class BoostedTreesClassifierTest(test_util.TensorFlowTestCase): +class BoostedTreesEstimatorTest(test_util.TensorFlowTestCase): def setUp(self): self._feature_columns = { @@ -79,10 +79,18 @@ class BoostedTreesClassifierTest(test_util.TensorFlowTestCase): for i in range(NUM_FEATURES) } - def _assert_checkpoint(self, model_dir, expected_global_step): - self.assertEqual(expected_global_step, - checkpoint_utils.load_variable(model_dir, - ops.GraphKeys.GLOBAL_STEP)) + def _assert_checkpoint(self, model_dir, global_step, finalized_trees, + attempted_layers): + reader = checkpoint_utils.load_checkpoint(model_dir) + self.assertEqual(global_step, reader.get_tensor(ops.GraphKeys.GLOBAL_STEP)) + serialized = reader.get_tensor('boosted_trees:0_serialized') + ensemble_proto = boosted_trees_pb2.TreeEnsemble() + ensemble_proto.ParseFromString(serialized) + self.assertEqual( + finalized_trees, + sum([1 for t in ensemble_proto.tree_metadata if t.is_finalized])) + self.assertEqual(attempted_layers, + ensemble_proto.growing_metadata.num_layers_attempted) def testTrainAndEvaluateBinaryClassifier(self): input_fn = _make_train_input_fn(is_classification=True) @@ -97,7 +105,8 @@ class BoostedTreesClassifierTest(test_util.TensorFlowTestCase): num_steps = 100 # Train for a few steps, and validate final checkpoint. est.train(input_fn, steps=num_steps) - self._assert_checkpoint(est.model_dir, 6) + self._assert_checkpoint( + est.model_dir, global_step=5, finalized_trees=1, attempted_layers=5) eval_res = est.evaluate(input_fn=input_fn, steps=1) self.assertAllClose(eval_res['accuracy'], 1.0) @@ -118,29 +127,9 @@ class BoostedTreesClassifierTest(test_util.TensorFlowTestCase): est.train(train_input_fn, steps=num_steps) predictions = list(est.predict(input_fn=predict_input_fn)) - self.assertEquals(5, len(predictions)) # All labels are correct. - self.assertAllClose([0], predictions[0]['class_ids']) - self.assertAllClose([1], predictions[1]['class_ids']) - self.assertAllClose([1], predictions[2]['class_ids']) - self.assertAllClose([0], predictions[3]['class_ids']) - self.assertAllClose([0], predictions[4]['class_ids']) - - -class BoostedTreesRegressionTest(test_util.TensorFlowTestCase): - - def setUp(self): - self._feature_columns = { - feature_column.bucketized_column( - feature_column.numeric_column('f_%d' % i, dtype=dtypes.float32), - BUCKET_BOUNDARIES) - for i in range(NUM_FEATURES) - } - - def _assert_checkpoint(self, model_dir, expected_global_step): - self.assertEqual(expected_global_step, - checkpoint_utils.load_variable(model_dir, - ops.GraphKeys.GLOBAL_STEP)) + self.assertAllClose([[0], [1], [1], [0], [0]], + [pred['class_ids'] for pred in predictions]) def testTrainAndEvaluateRegressor(self): input_fn = _make_train_input_fn(is_classification=False) @@ -155,9 +144,10 @@ class BoostedTreesRegressionTest(test_util.TensorFlowTestCase): num_steps = 100 # Train for a few steps, and validate final checkpoint. est.train(input_fn, steps=num_steps) - self._assert_checkpoint(est.model_dir, 11) + self._assert_checkpoint( + est.model_dir, global_step=10, finalized_trees=2, attempted_layers=10) eval_res = est.evaluate(input_fn=input_fn, steps=1) - self.assertAllClose(eval_res['average_loss'], 0.913176) + self.assertAllClose(eval_res['average_loss'], 1.008551) def testInferRegressor(self): train_input_fn = _make_train_input_fn(is_classification=False) @@ -174,16 +164,13 @@ class BoostedTreesRegressionTest(test_util.TensorFlowTestCase): num_steps = 100 # Train for a few steps, and validate final checkpoint. est.train(train_input_fn, steps=num_steps) - self._assert_checkpoint(est.model_dir, 6) + self._assert_checkpoint( + est.model_dir, global_step=5, finalized_trees=1, attempted_layers=5) predictions = list(est.predict(input_fn=predict_input_fn)) - - self.assertEquals(5, len(predictions)) - self.assertAllClose([0.703549], predictions[0]['predictions']) - self.assertAllClose([0.266539], predictions[1]['predictions']) - self.assertAllClose([0.256479], predictions[2]['predictions']) - self.assertAllClose([1.088732], predictions[3]['predictions']) - self.assertAllClose([1.901732], predictions[4]['predictions']) + self.assertAllClose( + [[0.571619], [0.262821], [0.124549], [0.956801], [1.769801]], + [pred['predictions'] for pred in predictions]) class ModelFnTests(test_util.TensorFlowTestCase): -- GitLab From 77bd95ab15bdfa6a821540ce1c826c7eb9c9f0a8 Mon Sep 17 00:00:00 2001 From: Rachel Lim Date: Mon, 9 Apr 2018 18:03:57 -0700 Subject: [PATCH 068/791] [tf.data] Add parameter for select_cols in tf.decode_csv for option to parse only a subset of CSV columns; added same parameter to make_csv_dataset function in tf.contrib.data. PiperOrigin-RevId: 192219509 --- .../kernel_tests/reader_dataset_ops_test.py | 197 +++++++++++++----- tensorflow/contrib/data/python/ops/readers.py | 76 ++++++- tensorflow/core/kernels/decode_csv_op.cc | 41 +++- tensorflow/core/ops/parsing_ops.cc | 1 + .../python/kernel_tests/decode_csv_op_test.py | 90 ++++++-- tensorflow/python/ops/parsing_ops.py | 26 ++- tensorflow/tools/api/golden/tensorflow.pbtxt | 2 +- 7 files changed, 351 insertions(+), 82 deletions(-) diff --git a/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py b/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py index f3e9302409..1075302bae 100644 --- a/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py @@ -295,18 +295,20 @@ class ReadBatchFeaturesTest(test.TestCase): ).get_next() def _record(self, f, r): - example = example_pb2.Example(features=feature_pb2.Features( - feature={ - "file": - feature_pb2.Feature(int64_list=feature_pb2.Int64List( - value=[f])), - "record": - feature_pb2.Feature(int64_list=feature_pb2.Int64List( - value=[r])), - "keywords": - feature_pb2.Feature(bytes_list=feature_pb2.BytesList( - value=self._get_keywords(f, r))) - })) + example = example_pb2.Example( + features=feature_pb2.Features( + feature={ + "file": + feature_pb2.Feature( + int64_list=feature_pb2.Int64List(value=[f])), + "record": + feature_pb2.Feature( + int64_list=feature_pb2.Int64List(value=[r])), + "keywords": + feature_pb2.Feature( + bytes_list=feature_pb2.BytesList( + value=self._get_keywords(f, r))) + })) return example.SerializeToString() def _get_keywords(self, f, r): @@ -374,8 +376,8 @@ class ReadBatchFeaturesTest(test.TestCase): record_batch.append(r) keywords = self._get_keywords(f, r) keywords_batch_values.extend(keywords) - keywords_batch_indices.extend([[batch_index, i] - for i in range(len(keywords))]) + keywords_batch_indices.extend( + [[batch_index, i] for i in range(len(keywords))]) batch_index += 1 keywords_batch_max_len = max(keywords_batch_max_len, len(keywords)) if len(file_batch) == batch_size: @@ -475,9 +477,10 @@ class ReadBatchFeaturesTest(test.TestCase): "file": parsing_ops.FixedLenFeature([], dtypes.int64), "record": parsing_ops.FixedLenFeature([], dtypes.int64), } - dataset = (core_readers.TFRecordDataset(self.test_filenames) - .map(lambda x: parsing_ops.parse_single_example(x, features)) - .repeat(10).batch(2)) + dataset = ( + core_readers.TFRecordDataset(self.test_filenames) + .map(lambda x: parsing_ops.parse_single_example(x, features)) + .repeat(10).batch(2)) iterator = dataset.make_initializable_iterator() init_op = iterator.initializer next_element = iterator.get_next() @@ -607,20 +610,25 @@ class MakeCsvDatasetTest(test.TestCase): "record %d" % recordno if recordno % 2 == 1 else "", ] - def _csv_record(self, fileno, recordno): - return ",".join(str(v) for v in self._csv_values(fileno, recordno)) + def _write_file(self, filename, rows): + for i in range(len(rows)): + if isinstance(rows[i], list): + rows[i] = ",".join(str(v) if v is not None else "" for v in rows[i]) + fn = os.path.join(self.get_temp_dir(), filename) + f = open(fn, "w") + f.write("\n".join(rows)) + f.close() + return fn def _create_file(self, fileno, header=True, comment=True): - fn = os.path.join(self.get_temp_dir(), "csv_file%d.csv" % fileno) - f = open(fn, "w") + rows = [] if header: - f.write(",".join(self.COLUMNS) + "\n") + rows.append(self.COLUMNS) for recno in range(self._num_records): - f.write(self._csv_record(fileno, recno) + "\n") + rows.append(self._csv_values(fileno, recno)) if comment: - f.write("# Some comment goes here. Should be ignored!\n") - f.close() - return fn + rows.append("# Some comment goes here. Ignore me.") + return self._write_file("csv_file%d.csv" % fileno, rows) def _create_files(self): filenames = [] @@ -634,6 +642,7 @@ class MakeCsvDatasetTest(test.TestCase): defaults, column_names=COLUMNS, label_name=LABEL, + select_cols=None, batch_size=1, num_epochs=1, shuffle=False, @@ -656,6 +665,7 @@ class MakeCsvDatasetTest(test.TestCase): comment=comment, na_value=na_value, default_float_type=default_float_type, + select_columns=select_cols, ) def _next_actual_batch(self, file_indices, batch_size, num_epochs, defaults): @@ -712,7 +722,7 @@ class MakeCsvDatasetTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) - def test_make_csv_dataset(self): + def testMakeCSVDataset(self): defaults = self.DEFAULTS with ops.Graph().as_default() as g: @@ -739,7 +749,7 @@ class MakeCsvDatasetTest(test.TestCase): self._verify_records( sess, dataset, range(self._num_files), batch_size=2, num_epochs=10) - def test_make_csv_dataset_with_bad_columns(self): + def testMakeCSVDataset_withBadColumns(self): """Tests that exception is raised when input is malformed. """ dupe_columns = self.COLUMNS[:-1] + self.COLUMNS[:1] @@ -755,7 +765,7 @@ class MakeCsvDatasetTest(test.TestCase): self._make_csv_dataset( self._test_filenames, defaults, label_name="not_a_real_label") - def test_make_csv_dataset_with_no_label(self): + def testMakeCSVDataset_withNoLabel(self): """Tests that CSV datasets can be created when no label is specified. """ defaults = self.DEFAULTS @@ -776,7 +786,7 @@ class MakeCsvDatasetTest(test.TestCase): num_epochs=10, label_name=None) - def test_make_csv_dataset_with_no_comments(self): + def testMakeCSVDataset_withNoComments(self): """Tests that datasets can be created from CSV files with no header line. """ defaults = self.DEFAULTS @@ -799,7 +809,7 @@ class MakeCsvDatasetTest(test.TestCase): num_epochs=10, ) - def test_make_csv_dataset_with_no_header(self): + def testMakeCSVDataset_withNoHeader(self): """Tests that datasets can be created from CSV files with no header line. """ defaults = self.DEFAULTS @@ -822,7 +832,7 @@ class MakeCsvDatasetTest(test.TestCase): num_epochs=10, ) - def test_make_csv_dataset_with_types(self): + def testMakeCSVDataset_withTypes(self): """Tests that defaults can be a dtype instead of a Tensor for required vals. """ defaults = [d for d in self.COLUMN_TYPES[:-1]] @@ -832,7 +842,7 @@ class MakeCsvDatasetTest(test.TestCase): dataset = self._make_csv_dataset(self._test_filenames, defaults) self._verify_records(sess, dataset, range(self._num_files)) - def test_make_csv_dataset_with_no_col_names(self): + def testMakeCSVDataset_withNoColNames(self): """Tests that datasets can be created when column names are not specified. In that case, we should infer the column names from the header lines. @@ -851,7 +861,17 @@ class MakeCsvDatasetTest(test.TestCase): self._verify_records( sess, dataset, range(self._num_files), batch_size=2, num_epochs=10) - def test_make_csv_dataset_type_inference(self): + def testMakeCSVDataset_withTypeInferenceMismatch(self): + # Test that error is thrown when num fields doesn't match columns + with self.assertRaises(ValueError): + self._make_csv_dataset( + self._test_filenames, + column_names=self.COLUMNS + ["extra_name"], + defaults=None, + batch_size=2, + num_epochs=10) + + def testMakeCSVDataset_withTypeInference(self): """Tests that datasets can be created when no defaults are specified. In that case, we should infer the types from the first N records. @@ -875,19 +895,16 @@ class MakeCsvDatasetTest(test.TestCase): dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float32, dtypes.string, dtypes.string ] - rows = [[0, 0, 0, "NAN", "", "a"], [1, 2**31 + 1, 2**64, 123, "NAN", ""], + col_names = ["col%d" % i for i in range(len(expected_dtypes))] + rows = [[None, None, None, "NAN", "", + "a"], [1, 2**31 + 1, 2**64, 123, "NAN", ""], ['"123"', 2, 2**64, 123.4, "NAN", '"cd,efg"']] expected = [[0, 0, 0, 0, "", "a"], [1, 2**31 + 1, 2**64, 123, "", ""], [123, 2, 2**64, 123.4, "", "cd,efg"]] for row in expected: row[-1] = row[-1].encode("utf-8") # py3 expects byte strings row[-2] = row[-2].encode("utf-8") # py3 expects byte strings - col_names = ["col%d" % i for i in range(len(expected_dtypes))] - with open(fn, "w") as f: - f.write(",".join(col_names)) - f.write("\n") - for row in rows: - f.write(",".join([str(v) if v else "" for v in row]) + "\n") + self._write_file("file.csv", [col_names] + rows) with ops.Graph().as_default() as g: with self.test_session(graph=g) as sess: @@ -895,8 +912,6 @@ class MakeCsvDatasetTest(test.TestCase): fn, defaults=None, column_names=None, - batch_size=1, - num_epochs=1, label_name=None, na_value="NAN", default_float_type=dtypes.float32, @@ -919,8 +934,6 @@ class MakeCsvDatasetTest(test.TestCase): fn, defaults=None, column_names=None, - batch_size=1, - num_epochs=1, label_name=None, na_value="NAN", default_float_type=dtypes.float64, @@ -928,11 +941,99 @@ class MakeCsvDatasetTest(test.TestCase): features = dataset.make_one_shot_iterator().get_next() # Check that types match for i in range(len(expected_dtypes)): - assert features["col%d" % i].dtype == expected_dtypes[i] + self.assertAllEqual(features["col%d" % i].dtype, expected_dtypes[i]) for i in range(len(rows)): - assert sess.run(features) == dict(zip(col_names, expected[i])) + self.assertAllEqual( + sess.run(features), dict(zip(col_names, expected[i]))) + + def testMakeCSVDataset_withSelectColsError(self): + data = [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]] + col_names = ["col%d" % i for i in range(5)] + fn = self._write_file("file.csv", [col_names] + data) + with self.assertRaises(ValueError): + # Mismatch in number of defaults and number of columns selected, + # should raise an error + self._make_csv_dataset( + fn, + defaults=[[0]] * 5, + column_names=col_names, + label_name=None, + select_cols=[1, 3]) + with self.assertRaises(ValueError): + # Invalid column name should raise an error + self._make_csv_dataset( + fn, + defaults=[[0]], + column_names=col_names, + label_name=None, + select_cols=["invalid_col_name"]) + + def testMakeCSVDataset_withSelectCols(self): + data = [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]] + col_names = ["col%d" % i for i in range(5)] + fn = self._write_file("file.csv", [col_names] + data) + # If select_cols is specified, should only yield a subset of columns + with ops.Graph().as_default() as g: + with self.test_session(graph=g) as sess: + dataset = self._make_csv_dataset( + fn, + defaults=[[0], [0]], + column_names=col_names, + label_name=None, + select_cols=[1, 3]) + expected = [[1, 3], [6, 8]] + features = dataset.make_one_shot_iterator().get_next() + for i in range(len(data)): + self.assertAllEqual( + sess.run(features), + dict(zip([col_names[1], col_names[3]], expected[i]))) + # Can still do default inference with select_cols + with ops.Graph().as_default() as g: + with self.test_session(graph=g) as sess: + dataset = self._make_csv_dataset( + fn, + defaults=None, + column_names=col_names, + label_name=None, + select_cols=[1, 3]) + expected = [[1, 3], [6, 8]] + features = dataset.make_one_shot_iterator().get_next() + for i in range(len(data)): + self.assertAllEqual( + sess.run(features), + dict(zip([col_names[1], col_names[3]], expected[i]))) + # Can still do column name inference + with ops.Graph().as_default() as g: + with self.test_session(graph=g) as sess: + dataset = self._make_csv_dataset( + fn, + defaults=None, + column_names=None, + label_name=None, + select_cols=[1, 3]) + expected = [[1, 3], [6, 8]] + features = dataset.make_one_shot_iterator().get_next() + for i in range(len(data)): + self.assertAllEqual( + sess.run(features), + dict(zip([col_names[1], col_names[3]], expected[i]))) + # Can specify column names instead of indices + with ops.Graph().as_default() as g: + with self.test_session(graph=g) as sess: + dataset = self._make_csv_dataset( + fn, + defaults=None, + column_names=None, + label_name=None, + select_cols=[col_names[1], col_names[3]]) + expected = [[1, 3], [6, 8]] + features = dataset.make_one_shot_iterator().get_next() + for i in range(len(data)): + self.assertAllEqual( + sess.run(features), + dict(zip([col_names[1], col_names[3]], expected[i]))) - def test_make_csv_dataset_with_shuffle(self): + def testMakeCSVDataset_withShuffle(self): total_records = self._num_files * self._num_records defaults = self.DEFAULTS for batch_size in [1, 2]: diff --git a/tensorflow/contrib/data/python/ops/readers.py b/tensorflow/contrib/data/python/ops/readers.py index b8eb09978e..4ec8ae1c79 100644 --- a/tensorflow/contrib/data/python/ops/readers.py +++ b/tensorflow/contrib/data/python/ops/readers.py @@ -124,18 +124,21 @@ def _next_csv_row(filenames, num_cols, field_delim, use_quote_delim, header, def _infer_column_defaults(filenames, num_cols, field_delim, use_quote_delim, na_value, header, comment, float_dtype, - rows_for_inference): + num_rows_for_inference, select_columns): """Infers column types from the first N valid CSV records of files.""" - inferred_types = [None] * num_cols + if select_columns is None: + select_columns = range(num_cols) + inferred_types = [None] * len(select_columns) - for rows_read, csv_row in enumerate( + for i, csv_row in enumerate( _next_csv_row(filenames, num_cols, field_delim, use_quote_delim, header, comment)): - if rows_for_inference is not None and rows_read >= rows_for_inference: + if num_rows_for_inference is not None and i >= num_rows_for_inference: break - for i, str_val in enumerate(csv_row): - inferred_types[i] = _infer_type(str_val, na_value, inferred_types[i], - float_dtype) + + for j, col_index in enumerate(select_columns): + inferred_types[j] = _infer_type(csv_row[col_index], na_value, + inferred_types[j], float_dtype) # Replace None's with a default type inferred_types = [t or dtypes.string for t in inferred_types] @@ -162,12 +165,37 @@ def _infer_column_names(filenames, field_delim, use_quote_delim): return column_names +def _get_sorted_col_indices(select_columns, column_names): + """Transforms select_columns argument into sorted column indices.""" + names_to_indices = {n: i for i, n in enumerate(column_names)} + num_cols = len(column_names) + for i, v in enumerate(select_columns): + if isinstance(v, int): + if v < 0 or v >= num_cols: + raise ValueError( + "Column index %d specified in select_columns out of valid range." % + v) + continue + if v not in names_to_indices: + raise ValueError( + "Value '%s' specified in select_columns not a valid column index or " + "name." % v) + select_columns[i] = names_to_indices[v] + + # Sort and ensure there are no duplicates + result = sorted(set(select_columns)) + if len(result) != len(select_columns): + raise ValueError("select_columns contains duplicate columns") + return result + + def make_csv_dataset( file_pattern, batch_size, column_names=None, column_defaults=None, label_name=None, + select_columns=None, field_delim=",", use_quote_delim=True, na_value="", @@ -201,20 +229,32 @@ def make_csv_dataset( provided, infers the column names from the first row of the records. These names will be the keys of the features dict of each dataset element. column_defaults: A optional list of default values for the CSV fields. One - item per column of the input record. Each item in the list is either a - valid CSV dtype (float32, float64, int32, int64, or string), or a + item per selected column of the input record. Each item in the list is + either a valid CSV dtype (float32, float64, int32, int64, or string), or a `Tensor` with one of the aforementioned types. The tensor can either be a scalar default value (if the column is optional), or an empty tensor (if the column is required). If a dtype is provided instead of a tensor, the column is also treated as required. If this list is not provided, tries to infer types based on reading the first num_rows_for_inference rows of files specified, and assumes all columns are optional, defaulting to `0` - for numeric values and `""` for string values. + for numeric values and `""` for string values. If both this and + `select_columns` are specified, these must have the same lengths, and + `column_defaults` is assumed to be sorted in order of increasing column + index. label_name: A optional string corresponding to the label column. If provided, the data for this column is returned as a separate `Tensor` from the features dictionary, so that the dataset complies with the format expected by a `tf.Estimator.train` or `tf.Estimator.evaluate` input function. + select_columns: An optional list of integer indices or string column + names, that specifies a subset of columns of CSV data to select. If + column names are provided, these must correspond to names provided in + `column_names` or inferred from the file header lines. When this argument + is specified, only a subset of CSV columns will be parsed and returned, + corresponding to the columns specified. Using this results in faster + parsing and lower memory usage. If both this and `column_defaults` are + specified, these must have the same lengths, and `column_defaults` is + assumed to be sorted in order of increasing column index. field_delim: An optional `string`. Defaults to `","`. Char delimiter to separate fields in a record. use_quote_delim: An optional bool. Defaults to `True`. If false, treats @@ -279,6 +319,9 @@ def make_csv_dataset( if len(column_names) != len(set(column_names)): raise ValueError("Cannot have duplicate column names.") + if select_columns is not None: + select_columns = _get_sorted_col_indices(select_columns, column_names) + if column_defaults is not None: column_defaults = [ constant_op.constant([], dtype=x) if x in _ACCEPTABLE_CSV_TYPES else x @@ -289,7 +332,17 @@ def make_csv_dataset( # construction time column_defaults = _infer_column_defaults( filenames, len(column_names), field_delim, use_quote_delim, na_value, - header, comment, default_float_type, num_rows_for_inference) + header, comment, default_float_type, num_rows_for_inference, + select_columns) + + if select_columns is not None and len(column_defaults) != len(select_columns): + raise ValueError( + "If specified, column_defaults and select_columns must have same " + "length." + ) + if select_columns is not None and len(column_names) > len(select_columns): + # Pick the relevant subset of column names + column_names = [column_names[i] for i in select_columns] if label_name is not None and label_name not in column_names: raise ValueError("`label_name` provided must be one of the columns.") @@ -322,6 +375,7 @@ def make_csv_dataset( field_delim=field_delim, use_quote_delim=use_quote_delim, na_value=na_value, + select_cols=select_columns, ) features = dict(zip(column_names, columns)) if label_name is not None: diff --git a/tensorflow/core/kernels/decode_csv_op.cc b/tensorflow/core/kernels/decode_csv_op.cc index 0c42f63252..3eed847c16 100644 --- a/tensorflow/core/kernels/decode_csv_op.cc +++ b/tensorflow/core/kernels/decode_csv_op.cc @@ -34,6 +34,19 @@ class DecodeCSVOp : public OpKernel { errors::InvalidArgument("Out type too large")); OP_REQUIRES_OK(ctx, ctx->GetAttr("field_delim", &delim)); OP_REQUIRES_OK(ctx, ctx->GetAttr("use_quote_delim", &use_quote_delim_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("select_cols", &select_cols_)); + OP_REQUIRES( + ctx, out_type_.size() == select_cols_.size() || select_cols_.empty(), + errors::InvalidArgument("select_cols should match output size")); + select_all_cols_ = select_cols_.empty(); + for (int i = 1; i < select_cols_.size(); i++) { + OP_REQUIRES(ctx, select_cols_[i - 1] < select_cols_[i], + errors::InvalidArgument( + "select_cols should be strictly increasing indices")); + } + OP_REQUIRES( + ctx, select_cols_.empty() || select_cols_.front() >= 0, + errors::InvalidArgument("select_cols should be non-negative indices")); OP_REQUIRES(ctx, delim.size() == 1, errors::InvalidArgument("field_delim should be only 1 char")); delim_ = delim[0]; @@ -183,13 +196,18 @@ class DecodeCSVOp : public OpKernel { private: std::vector out_type_; + std::vector select_cols_; char delim_; bool use_quote_delim_; + bool select_all_cols_; string na_value_; void ExtractFields(OpKernelContext* ctx, StringPiece input, std::vector* result) { int64 current_idx = 0; + int64 num_fields_parsed = 0; + int64 selector_idx = 0; // Keep track of index into select_cols + if (!input.empty()) { while (static_cast(current_idx) < input.size()) { if (input[current_idx] == '\n' || input[current_idx] == '\r') { @@ -198,6 +216,10 @@ class DecodeCSVOp : public OpKernel { } bool quoted = false; + bool include = + (select_all_cols_ || select_cols_[selector_idx] == + static_cast(num_fields_parsed)); + if (use_quote_delim_ && input[current_idx] == '"') { quoted = true; current_idx++; @@ -214,7 +236,7 @@ class DecodeCSVOp : public OpKernel { input[current_idx] != '\r', errors::InvalidArgument( "Unquoted fields cannot have quotes/CRLFs inside")); - field += input[current_idx]; + if (include) field += input[current_idx]; current_idx++; } @@ -226,14 +248,14 @@ class DecodeCSVOp : public OpKernel { (static_cast(current_idx) < input.size() - 1) && (input[current_idx] != '"' || input[current_idx + 1] != delim_)) { if (input[current_idx] != '"') { - field += input[current_idx]; + if (include) field += input[current_idx]; current_idx++; } else { OP_REQUIRES( ctx, input[current_idx + 1] == '"', errors::InvalidArgument("Quote inside a string has to be " "escaped by another quote")); - field += '"'; + if (include) field += '"'; current_idx += 2; } } @@ -250,11 +272,20 @@ class DecodeCSVOp : public OpKernel { current_idx += 2; } - result->push_back(field); + num_fields_parsed++; + if (include) { + result->push_back(field); + selector_idx++; + if (selector_idx == select_cols_.size()) return; + } } + bool include = + (select_all_cols_ || select_cols_[selector_idx] == + static_cast(num_fields_parsed)); // Check if the last field is missing - if (input[input.size() - 1] == delim_) result->push_back(string()); + if (include && input[input.size() - 1] == delim_) + result->push_back(string()); } } }; diff --git a/tensorflow/core/ops/parsing_ops.cc b/tensorflow/core/ops/parsing_ops.cc index ddd2aa9274..ddb714b4e9 100644 --- a/tensorflow/core/ops/parsing_ops.cc +++ b/tensorflow/core/ops/parsing_ops.cc @@ -245,6 +245,7 @@ REGISTER_OP("DecodeCSV") .Attr("field_delim: string = ','") .Attr("use_quote_delim: bool = true") .Attr("na_value: string = ''") + .Attr("select_cols: list(int) = []") .SetShapeFn([](InferenceContext* c) { // Validate the record_defaults inputs. for (int i = 1; i < c->num_inputs(); ++i) { diff --git a/tensorflow/python/kernel_tests/decode_csv_op_test.py b/tensorflow/python/kernel_tests/decode_csv_op_test.py index fec52fa9cc..4f49d72676 100644 --- a/tensorflow/python/kernel_tests/decode_csv_op_test.py +++ b/tensorflow/python/kernel_tests/decode_csv_op_test.py @@ -78,9 +78,11 @@ class DecodeCSVOpTest(test.TestCase): self._test(args, expected_out) def test2DNoQuoteDelimiter(self): - args = {"records": [["1", "2"], ['""', '"']], - "record_defaults": [[""]], - "use_quote_delim": False} + args = { + "records": [["1", "2"], ['""', '"']], + "record_defaults": [[""]], + "use_quote_delim": False + } expected_out = [[[b"1", b"2"], [b'""', b'"']]] self._test(args, expected_out) @@ -88,8 +90,7 @@ class DecodeCSVOpTest(test.TestCase): def testDouble(self): args = { "records": ["1.0", "-1.79e+308", '"1.79e+308"'], - "record_defaults": [np.array( - [], dtype=np.double)], + "record_defaults": [np.array([], dtype=np.double)], } expected_out = [[1.0, -1.79e+308, 1.79e+308]] @@ -99,8 +100,7 @@ class DecodeCSVOpTest(test.TestCase): def testInt64(self): args = { "records": ["1", "2", '"2147483648"'], - "record_defaults": [np.array( - [], dtype=np.int64)], + "record_defaults": [np.array([], dtype=np.int64)], } expected_out = [[1, 2, 2147483648]] @@ -173,8 +173,7 @@ class DecodeCSVOpTest(test.TestCase): def testWithoutDefaultsError(self): args = { "records": [",1", "0.2,3", "3.0,"], - "record_defaults": [[1.0], np.array( - [], dtype=np.int32)] + "record_defaults": [[1.0], np.array([], dtype=np.int32)] } self._test( @@ -183,8 +182,7 @@ class DecodeCSVOpTest(test.TestCase): def testWrongFieldIntError(self): args = { "records": [",1", "0.2,234a", "3.0,2"], - "record_defaults": [[1.0], np.array( - [], dtype=np.int32)] + "record_defaults": [[1.0], np.array([], dtype=np.int32)] } self._test( @@ -202,8 +200,7 @@ class DecodeCSVOpTest(test.TestCase): def testWrongFieldFloatError(self): args = { "records": [",1", "0.2,2", "3.0adf,3"], - "record_defaults": [[1.0], np.array( - [], dtype=np.int32)] + "record_defaults": [[1.0], np.array([], dtype=np.int32)] } self._test( @@ -229,6 +226,73 @@ class DecodeCSVOpTest(test.TestCase): self._test( args, expected_err_re="Quoted field has to end with quote followed.*") + def testSelectCols(self): + args = { + "records": [",,", "4,5,6"], + "record_defaults": [[1], [2]], + "select_cols": [0, 1] + } + expected_out = [[1, 4], [2, 5]] + self._test(args, expected_out) + + def testSelectColsInclLast(self): + # The last col is a edge-casey; add test for that + args = { + "records": [",,", "4,5,6"], + "record_defaults": [[0], [1], [2]], + "select_cols": [0, 1, 2] + } + expected_out = [[0, 4], [1, 5], [2, 6]] + self._test(args, expected_out) + + def testWrongSelectColsInclLast(self): + # The last col is a edge-casey; add test for that + args = { + "records": [",,", "4,5,6"], + "record_defaults": [[0], [1], [2]], + "select_cols": [0, 1, 3] + } + self._test(args, expected_err_re="Expect 3 fields but have 2 in record 0") + + def testWrongSelectColsLen(self): + args = { + "records": ["1,2,3", "4,5,6"], + "record_defaults": [[0], [0], [0]], + "select_cols": [0] + } + with self.assertRaisesWithPredicateMatch( + ValueError, "Length of select_cols and record_defaults do not match."): + self._test(args) + + def testWrongSelectColsSorting(self): + args = { + "records": ["1,2,3"], + "record_defaults": [[0], [1]], + "select_cols": [1, 0] + } + with self.assertRaisesWithPredicateMatch( + ValueError, "select_cols is not strictly increasing."): + self._test(args) + + def testWrongSelectColsIndicesNegative(self): + args = { + "records": ["1,2,3"], + "record_defaults": [[0], [1]], + "select_cols": [-1, 0] # -1 is not a valid index + } + with self.assertRaisesWithPredicateMatch( + ValueError, "select_cols contains negative values."): + self._test(args) + + def testWrongSelectColsIndicesTooHigh(self): + args = { + "records": ["1,2,3"], + "record_defaults": [[0], [1]], + "select_cols": [0, 3] # 3 is not a valid index + } + # Only successfully parses one of the columns + self._test(args, expected_err_re="Expect 2 fields but have 1 in record 0") + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/ops/parsing_ops.py b/tensorflow/python/ops/parsing_ops.py index 075b38d743..d8d9af545f 100644 --- a/tensorflow/python/ops/parsing_ops.py +++ b/tensorflow/python/ops/parsing_ops.py @@ -1176,8 +1176,13 @@ def _parse_single_sequence_example_raw(serialized, # Swap `name` and `na_value` for backward compatibility. @tf_export("decode_csv") -def decode_csv(records, record_defaults, field_delim=",", - use_quote_delim=True, name=None, na_value=""): +def decode_csv(records, + record_defaults, + field_delim=",", + use_quote_delim=True, + name=None, + na_value="", + select_cols=None): """Convert CSV records to tensors. Each column maps to one tensor. RFC 4180 format is expected for the CSV records. @@ -1200,19 +1205,32 @@ def decode_csv(records, record_defaults, field_delim=",", Bullet 5). name: A name for the operation (optional). na_value: Additional string to recognize as NA/NaN. + select_cols: Optional sorted list of column indices to select. If specified, + only this subset of columns will be parsed and returned. Returns: A list of `Tensor` objects. Has the same type as `record_defaults`. Each tensor will have the same shape as records. + + Raises: + ValueError: If any of the arguments is malformed. """ - # TODO(martinwicke), remove the wrapper when new Python API generator is done. + if select_cols is not None and any(select_cols[i] >= select_cols[i + 1] + for i in range(len(select_cols) - 1)): + raise ValueError("select_cols is not strictly increasing.") + if select_cols is not None and select_cols[0] < 0: + raise ValueError("select_cols contains negative values.") + if select_cols is not None and len(select_cols) != len(record_defaults): + raise ValueError("Length of select_cols and record_defaults do not match.") return gen_parsing_ops.decode_csv( records=records, record_defaults=record_defaults, field_delim=field_delim, use_quote_delim=use_quote_delim, na_value=na_value, - name=name) + name=name, + select_cols=select_cols, + ) # TODO(b/70890287): Combine the implementation of this op and diff --git a/tensorflow/tools/api/golden/tensorflow.pbtxt b/tensorflow/tools/api/golden/tensorflow.pbtxt index afa3b78eb7..be64fd19d8 100644 --- a/tensorflow/tools/api/golden/tensorflow.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.pbtxt @@ -914,7 +914,7 @@ tf_module { } member_method { name: "decode_csv" - argspec: "args=[\'records\', \'record_defaults\', \'field_delim\', \'use_quote_delim\', \'name\', \'na_value\'], varargs=None, keywords=None, defaults=[\',\', \'True\', \'None\', \'\'], " + argspec: "args=[\'records\', \'record_defaults\', \'field_delim\', \'use_quote_delim\', \'name\', \'na_value\', \'select_cols\'], varargs=None, keywords=None, defaults=[\',\', \'True\', \'None\', \'\', \'None\'], " } member_method { name: "decode_json_example" -- GitLab From 2782350e7ec81867dc68b0fb8bf0a6ca3dde5c12 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 9 Apr 2018 18:08:13 -0700 Subject: [PATCH 069/791] [XLA] Redesign: implement fft. PiperOrigin-RevId: 192220070 --- .../compiler/xla/client/xla_client/xla_builder.cc | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc index ed9f994d39..170dd59c79 100644 --- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc +++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc @@ -777,7 +777,20 @@ XlaOp XlaBuilder::ConvGeneralDilated( XlaOp XlaBuilder::Fft(const XlaOp& operand, const FftType fft_type, const tensorflow::gtl::ArraySlice fft_length) { - return UnimplementedOp(); + return NoteErrorOrReturn([&]() -> StatusOr { + HloInstructionProto instr; + TF_ASSIGN_OR_RETURN(const Shape& operand_shape, operand.GetShape()); + TF_ASSIGN_OR_RETURN( + *instr.mutable_shape(), + ShapeInference::InferFftShape(operand_shape, fft_type, fft_length)); + + instr.set_fft_type(fft_type); + for (int64 i : fft_length) { + instr.add_fft_length(i); + } + + return AddInstruction(std::move(instr), HloOpcode::kFft, {operand}); + }); } XlaOp XlaBuilder::Infeed(const Shape& shape, const string& config) { -- GitLab From 0166491e89487ce6cd6e10bfa77ba82ea42d8a59 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 9 Apr 2018 18:46:09 -0700 Subject: [PATCH 070/791] Add the total device time and total host time to tf_op_stats. PiperOrigin-RevId: 192223560 --- tensorflow/contrib/tpu/profiler/tf_op_stats.proto | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tensorflow/contrib/tpu/profiler/tf_op_stats.proto b/tensorflow/contrib/tpu/profiler/tf_op_stats.proto index 2a15875627..63955d1806 100644 --- a/tensorflow/contrib/tpu/profiler/tf_op_stats.proto +++ b/tensorflow/contrib/tpu/profiler/tf_op_stats.proto @@ -66,6 +66,10 @@ message OpMetricsDbResult { // The total of the difference between the start times of two // consecutive infeed-enqueues (per host) in picoseconds. optional uint64 total_host_infeed_enq_start_timestamp_ps_diff = 3; + // The total device time in microseconds. + optional double total_device_time_in_us = 4; + // The total host time in microseconds. + optional double total_host_time_in_us = 5; } // Result proto for StepInfo. -- GitLab From a45caf444228dbb57343092dc3054f85ff081ef6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 9 Apr 2018 18:53:22 -0700 Subject: [PATCH 071/791] Include standard headers when libc++ requires them. PiperOrigin-RevId: 192224104 --- tensorflow/contrib/lite/allocation.cc | 1 + tensorflow/contrib/lite/arena_planner.cc | 1 + tensorflow/core/framework/attr_value_util_test.cc | 1 + tensorflow/core/grappler/costs/robust_stats.cc | 1 + 4 files changed, 4 insertions(+) diff --git a/tensorflow/contrib/lite/allocation.cc b/tensorflow/contrib/lite/allocation.cc index 4b322e027d..a4772731ec 100644 --- a/tensorflow/contrib/lite/allocation.cc +++ b/tensorflow/contrib/lite/allocation.cc @@ -22,6 +22,7 @@ limitations under the License. #include #include #include +#include #include "tensorflow/contrib/lite/allocation.h" #include "tensorflow/contrib/lite/context.h" diff --git a/tensorflow/contrib/lite/arena_planner.cc b/tensorflow/contrib/lite/arena_planner.cc index 8e47e2375e..4f836d3677 100644 --- a/tensorflow/contrib/lite/arena_planner.cc +++ b/tensorflow/contrib/lite/arena_planner.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ #include "tensorflow/contrib/lite/arena_planner.h" +#include namespace tflite { diff --git a/tensorflow/core/framework/attr_value_util_test.cc b/tensorflow/core/framework/attr_value_util_test.cc index e4fad917ff..1a3994736c 100644 --- a/tensorflow/core/framework/attr_value_util_test.cc +++ b/tensorflow/core/framework/attr_value_util_test.cc @@ -15,6 +15,7 @@ limitations under the License. #include "tensorflow/core/framework/attr_value_util.h" +#include #include #include "tensorflow/core/framework/attr_value.pb.h" #include "tensorflow/core/lib/core/status_test_util.h" diff --git a/tensorflow/core/grappler/costs/robust_stats.cc b/tensorflow/core/grappler/costs/robust_stats.cc index 9866bc8688..5151b87c59 100644 --- a/tensorflow/core/grappler/costs/robust_stats.cc +++ b/tensorflow/core/grappler/costs/robust_stats.cc @@ -16,6 +16,7 @@ limitations under the License. #include "tensorflow/core/grappler/costs/robust_stats.h" #include #include +#include namespace tensorflow { namespace grappler { -- GitLab From d09bd1b06c4f4c8efad15c1e77c8c54710ccf077 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 9 Apr 2018 19:06:05 -0700 Subject: [PATCH 072/791] [XLA] Attach a reference client to the client_library_test_base, and implement ComputeAndCompare for XlaBuilder. - In ComputeAndCompare(XlaBuilder..), compute the reference result by executing on the interpreter backend. - Also migrate some tests to use the new ComputeAndCompare(XlaBuilder..). PiperOrigin-RevId: 192225152 --- tensorflow/compiler/xla/tests/BUILD | 4 + .../xla/tests/client_library_test_base.cc | 95 +++++++++++++++++++ .../xla/tests/client_library_test_base.h | 26 ++++- tensorflow/compiler/xla/tests/reduce_test.cc | 19 ++-- 4 files changed, 133 insertions(+), 11 deletions(-) diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD index 218345772f..8ecb421780 100644 --- a/tensorflow/compiler/xla/tests/BUILD +++ b/tensorflow/compiler/xla/tests/BUILD @@ -191,6 +191,8 @@ cc_library( "//tensorflow/compiler/xla/client:global_data", "//tensorflow/compiler/xla/client:local_client", "//tensorflow/compiler/xla/client/xla_client:xla_builder", + "//tensorflow/compiler/xla/service:interpreter_plugin", # reference backend + "//tensorflow/compiler/xla/service:platform_util", "//tensorflow/compiler/xla/tests:literal_test_util", "//tensorflow/compiler/xla/tests:test_utils", "//tensorflow/core:lib", @@ -1063,6 +1065,8 @@ xla_test( "//tensorflow/compiler/xla/client:global_data", "//tensorflow/compiler/xla/client:local_client", "//tensorflow/compiler/xla/client/lib:arithmetic", + "//tensorflow/compiler/xla/client/xla_client:xla_builder", + "//tensorflow/compiler/xla/client/xla_client:xla_computation", "//tensorflow/compiler/xla/tests:client_library_test_base", "//tensorflow/compiler/xla/tests:literal_test_util", "//tensorflow/compiler/xla/tests:xla_internal_test_main", diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.cc b/tensorflow/compiler/xla/tests/client_library_test_base.cc index c2e3cd2350..312d8f284d 100644 --- a/tensorflow/compiler/xla/tests/client_library_test_base.cc +++ b/tensorflow/compiler/xla/tests/client_library_test_base.cc @@ -23,6 +23,7 @@ limitations under the License. #include "tensorflow/compiler/xla/execution_options_util.h" #include "tensorflow/compiler/xla/literal_util.h" #include "tensorflow/compiler/xla/ptr_util.h" +#include "tensorflow/compiler/xla/service/platform_util.h" #include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/status_macros.h" #include "tensorflow/compiler/xla/statusor.h" @@ -35,6 +36,10 @@ namespace se = ::perftools::gputools; namespace xla { namespace { + +// Name of the interpreter backend. +constexpr char kInterpreter[] = "interpreter"; + // Wrapper function that creates a nicer error message (than a bare // ValueOrDie()) if the platform we intend to test is not available. Client* GetOrCreateLocalClientOrDie(const LocalClientOptions& client_options) { @@ -43,6 +48,14 @@ Client* GetOrCreateLocalClientOrDie(const LocalClientOptions& client_options) { TF_CHECK_OK(result.status()) << " could not create local client for testing"; return result.ValueOrDie(); } + +// Helper functions to get the reference platform. +se::Platform* GetReferencePlatform() { + auto result = PlatformUtil::GetPlatform(kInterpreter); + TF_CHECK_OK(result.status()) << "could not get interpreter platform"; + return result.ValueOrDie(); +} + } // namespace ClientLibraryTestBase::ClientLibraryTestBase( @@ -66,6 +79,11 @@ ClientLibraryTestBase::ClientLibraryTestBase(se::Platform* platform) LocalClientOptions default_options; default_options.set_platform(platform); client_ = GetOrCreateLocalClientOrDie(default_options); + + LocalClientOptions ref_options; + ref_options.set_platform(GetReferencePlatform()); + ref_client_ = GetOrCreateLocalClientOrDie(ref_options); + execution_options_.mutable_debug_options()->add_xla_disable_hlo_passes( "constant_folding"); } @@ -127,6 +145,20 @@ StatusOr> ClientLibraryTestBase::ExecuteAndTransfer( return ExecuteAndTransfer(computation, arguments, shape_with_output_layout); } +StatusOr> +ClientLibraryTestBase::ExecuteAndTransferReference( + const XlaComputation& computation, + tensorflow::gtl::ArraySlice arguments, + const Shape* shape_with_output_layout) { + ExecutionOptions execution_options = execution_options_; + if (shape_with_output_layout != nullptr) { + *execution_options.mutable_shape_with_output_layout() = + *shape_with_output_layout; + } + return ref_client_->ExecuteAndTransfer(computation, arguments, + &execution_options); +} + std::unique_ptr ClientLibraryTestBase::ExecuteOrDie( ComputationBuilder* builder, tensorflow::gtl::ArraySlice arguments) { @@ -521,6 +553,69 @@ ClientLibraryTestBase::ComputeValueAndReference( return std::make_pair(std::move(reference), std::move(result)); } +void ClientLibraryTestBase::ComputeAndCompare( + XlaBuilder* builder, tensorflow::gtl::ArraySlice arguments) { + auto status_or_data = ComputeValueAndReference(builder, arguments); + EXPECT_IS_OK(status_or_data); + if (!status_or_data.ok()) { + return; + } + std::unique_ptr reference, result; + std::tie(reference, result) = status_or_data.ConsumeValueOrDie(); + LiteralTestUtil::ExpectEqual(*reference, *result); +} + +void ClientLibraryTestBase::ComputeAndCompare( + XlaBuilder* builder, tensorflow::gtl::ArraySlice arguments, + ErrorSpec error) { + auto status_or_data = ComputeValueAndReference(builder, arguments); + EXPECT_IS_OK(status_or_data); + if (!status_or_data.ok()) { + return; + } + std::unique_ptr reference, result; + std::tie(reference, result) = status_or_data.ConsumeValueOrDie(); + LiteralTestUtil::ExpectNear(*reference, *result, error); +} + +StatusOr, std::unique_ptr>> +ClientLibraryTestBase::ComputeValueAndReference( + XlaBuilder* builder, tensorflow::gtl::ArraySlice arguments) { + // Transfer the arguments to the executor service. We put the unique_ptr's + // into a vector to keep the data alive on the service until the end of this + // function. + std::vector> argument_data; + std::vector> ref_argument_data; + for (const auto& arg : arguments) { + TF_ASSIGN_OR_RETURN(auto data, client_->TransferToServer(arg.Clone())); + TF_ASSIGN_OR_RETURN(auto ref_data, ref_client_->TransferToServer(arg)); + argument_data.push_back(std::move(data)); + ref_argument_data.push_back(std::move(ref_data)); + } + + // Create raw pointers to the GlobalData for the rest of the call stack. + std::vector argument_data_ptr; + std::transform( + argument_data.begin(), argument_data.end(), + std::back_inserter(argument_data_ptr), + [](const std::unique_ptr& data) { return data.get(); }); + std::vector ref_argument_data_ptr; + std::transform( + ref_argument_data.begin(), ref_argument_data.end(), + std::back_inserter(ref_argument_data_ptr), + [](const std::unique_ptr& data) { return data.get(); }); + + TF_ASSIGN_OR_RETURN(auto computation, builder->Build()); + + TF_ASSIGN_OR_RETURN(auto result, + ExecuteAndTransfer(computation, argument_data_ptr)); + + TF_ASSIGN_OR_RETURN(auto reference, ExecuteAndTransferReference( + computation, ref_argument_data_ptr)); + + return std::make_pair(std::move(reference), std::move(result)); +} + Computation ClientLibraryTestBase::CreateScalarRelu() { ComputationBuilder builder(client_, "relu"); auto shape = ShapeUtil::MakeShape(use_bfloat16_ ? BF16 : F32, {}); diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.h b/tensorflow/compiler/xla/tests/client_library_test_base.h index 0572acff88..b3212dd228 100644 --- a/tensorflow/compiler/xla/tests/client_library_test_base.h +++ b/tensorflow/compiler/xla/tests/client_library_test_base.h @@ -114,6 +114,14 @@ class ClientLibraryTestBase : public ::testing::Test { tensorflow::gtl::ArraySlice arguments, const Shape* shape_with_output_layout = nullptr); + // This executes the computation via the reference client (which connects a + // interpreter backend). The result is used as the expected values of the + // computation. + StatusOr> ExecuteAndTransferReference( + const XlaComputation& computation, + tensorflow::gtl::ArraySlice arguments, + const Shape* shape_with_output_layout = nullptr); + // Convenience OrDie variants of above methods. std::unique_ptr ExecuteOrDie( ComputationBuilder* builder, @@ -236,6 +244,14 @@ class ClientLibraryTestBase : public ::testing::Test { tensorflow::gtl::ArraySlice arguments, ErrorSpec error); + // Convenience method for running a built computation and comparing the result + // with the reference result. + void ComputeAndCompare(XlaBuilder* builder, + tensorflow::gtl::ArraySlice arguments); + void ComputeAndCompare(XlaBuilder* builder, + tensorflow::gtl::ArraySlice arguments, + ErrorSpec error); + // Create scalar operations for use in reductions. Computation CreateScalarRelu(); Computation CreateScalarMax(); @@ -413,6 +429,7 @@ class ClientLibraryTestBase : public ::testing::Test { PrimitiveType FloatType() const { return use_bfloat16_ ? BF16 : F32; } Client* client_; + Client* ref_client_; // To compute reference result. ExecutionOptions execution_options_; private: @@ -444,12 +461,19 @@ class ClientLibraryTestBase : public ::testing::Test { const Shape* output_with_layout = nullptr); // Executes the computation and calculates the expected reference value using - // the HloEvaluator. Returns two literal in the order of (expected, actual). + // the HloEvaluator. Returns two literals in the order of (expected, actual). StatusOr, std::unique_ptr>> ComputeValueAndReference(ComputationBuilder* builder, const ComputationDataHandle& operand, tensorflow::gtl::ArraySlice arguments); + // Executes the computation and calculates the expected reference value using + // the reference client. Returns two literals in the order of (expected, + // actual). + StatusOr, std::unique_ptr>> + ComputeValueAndReference(XlaBuilder* builder, + tensorflow::gtl::ArraySlice arguments); + // Whether to run tests with all float-type input/output converted to // bfloat16. bool use_bfloat16_ = false; diff --git a/tensorflow/compiler/xla/tests/reduce_test.cc b/tensorflow/compiler/xla/tests/reduce_test.cc index d24927d22b..768beec15e 100644 --- a/tensorflow/compiler/xla/tests/reduce_test.cc +++ b/tensorflow/compiler/xla/tests/reduce_test.cc @@ -39,6 +39,8 @@ limitations under the License. #include "tensorflow/compiler/xla/client/global_data.h" #include "tensorflow/compiler/xla/client/lib/arithmetic.h" #include "tensorflow/compiler/xla/client/local_client.h" +#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h" +#include "tensorflow/compiler/xla/client/xla_client/xla_computation.h" #include "tensorflow/compiler/xla/layout_util.h" #include "tensorflow/compiler/xla/literal_util.h" #include "tensorflow/compiler/xla/reference_util.h" @@ -502,21 +504,18 @@ XLA_TEST_F(ReduceTest, TransposeAndReduceElementwiseR2_111x50_To_R1) { // Test that algebraic simplifier does not incorrectly fold a transpose into a // reduction operation. XLA_TEST_F(ReduceTest, TransposeAndReduceR3_12x111x50_To_R2) { - ComputationBuilder builder(client_, TestName()); - Computation add_f32 = CreateScalarAddComputation(F32, &builder); + XlaBuilder builder(TestName()); + XlaComputation add_f32 = CreateScalarAddComputation(F32, &builder); const Shape input_shape = ShapeUtil::MakeShape(F32, {12, 111, 50}); - ComputationDataHandle input = builder.Parameter(0, input_shape, "input"); - ComputationDataHandle zero = builder.ConstantR0(0.0); - ComputationDataHandle transpose = - builder.Transpose(input, /*permutation=*/{1, 0, 2}); - ComputationDataHandle reduce = - builder.Reduce(transpose, zero, add_f32, /*dimensions_to_reduce=*/{0}); + XlaOp input = builder.Parameter(0, input_shape, "input"); + XlaOp zero = builder.ConstantR0(0.0); + XlaOp transpose = builder.Transpose(input, /*permutation=*/{1, 0, 2}); + builder.Reduce(transpose, zero, add_f32, /*dimensions_to_reduce=*/{0}); TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr input_data, MakeFakeLiteral(input_shape)); - ComputeAndCompare(&builder, reduce, {std::move(*input_data)}, - ErrorSpec(0.01, 1e-4)); + ComputeAndCompare(&builder, {std::move(*input_data)}, ErrorSpec(0.01, 1e-4)); } XLA_TEST_F(ReduceTest, Reshape_111x2x25Reduce_111x50_To_R1) { -- GitLab From e3e0af4bd9b1d7a4628a5a4d6901a2d8529cfda5 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 9 Apr 2018 19:19:38 -0700 Subject: [PATCH 073/791] Update ops-related pbtxt files. PiperOrigin-RevId: 192226063 --- .../core/ops/compat/ops_history.v1.pbtxt | 59 +++++++++++++++++++ tensorflow/core/ops/ops.pbtxt | 8 +++ 2 files changed, 67 insertions(+) diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt index 026bfa89cf..fe4b7a7be0 100644 --- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt +++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt @@ -16407,6 +16407,65 @@ op { } } } +op { + name: "DecodeCSV" + input_arg { + name: "records" + type: DT_STRING + } + input_arg { + name: "record_defaults" + type_list_attr: "OUT_TYPE" + } + output_arg { + name: "output" + type_list_attr: "OUT_TYPE" + } + attr { + name: "OUT_TYPE" + type: "list(type)" + has_minimum: true + minimum: 1 + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT32 + type: DT_INT64 + type: DT_STRING + } + } + } + attr { + name: "field_delim" + type: "string" + default_value { + s: "," + } + } + attr { + name: "use_quote_delim" + type: "bool" + default_value { + b: true + } + } + attr { + name: "na_value" + type: "string" + default_value { + s: "" + } + } + attr { + name: "select_cols" + type: "list(int)" + default_value { + list { + } + } + } +} op { name: "DecodeCompressed" input_arg { diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index b61a3b0e64..9950388357 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -7361,6 +7361,14 @@ op { s: "" } } + attr { + name: "select_cols" + type: "list(int)" + default_value { + list { + } + } + } } op { name: "DecodeCompressed" -- GitLab From a356b2128a9bdbc33eceeff4b058f4d5d2e97738 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 9 Apr 2018 19:46:01 -0700 Subject: [PATCH 074/791] Go: Update generated wrapper functions for TensorFlow ops. PiperOrigin-RevId: 192227995 --- tensorflow/go/op/wrappers.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index 3d261c9d0a..09da8c1892 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -13882,6 +13882,14 @@ func DecodeCSVNaValue(value string) DecodeCSVAttr { } } +// DecodeCSVSelectCols sets the optional select_cols attribute to value. +// If not specified, defaults to <> +func DecodeCSVSelectCols(value []int64) DecodeCSVAttr { + return func(m optionalAttr) { + m["select_cols"] = value + } +} + // Convert CSV records to tensors. Each column maps to one tensor. // // RFC 4180 format is expected for the CSV records. -- GitLab From 405553005f9742203d1f0ac0c0a1740fe19766bd Mon Sep 17 00:00:00 2001 From: Michael Case Date: Mon, 9 Apr 2018 20:26:16 -0700 Subject: [PATCH 075/791] Fix the GCS Kokoro build badge links. The original badges I added were redirected/authenticated links (I think). This resulted in broken images on the README. These new links should not be redirected and should just link to the badge images. PiperOrigin-RevId: 192230689 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 177265500f..a69cf1ffea 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ | **`Documentation`** | **`Linux CPU`** | **`Linux GPU`** | **`Mac OS CPU`** | **`Windows CPU`** | **`Android`** | |-----------------|---------------------|------------------|-------------------|---------------|---------------| -| [![Documentation](https://img.shields.io/badge/api-reference-blue.svg)](https://www.tensorflow.org/api_docs/) | ![Build Status](https://storage.cloud.google.com/tensorflow-kokoro-build-badges/ubuntu-cc.png) | ![Build Status](https://storage.cloud.google.com/tensorflow-kokoro-build-badges/ubuntu-gpu-cc.png) | ![Build Status](https://storage.cloud.google.com/tensorflow-kokoro-build-badges/macos-py2-cc.png) | [![Build Status](https://ci.tensorflow.org/buildStatus/icon?job=tensorflow-master-win-cmake-py)](https://ci.tensorflow.org/job/tensorflow-master-win-cmake-py) | [![Build Status](https://ci.tensorflow.org/buildStatus/icon?job=tensorflow-master-android)](https://ci.tensorflow.org/job/tensorflow-master-android) [ ![Download](https://api.bintray.com/packages/google/tensorflow/tensorflow/images/download.svg) ](https://bintray.com/google/tensorflow/tensorflow/_latestVersion) +| [![Documentation](https://img.shields.io/badge/api-reference-blue.svg)](https://www.tensorflow.org/api_docs/) | ![Build Status](https://storage.googleapis.com/tensorflow-kokoro-build-badges/ubuntu-cc.png) | ![Build Status](https://storage.googleapis.com/tensorflow-kokoro-build-badges/ubuntu-gpu-cc.png) | ![Build Status](https://storage.googleapis.com/tensorflow-kokoro-build-badges/macos-py2-cc.png) | [![Build Status](https://ci.tensorflow.org/buildStatus/icon?job=tensorflow-master-win-cmake-py)](https://ci.tensorflow.org/job/tensorflow-master-win-cmake-py) | [![Build Status](https://ci.tensorflow.org/buildStatus/icon?job=tensorflow-master-android)](https://ci.tensorflow.org/job/tensorflow-master-android) [ ![Download](https://api.bintray.com/packages/google/tensorflow/tensorflow/images/download.svg) ](https://bintray.com/google/tensorflow/tensorflow/_latestVersion) **TensorFlow** is an open source software library for numerical computation using data flow graphs. The graph nodes represent mathematical operations, while -- GitLab From 1c8f3c81698b67b8fffce86c97df27d392b84cb8 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 9 Apr 2018 21:36:36 -0700 Subject: [PATCH 076/791] Updating three tests in constant_folding_test.cc with PlaceHolders Nodes to check EvaluateNodes returns the same output for the original and optimized graph PiperOrigin-RevId: 192235310 --- .../optimizers/constant_folding_test.cc | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc index 08c92687e3..31abe43846 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc +++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc @@ -1925,6 +1925,14 @@ TEST_F(ConstantFoldingTest, MaterializeReductionIndices) { TF_CHECK_OK(s.ToGraphDef(&item.graph)); item.fetch.push_back("reshape"); + auto input_t = GenerateRandomTensor(TensorShape({3, 4})); + Tensor indices_t(DT_INT32, TensorShape({2})); + indices_t.flat()(0) = 0; + indices_t.flat()(1) = 1; + auto tensors_expected = EvaluateNodes( + item.graph, item.fetch, {{"input", input_t}, {"indices", indices_t}}); + EXPECT_EQ(1, tensors_expected.size()); + ConstantFolding optimizer(nullptr /* cpu_device */); GraphDef output; Status status = optimizer.Optimize(nullptr, item, &output); @@ -1951,6 +1959,11 @@ TEST_F(ConstantFoldingTest, MaterializeReductionIndices) { } } EXPECT_EQ(3, found); + + auto tensors = EvaluateNodes(output, item.fetch, + {{"input", input_t}, {"indices", indices_t}}); + EXPECT_EQ(1, tensors.size()); + test::ExpectTensorNear(tensors_expected[0], tensors[0], 1e-5); } TEST_F(ConstantFoldingTest, LargeConstant) { @@ -2047,6 +2060,23 @@ TEST_F(ConstantFoldingTest, SwitchIdenticalInputs) { } } EXPECT_EQ(6, found); + + // Evaluate id_true when input tensor x is true. + Tensor x_t(DT_BOOL, TensorShape({})); + x_t.flat()(0) = true; + auto tensors_expected = EvaluateNodes(item.graph, {"id_true"}, {{"x", x_t}}); + EXPECT_EQ(1, tensors_expected.size()); + auto tensors = EvaluateNodes(output, {"id_true"}, {{"x", x_t}}); + EXPECT_EQ(1, tensors.size()); + test::ExpectTensorEqual(tensors_expected[0], tensors[0]); + + // Evalute id_false when input tensor is false. + x_t.flat()(0) = false; + tensors_expected = EvaluateNodes(item.graph, {"id_false"}, {{"x", x_t}}); + EXPECT_EQ(1, tensors_expected.size()); + tensors = EvaluateNodes(output, {"id_false"}, {{"x", x_t}}); + EXPECT_EQ(1, tensors.size()); + test::ExpectTensorEqual(tensors_expected[0], tensors[0]); } TEST_F(ConstantFoldingTest, PartialFolding_AssociativeAndCommutative) { @@ -2288,6 +2318,15 @@ TEST_F(ConstantFoldingTest, PartialFolding_IdentityN) { EXPECT_EQ("^id_n", node.input(0)); } } + + auto x_t = GenerateRandomTensor(TensorShape({})); + auto tensors_expected = EvaluateNodes(item.graph, item.fetch, {{"x", x_t}}); + EXPECT_EQ(4, tensors_expected.size()); + auto tensors = EvaluateNodes(output, item.fetch, {{"x", x_t}}); + EXPECT_EQ(4, tensors.size()); + for (int i = 0; i < tensors.size(); i++) { + test::ExpectTensorNear(tensors_expected[i], tensors[i], 1e-5); + } } TEST_F(ConstantFoldingTest, TrivialPack) { -- GitLab From 3f3a6e6685449130389e0b8d76f6ba0fc457bcfe Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 9 Apr 2018 22:01:59 -0700 Subject: [PATCH 077/791] Implementation of ArgMax PiperOrigin-RevId: 192236845 --- tensorflow/contrib/lite/builtin_op_data.h | 4 + tensorflow/contrib/lite/builtin_ops.h | 1 + tensorflow/contrib/lite/kernels/BUILD | 17 ++ tensorflow/contrib/lite/kernels/arg_max.cc | 178 ++++++++++++++++++ .../contrib/lite/kernels/arg_max_test.cc | 107 +++++++++++ tensorflow/contrib/lite/kernels/register.cc | 2 + tensorflow/contrib/lite/model.cc | 9 + tensorflow/contrib/lite/nnapi_delegate.cc | 1 + tensorflow/contrib/lite/schema/schema.fbs | 6 + .../contrib/lite/schema/schema_generated.h | 141 +++++++++++++- tensorflow/contrib/lite/testing/BUILD | 1 + .../contrib/lite/testing/generate_examples.py | 38 +++- .../testing/generated_examples_zip_test.cc | 6 + .../contrib/lite/toco/tflite/operator.cc | 19 ++ .../contrib/lite/toco/tflite/operator_test.cc | 7 + 15 files changed, 529 insertions(+), 8 deletions(-) create mode 100644 tensorflow/contrib/lite/kernels/arg_max.cc create mode 100644 tensorflow/contrib/lite/kernels/arg_max_test.cc diff --git a/tensorflow/contrib/lite/builtin_op_data.h b/tensorflow/contrib/lite/builtin_op_data.h index 2b6c24768c..f5fb2f15e3 100644 --- a/tensorflow/contrib/lite/builtin_op_data.h +++ b/tensorflow/contrib/lite/builtin_op_data.h @@ -221,6 +221,10 @@ typedef struct { int shrink_axis_mask; } TfLiteStridedSliceParams; +typedef struct { + TfLiteType output_type; +} TfLiteArgMaxParams; + #ifdef __cplusplus } // extern "C" #endif // __cplusplus diff --git a/tensorflow/contrib/lite/builtin_ops.h b/tensorflow/contrib/lite/builtin_ops.h index 17b791e4e2..e11c7fb2e4 100644 --- a/tensorflow/contrib/lite/builtin_ops.h +++ b/tensorflow/contrib/lite/builtin_ops.h @@ -80,6 +80,7 @@ typedef enum { kTfLiteBuiltinCast = 53, kTfLiteBuiltinPrelu = 54, kTfLiteBuiltinMaximum = 55, + kTfLiteBuiltinArgMax = 56, } TfLiteBuiltinOperator; #ifdef __cplusplus diff --git a/tensorflow/contrib/lite/kernels/BUILD b/tensorflow/contrib/lite/kernels/BUILD index df0f3cbeb0..b79900623e 100644 --- a/tensorflow/contrib/lite/kernels/BUILD +++ b/tensorflow/contrib/lite/kernels/BUILD @@ -135,6 +135,7 @@ cc_library( srcs = [ "activations.cc", "add.cc", + "arg_max.cc", "audio_spectrogram.cc", "basic_rnn.cc", "batch_to_space_nd.cc", @@ -270,6 +271,22 @@ tf_cc_test( ], ) +tf_cc_test( + name = "arg_max_test", + size = "small", + srcs = ["arg_max_test.cc"], + tags = [ + "tflite_not_portable_ios_arm64", + "tflite_not_portable_ios_x86_64", + ], + deps = [ + ":builtin_ops", + "//tensorflow/contrib/lite:framework", + "//tensorflow/contrib/lite/kernels:test_util", + "@com_google_googletest//:gtest", + ], +) + tf_cc_test( name = "div_test", size = "small", diff --git a/tensorflow/contrib/lite/kernels/arg_max.cc b/tensorflow/contrib/lite/kernels/arg_max.cc new file mode 100644 index 0000000000..a2c5e4cead --- /dev/null +++ b/tensorflow/contrib/lite/kernels/arg_max.cc @@ -0,0 +1,178 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/builtin_op_data.h" +#include "tensorflow/contrib/lite/context.h" +#include "tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h" +#include "tensorflow/contrib/lite/kernels/internal/quantization_util.h" +#include "tensorflow/contrib/lite/kernels/internal/tensor.h" +#include "tensorflow/contrib/lite/kernels/kernel_util.h" +#include "tensorflow/contrib/lite/kernels/op_macros.h" + +namespace tflite { +namespace ops { +namespace builtin { +namespace arg_max { + +constexpr int kInputTensor = 0; +constexpr int kAxis = 1; +constexpr int kOutputTensor = 0; + +TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + TF_LITE_ENSURE_EQ(context, NumInputs(node), 2); + TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); + + TfLiteTensor* input = GetInput(context, node, kInputTensor); + TfLiteTensor* axis = GetInput(context, node, kAxis); + // Make sure the axis is only 1 dimension. + TF_LITE_ENSURE_EQ(context, NumElements(axis), 1); + + // Make sure the axis is only either int32 or int64. + TF_LITE_ENSURE(context, + axis->type == kTfLiteInt32 || axis->type == kTfLiteInt64); + TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + + auto* params = reinterpret_cast(node->builtin_data); + switch (params->output_type) { + case kTfLiteInt32: + output->type = kTfLiteInt32; + break; + case kTfLiteInt64: + output->type = kTfLiteInt64; + break; + default: + context->ReportError(context, "Unknown index output data type"); + return kTfLiteError; + } + + // Check conditions for different types. + switch (input->type) { + case kTfLiteFloat32: + case kTfLiteUInt8: + case kTfLiteInt32: + break; + + default: + context->ReportError(context, "Only float32 and int types are supported"); + return kTfLiteError; + } + + // Copy the input dimensions to output except make the last dimension 1. + TF_LITE_ENSURE(context, NumDimensions(input) >= 1); + TfLiteIntArray* output_size = TfLiteIntArrayCopy(input->dims); + output_size->data[NumDimensions(input) - 1] = 1; + + return context->ResizeTensor(context, output, output_size); +} + +// The current impl actually ignores the axis argument. +// Only determine the index of the maximum value in the last dimension. +TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { + TfLiteTensor* input = GetInput(context, node, kInputTensor); + TfLiteTensor* axis = GetInput(context, node, kAxis); + TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + +#define TF_LITE_ARG_MAX(data_type, axis_type, output_type) \ + TF_LITE_ENSURE_EQ(context, GetTensorData(axis)[0], 3); \ + optimized_ops::ArgMax(GetTensorData(axis), \ + GetTensorData(input), GetTensorDims(input), \ + GetTensorData(output), \ + GetTensorDims(output)) + if (axis->type == kTfLiteInt32) { + switch (output->type) { + case kTfLiteInt32: { + switch (input->type) { + case kTfLiteFloat32: + TF_LITE_ARG_MAX(float, int32_t, int32_t); + break; + case kTfLiteUInt8: + TF_LITE_ARG_MAX(uint8_t, int32_t, int32_t); + break; + case kTfLiteInt32: + TF_LITE_ARG_MAX(int32_t, int32_t, int32_t); + break; + default: + return kTfLiteError; + } + } break; + case kTfLiteInt64: { + switch (input->type) { + case kTfLiteFloat32: + TF_LITE_ARG_MAX(float, int32_t, int64_t); + break; + case kTfLiteUInt8: + TF_LITE_ARG_MAX(uint8_t, int32_t, int64_t); + break; + case kTfLiteInt32: + TF_LITE_ARG_MAX(int32_t, int32_t, int64_t); + break; + default: + return kTfLiteError; + } + } break; + default: + return kTfLiteError; + } + } else { + switch (output->type) { + case kTfLiteInt32: { + switch (input->type) { + case kTfLiteFloat32: + TF_LITE_ARG_MAX(float, int64_t, int32_t); + break; + case kTfLiteUInt8: + TF_LITE_ARG_MAX(uint8_t, int64_t, int32_t); + break; + case kTfLiteInt32: + TF_LITE_ARG_MAX(int32_t, int64_t, int32_t); + break; + default: + return kTfLiteError; + } + } break; + case kTfLiteInt64: { + switch (input->type) { + case kTfLiteFloat32: + TF_LITE_ARG_MAX(float, int64_t, int64_t); + break; + case kTfLiteUInt8: + TF_LITE_ARG_MAX(uint8_t, int64_t, int64_t); + break; + case kTfLiteInt32: + TF_LITE_ARG_MAX(int32_t, int64_t, int64_t); + break; + default: + return kTfLiteError; + } + } break; + default: + return kTfLiteError; + } + } +#undef TF_LITE_ARG_MAX + + return kTfLiteOk; +} + +} // namespace arg_max + +TfLiteRegistration* Register_ARG_MAX() { + static TfLiteRegistration r = {nullptr, nullptr, arg_max::Prepare, + arg_max::Eval}; + return &r; +} + +} // namespace builtin +} // namespace ops +} // namespace tflite diff --git a/tensorflow/contrib/lite/kernels/arg_max_test.cc b/tensorflow/contrib/lite/kernels/arg_max_test.cc new file mode 100644 index 0000000000..f4e1da3a6e --- /dev/null +++ b/tensorflow/contrib/lite/kernels/arg_max_test.cc @@ -0,0 +1,107 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include "tensorflow/contrib/lite/interpreter.h" +#include "tensorflow/contrib/lite/kernels/register.h" +#include "tensorflow/contrib/lite/kernels/test_util.h" +#include "tensorflow/contrib/lite/model.h" + +namespace tflite { +namespace { + +using ::testing::ElementsAreArray; + +template +class ArgMaxOpModel : public SingleOpModel { + public: + ArgMaxOpModel(std::initializer_list input_shape, TensorType input_type, + TensorType output_type, TensorType index_output_type) { + input_ = AddInput(input_type); + axis_ = AddInput(TensorType_INT32); + output_ = AddOutput(output_type); + SetBuiltinOp(BuiltinOperator_ARG_MAX, BuiltinOptions_ArgMaxOptions, + CreateArgMaxOptions(builder_, index_output_type).Union()); + BuildInterpreter({input_shape, {1, 1, 1, 1}}); + } + + int input() { return input_; } + int axis() { return axis_; } + + std::vector GetOutput() { return ExtractVector(output_); } + std::vector GetOutputShape() { return GetTensorShape(output_); } + + private: + int input_; + int axis_; + int output_; +}; + +TEST(ArgMaxOpTest, GetMaxArgFloat) { + ArgMaxOpModel model({1, 1, 1, 4}, TensorType_FLOAT32, + TensorType_INT32, TensorType_INT32); + model.PopulateTensor(model.input(), {0.1, 0.9, 0.7, 0.3}); + // Currently only support the last dimension. + model.PopulateTensor(model.axis(), {3}); + model.Invoke(); + + EXPECT_THAT(model.GetOutput(), ElementsAreArray({1})); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 1, 1})); +} + +TEST(ArgMaxOpTest, GetMaxArgInt) { + ArgMaxOpModel model({1, 1, 1, 4}, TensorType_INT32, TensorType_INT32, + TensorType_INT32); + model.PopulateTensor(model.input(), {1, 9, 7, 3}); + // Currently only support the last dimension. + model.PopulateTensor(model.axis(), {3}); + model.Invoke(); + + EXPECT_THAT(model.GetOutput(), ElementsAreArray({1})); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 1, 1})); +} + +TEST(ArgMaxOpTest, GetMaxArgMulDimensions) { + ArgMaxOpModel model({1, 1, 2, 4}, TensorType_INT32, TensorType_INT32, + TensorType_INT32); + model.PopulateTensor(model.input(), {1, 2, 7, 8, 1, 9, 7, 3}); + // Currently only support the last dimension. + model.PopulateTensor(model.axis(), {3}); + model.Invoke(); + + EXPECT_THAT(model.GetOutput(), ElementsAreArray({3, 1})); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 2, 1})); +} + +TEST(ArgMaxOpTest, GetMaxArgOutput64) { + ArgMaxOpModel model({1, 1, 2, 4}, TensorType_INT32, TensorType_INT64, + TensorType_INT64); + model.PopulateTensor(model.input(), {10, 2, 7, 8, 1, 9, 7, 3}); + // Currently only support the last dimension. + model.PopulateTensor(model.axis(), {3}); + model.Invoke(); + + EXPECT_THAT(model.GetOutput(), ElementsAreArray({0, 1})); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 2, 1})); +} + +} // namespace +} // namespace tflite + +int main(int argc, char** argv) { + // On Linux, add: FLAGS_logtostderr = true; + FLAGS_logtostderr = true; + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tensorflow/contrib/lite/kernels/register.cc b/tensorflow/contrib/lite/kernels/register.cc index 0f98154b90..384e1afaa4 100644 --- a/tensorflow/contrib/lite/kernels/register.cc +++ b/tensorflow/contrib/lite/kernels/register.cc @@ -77,6 +77,7 @@ TfLiteRegistration* Register_CAST(); TfLiteRegistration* Register_DEQUANTIZE(); TfLiteRegistration* Register_PRELU(); TfLiteRegistration* Register_MAXIMUM(); +TfLiteRegistration* Register_ARG_MAX(); BuiltinOpResolver::BuiltinOpResolver() { AddBuiltin(BuiltinOperator_RELU, Register_RELU()); @@ -135,6 +136,7 @@ BuiltinOpResolver::BuiltinOpResolver() { AddBuiltin(BuiltinOperator_DEQUANTIZE, Register_DEQUANTIZE()); AddBuiltin(BuiltinOperator_PRELU, Register_PRELU()); AddBuiltin(BuiltinOperator_MAXIMUM, Register_MAXIMUM()); + AddBuiltin(BuiltinOperator_ARG_MAX, Register_ARG_MAX()); // TODO(andrewharp, ahentz): Move these somewhere more appropriate so that // custom ops aren't always included by default. diff --git a/tensorflow/contrib/lite/model.cc b/tensorflow/contrib/lite/model.cc index 3448de68e8..921c139e30 100644 --- a/tensorflow/contrib/lite/model.cc +++ b/tensorflow/contrib/lite/model.cc @@ -653,6 +653,15 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, case BuiltinOperator_MAXIMUM: { break; } + case BuiltinOperator_ARG_MAX: { + auto* params = MallocPOD(); + if (auto* schema_params = op->builtin_options_as_ArgMaxOptions()) { + ConvertTensorType(schema_params->output_type(), ¶ms->output_type, + error_reporter); + } + builtin_data = reinterpret_cast(params); + break; + } case BuiltinOperator_DELEGATE: { // TODO(ycling): Revisit when supporting saving delegated models. error_reporter->Report("DELEGATE op shouldn't exist in model."); diff --git a/tensorflow/contrib/lite/nnapi_delegate.cc b/tensorflow/contrib/lite/nnapi_delegate.cc index bc13444dc7..04d53d955a 100644 --- a/tensorflow/contrib/lite/nnapi_delegate.cc +++ b/tensorflow/contrib/lite/nnapi_delegate.cc @@ -351,6 +351,7 @@ void AddOpsAndParams(tflite::Interpreter* interpreter, case tflite::BuiltinOperator_CAST: case tflite::BuiltinOperator_PRELU: case tflite::BuiltinOperator_MAXIMUM: + case tflite::BuiltinOperator_ARG_MAX: FATAL("Op code %d is currently not delegated to NNAPI", builtin); nn_op_type = -1; // set to invalid break; diff --git a/tensorflow/contrib/lite/schema/schema.fbs b/tensorflow/contrib/lite/schema/schema.fbs index c63bfb28cc..238a406af5 100644 --- a/tensorflow/contrib/lite/schema/schema.fbs +++ b/tensorflow/contrib/lite/schema/schema.fbs @@ -132,6 +132,7 @@ enum BuiltinOperator : byte { CAST = 53, PRELU = 54, MAXIMUM = 55, + ARG_MAX = 56, } // Options for the builtin operators. @@ -175,6 +176,7 @@ union BuiltinOptions { CastOptions, DequantizeOptions, MaximumOptions, + ArgMaxOptions, } enum Padding : byte { SAME, VALID } @@ -391,6 +393,10 @@ table DequantizeOptions { table MaximumOptions { } +table ArgMaxOptions { + output_type : TensorType; +} + // An OperatorCode can be an enum value (BuiltinOperator) if the operator is a // builtin, or a string if the operator is custom. table OperatorCode { diff --git a/tensorflow/contrib/lite/schema/schema_generated.h b/tensorflow/contrib/lite/schema/schema_generated.h index 0735be5c8f..8b355b0dc6 100755 --- a/tensorflow/contrib/lite/schema/schema_generated.h +++ b/tensorflow/contrib/lite/schema/schema_generated.h @@ -148,6 +148,9 @@ struct DequantizeOptionsT; struct MaximumOptions; struct MaximumOptionsT; +struct ArgMaxOptions; +struct ArgMaxOptionsT; + struct OperatorCode; struct OperatorCodeT; @@ -259,11 +262,12 @@ enum BuiltinOperator { BuiltinOperator_CAST = 53, BuiltinOperator_PRELU = 54, BuiltinOperator_MAXIMUM = 55, + BuiltinOperator_ARG_MAX = 56, BuiltinOperator_MIN = BuiltinOperator_ADD, - BuiltinOperator_MAX = BuiltinOperator_MAXIMUM + BuiltinOperator_MAX = BuiltinOperator_ARG_MAX }; -inline BuiltinOperator (&EnumValuesBuiltinOperator())[54] { +inline BuiltinOperator (&EnumValuesBuiltinOperator())[55] { static BuiltinOperator values[] = { BuiltinOperator_ADD, BuiltinOperator_AVERAGE_POOL_2D, @@ -318,7 +322,8 @@ inline BuiltinOperator (&EnumValuesBuiltinOperator())[54] { BuiltinOperator_BIDIRECTIONAL_SEQUENCE_LSTM, BuiltinOperator_CAST, BuiltinOperator_PRELU, - BuiltinOperator_MAXIMUM + BuiltinOperator_MAXIMUM, + BuiltinOperator_ARG_MAX }; return values; } @@ -381,6 +386,7 @@ inline const char **EnumNamesBuiltinOperator() { "CAST", "PRELU", "MAXIMUM", + "ARG_MAX", nullptr }; return names; @@ -432,11 +438,12 @@ enum BuiltinOptions { BuiltinOptions_CastOptions = 37, BuiltinOptions_DequantizeOptions = 38, BuiltinOptions_MaximumOptions = 39, + BuiltinOptions_ArgMaxOptions = 40, BuiltinOptions_MIN = BuiltinOptions_NONE, - BuiltinOptions_MAX = BuiltinOptions_MaximumOptions + BuiltinOptions_MAX = BuiltinOptions_ArgMaxOptions }; -inline BuiltinOptions (&EnumValuesBuiltinOptions())[40] { +inline BuiltinOptions (&EnumValuesBuiltinOptions())[41] { static BuiltinOptions values[] = { BuiltinOptions_NONE, BuiltinOptions_Conv2DOptions, @@ -477,7 +484,8 @@ inline BuiltinOptions (&EnumValuesBuiltinOptions())[40] { BuiltinOptions_LogSoftmaxOptions, BuiltinOptions_CastOptions, BuiltinOptions_DequantizeOptions, - BuiltinOptions_MaximumOptions + BuiltinOptions_MaximumOptions, + BuiltinOptions_ArgMaxOptions }; return values; } @@ -524,6 +532,7 @@ inline const char **EnumNamesBuiltinOptions() { "CastOptions", "DequantizeOptions", "MaximumOptions", + "ArgMaxOptions", nullptr }; return names; @@ -694,6 +703,10 @@ template<> struct BuiltinOptionsTraits { static const BuiltinOptions enum_value = BuiltinOptions_MaximumOptions; }; +template<> struct BuiltinOptionsTraits { + static const BuiltinOptions enum_value = BuiltinOptions_ArgMaxOptions; +}; + struct BuiltinOptionsUnion { BuiltinOptions type; void *value; @@ -1037,6 +1050,14 @@ struct BuiltinOptionsUnion { return type == BuiltinOptions_MaximumOptions ? reinterpret_cast(value) : nullptr; } + ArgMaxOptionsT *AsArgMaxOptions() { + return type == BuiltinOptions_ArgMaxOptions ? + reinterpret_cast(value) : nullptr; + } + const ArgMaxOptionsT *AsArgMaxOptions() const { + return type == BuiltinOptions_ArgMaxOptions ? + reinterpret_cast(value) : nullptr; + } }; bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *obj, BuiltinOptions type); @@ -3846,6 +3867,60 @@ inline flatbuffers::Offset CreateMaximumOptions( flatbuffers::Offset CreateMaximumOptions(flatbuffers::FlatBufferBuilder &_fbb, const MaximumOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +struct ArgMaxOptionsT : public flatbuffers::NativeTable { + typedef ArgMaxOptions TableType; + TensorType output_type; + ArgMaxOptionsT() + : output_type(TensorType_FLOAT32) { + } +}; + +struct ArgMaxOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + typedef ArgMaxOptionsT NativeTableType; + enum { + VT_OUTPUT_TYPE = 4 + }; + TensorType output_type() const { + return static_cast(GetField(VT_OUTPUT_TYPE, 0)); + } + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyField(verifier, VT_OUTPUT_TYPE) && + verifier.EndTable(); + } + ArgMaxOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(ArgMaxOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const; + static flatbuffers::Offset Pack(flatbuffers::FlatBufferBuilder &_fbb, const ArgMaxOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +}; + +struct ArgMaxOptionsBuilder { + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + void add_output_type(TensorType output_type) { + fbb_.AddElement(ArgMaxOptions::VT_OUTPUT_TYPE, static_cast(output_type), 0); + } + explicit ArgMaxOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + ArgMaxOptionsBuilder &operator=(const ArgMaxOptionsBuilder &); + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + return o; + } +}; + +inline flatbuffers::Offset CreateArgMaxOptions( + flatbuffers::FlatBufferBuilder &_fbb, + TensorType output_type = TensorType_FLOAT32) { + ArgMaxOptionsBuilder builder_(_fbb); + builder_.add_output_type(output_type); + return builder_.Finish(); +} + +flatbuffers::Offset CreateArgMaxOptions(flatbuffers::FlatBufferBuilder &_fbb, const ArgMaxOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); + struct OperatorCodeT : public flatbuffers::NativeTable { typedef OperatorCode TableType; BuiltinOperator builtin_code; @@ -4080,6 +4155,9 @@ struct Operator FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { const MaximumOptions *builtin_options_as_MaximumOptions() const { return builtin_options_type() == BuiltinOptions_MaximumOptions ? static_cast(builtin_options()) : nullptr; } + const ArgMaxOptions *builtin_options_as_ArgMaxOptions() const { + return builtin_options_type() == BuiltinOptions_ArgMaxOptions ? static_cast(builtin_options()) : nullptr; + } const flatbuffers::Vector *custom_options() const { return GetPointer *>(VT_CUSTOM_OPTIONS); } @@ -4262,6 +4340,10 @@ template<> inline const MaximumOptions *Operator::builtin_options_as inline const ArgMaxOptions *Operator::builtin_options_as() const { + return builtin_options_as_ArgMaxOptions(); +} + struct OperatorBuilder { flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; @@ -5819,6 +5901,32 @@ inline flatbuffers::Offset CreateMaximumOptions(flatbuffers::Fla _fbb); } +inline ArgMaxOptionsT *ArgMaxOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { + auto _o = new ArgMaxOptionsT(); + UnPackTo(_o, _resolver); + return _o; +} + +inline void ArgMaxOptions::UnPackTo(ArgMaxOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { + (void)_o; + (void)_resolver; + { auto _e = output_type(); _o->output_type = _e; }; +} + +inline flatbuffers::Offset ArgMaxOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const ArgMaxOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) { + return CreateArgMaxOptions(_fbb, _o, _rehasher); +} + +inline flatbuffers::Offset CreateArgMaxOptions(flatbuffers::FlatBufferBuilder &_fbb, const ArgMaxOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) { + (void)_rehasher; + (void)_o; + struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const ArgMaxOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + auto _output_type = _o->output_type; + return tflite::CreateArgMaxOptions( + _fbb, + _output_type); +} + inline OperatorCodeT *OperatorCode::UnPack(const flatbuffers::resolver_function_t *_resolver) const { auto _o = new OperatorCodeT(); UnPackTo(_o, _resolver); @@ -6155,6 +6263,10 @@ inline bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *ob auto ptr = reinterpret_cast(obj); return verifier.VerifyTable(ptr); } + case BuiltinOptions_ArgMaxOptions: { + auto ptr = reinterpret_cast(obj); + return verifier.VerifyTable(ptr); + } default: return false; } } @@ -6329,6 +6441,10 @@ inline void *BuiltinOptionsUnion::UnPack(const void *obj, BuiltinOptions type, c auto ptr = reinterpret_cast(obj); return ptr->UnPack(resolver); } + case BuiltinOptions_ArgMaxOptions: { + auto ptr = reinterpret_cast(obj); + return ptr->UnPack(resolver); + } default: return nullptr; } } @@ -6491,6 +6607,10 @@ inline flatbuffers::Offset BuiltinOptionsUnion::Pack(flatbuffers::FlatBuff auto ptr = reinterpret_cast(value); return CreateMaximumOptions(_fbb, ptr, _rehasher).Union(); } + case BuiltinOptions_ArgMaxOptions: { + auto ptr = reinterpret_cast(value); + return CreateArgMaxOptions(_fbb, ptr, _rehasher).Union(); + } default: return 0; } } @@ -6653,6 +6773,10 @@ inline BuiltinOptionsUnion::BuiltinOptionsUnion(const BuiltinOptionsUnion &u) FL value = new MaximumOptionsT(*reinterpret_cast(u.value)); break; } + case BuiltinOptions_ArgMaxOptions: { + value = new ArgMaxOptionsT(*reinterpret_cast(u.value)); + break; + } default: break; } @@ -6855,6 +6979,11 @@ inline void BuiltinOptionsUnion::Reset() { delete ptr; break; } + case BuiltinOptions_ArgMaxOptions: { + auto ptr = reinterpret_cast(value); + delete ptr; + break; + } default: break; } value = nullptr; diff --git a/tensorflow/contrib/lite/testing/BUILD b/tensorflow/contrib/lite/testing/BUILD index 62f20638ba..386cfdb524 100644 --- a/tensorflow/contrib/lite/testing/BUILD +++ b/tensorflow/contrib/lite/testing/BUILD @@ -18,6 +18,7 @@ gen_zipped_test_files( name = "optest", files = [ "add.zip", + "arg_max.zip", "avg_pool.zip", "batch_to_space_nd.zip", "concat.zip", diff --git a/tensorflow/contrib/lite/testing/generate_examples.py b/tensorflow/contrib/lite/testing/generate_examples.py index f919517e93..42aa92c1bb 100644 --- a/tensorflow/contrib/lite/testing/generate_examples.py +++ b/tensorflow/contrib/lite/testing/generate_examples.py @@ -104,6 +104,10 @@ KNOWN_BUGS = { r"strided_slice.*begin=\[0\].*end=\[1\].*": "73170889", # No support for SplitV r"split.*num_or_size_splits=\[2,2\]": "73377559", + # Needs support for dimensions other than the last one in argmax. + r"arg_max.*axis=0.*": "77546240", + r"arg_max.*axis=1.*": "77546240", + r"arg_max.*axis=2.*": "77546240", } @@ -1954,7 +1958,7 @@ def make_l2_pool(input_tensor, ksize, strides, padding, data_format): def make_topk_tests(zip_path): - """Make a set of tests to do gather.""" + """Make a set of tests to do topk.""" test_parameters = [{ "input_dtype": [tf.float32, tf.int32], @@ -1962,7 +1966,7 @@ def make_topk_tests(zip_path): }] def build_graph(parameters): - """Build the gather op testing graph.""" + """Build the topk op testing graph.""" input_value = tf.placeholder( dtype=parameters["input_dtype"], name="input", @@ -1979,6 +1983,36 @@ def make_topk_tests(zip_path): make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs) + +def make_arg_max_tests(zip_path): + """Make a set of tests to do arg_max.""" + + test_parameters = [{ + "input_dtype": [tf.float32, tf.int32], + "input_shape": [[1, 1, 1, 3], [2, 3, 4, 5], [2, 3, 3], [5, 5], [10]], + "axis": [0, 1, 2, 3], + "output_type": [tf.int32, tf.int64], + }] + + def build_graph(parameters): + """Build the topk op testing graph.""" + input_value = tf.placeholder( + dtype=parameters["input_dtype"], + name="input", + shape=parameters["input_shape"]) + axis = tf.constant(parameters["axis"], name="axis") + out = tf.arg_max(input_value, axis, output_type=parameters["output_type"]) + return [input_value], [out] + + def build_inputs(parameters, sess, inputs, outputs): + input_value = create_tensor_data(parameters["input_dtype"], + parameters["input_shape"]) + return [input_value], sess.run( + outputs, feed_dict=dict(zip(inputs, [input_value]))) + + make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs) + + # Toco binary path provided by the generate rule. bin_path = None diff --git a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc index 6697b86e79..291c974545 100644 --- a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc +++ b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc @@ -94,6 +94,11 @@ std::map kBrokenTests = { // No support for axis!=0 in GatherV2. {R"(^\/gather.*axis=1)", "76910444"}, + + // No support for arbitrary dimensions in ArgMax. + {R"(^\/arg_max.*axis=0)", "77546240"}, + {R"(^\/arg_max.*axis=1)", "77546240"}, + {R"(^\/arg_max.*axis=2)", "77546240"}, }; // Allows test data to be unzipped into a temporary directory and makes @@ -236,6 +241,7 @@ TEST_P(OpsTest, RunStuff) { ::testing::ValuesIn(UnarchiveZipAndFindTestNames(#zip_base ".zip"))); INSTANTIATE_TESTS(add) +INSTANTIATE_TESTS(arg_max) INSTANTIATE_TESTS(avg_pool) INSTANTIATE_TESTS(batch_to_space_nd) INSTANTIATE_TESTS(concat) diff --git a/tensorflow/contrib/lite/toco/tflite/operator.cc b/tensorflow/contrib/lite/toco/tflite/operator.cc index f991529569..4df16827b4 100644 --- a/tensorflow/contrib/lite/toco/tflite/operator.cc +++ b/tensorflow/contrib/lite/toco/tflite/operator.cc @@ -662,6 +662,23 @@ class TopK_V2 : public BuiltinOperator { + public: + using BuiltinOperator::BuiltinOperator; + flatbuffers::Offset WriteOptions( + const TocoOperator& op, + flatbuffers::FlatBufferBuilder* builder) const override { + return ::tflite::CreateArgMaxOptions( + *builder, DataType::Serialize(op.output_data_type)); + } + + void ReadOptions(const TfLiteOptions& options, + TocoOperator* op) const override { + op->output_data_type = DataType::Deserialize(options.output_type()); + } +}; + class TensorFlowUnsupported : public BaseOperator { public: using BaseOperator::BaseOperator; @@ -834,6 +851,8 @@ std::vector> BuildOperatorList() { new Lstm(::tflite::BuiltinOperator_LSTM, OperatorType::kLstmCell)); ops.emplace_back( new Cast(::tflite::BuiltinOperator_CAST, OperatorType::kCast)); + ops.emplace_back( + new ArgMax(::tflite::BuiltinOperator_ARG_MAX, OperatorType::kArgMax)); // Custom Operators. ops.emplace_back( diff --git a/tensorflow/contrib/lite/toco/tflite/operator_test.cc b/tensorflow/contrib/lite/toco/tflite/operator_test.cc index 4783843b7f..5546bda696 100644 --- a/tensorflow/contrib/lite/toco/tflite/operator_test.cc +++ b/tensorflow/contrib/lite/toco/tflite/operator_test.cc @@ -391,6 +391,13 @@ TEST_F(OperatorTest, BuiltinTopKV2) { ASSERT_NE(nullptr, output_toco_op.get()); } +TEST_F(OperatorTest, BuiltinArgMax) { + ArgMaxOperator op; + auto output_toco_op = SerializeAndDeserialize( + GetOperator("ARG_MAX", OperatorType::kArgMax), op); + EXPECT_EQ(op.output_data_type, output_toco_op->output_data_type); +} + TEST_F(OperatorTest, TensorFlowUnsupported) { TensorFlowUnsupportedOperator op; op.tensorflow_op = "MyCustomUnsupportedOp"; -- GitLab From 022d1fe0ecb46a2a7b77f8b99de8c273ef804c82 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 9 Apr 2018 22:04:04 -0700 Subject: [PATCH 078/791] [XLA] Redesign: implement XlaBuilder::IsConstant, XlaBuidler::BuildConstantSubGraph, and Client::ComputeConstant(XlaComputation...). - Since the builder no longer holds a client, we moved the ComputeConstant to the client side so that it can communicate with the service side. Now we add XlaBuilder::BuildConstantSubGraph, which is only responsible for building a subgraph that is compile-time constant. - Before this change, every XlaBuilder has a unique id. Now since it also builds constant subgraph, we give every XlaComputation being built a global unique id, and uniquify instruction names when actually building the XlaComputation. PiperOrigin-RevId: 192236997 --- tensorflow/compiler/xla/client/client.cc | 28 +++ tensorflow/compiler/xla/client/client.h | 21 ++ .../compiler/xla/client/xla_client/BUILD | 1 + .../xla/client/xla_client/xla_builder.cc | 183 +++++++++++++++--- .../xla/client/xla_client/xla_builder.h | 69 +++---- tensorflow/compiler/xla/service/service.cc | 44 +++++ tensorflow/compiler/xla/service/service.h | 3 + tensorflow/compiler/xla/service_interface.h | 4 + tensorflow/compiler/xla/tests/BUILD | 2 + .../xla/tests/compute_constant_test.cc | 100 +++++----- tensorflow/compiler/xla/xla.proto | 5 + 11 files changed, 345 insertions(+), 115 deletions(-) diff --git a/tensorflow/compiler/xla/client/client.cc b/tensorflow/compiler/xla/client/client.cc index 3f45167fcb..f0f94298a0 100644 --- a/tensorflow/compiler/xla/client/client.cc +++ b/tensorflow/compiler/xla/client/client.cc @@ -193,6 +193,34 @@ StatusOr> Client::ExecuteAndTransfer( return Transfer(*data, shape_with_output_layout); } +StatusOr> Client::ComputeConstant( + const XlaComputation& computation, const Layout* output_layout) const { + ComputeConstantGraphRequest request; + *request.mutable_computation() = computation.proto(); + if (output_layout != nullptr) { + *request.mutable_output_layout() = *output_layout; + } + + ComputeConstantResponse response; + + VLOG(2) << "making compute-constant-graph request"; + Status s = stub_->ComputeConstantGraph(&request, &response); + VLOG(2) << "done with request"; + + if (!s.ok()) { + return s; + } + + VLOG(3) << "ComputeConstant: {" << response.DebugString() << "}"; + + if (!response.has_literal()) { + return InternalError( + "no computed literal in the provided response in ComputeConstantGraph " + "request"); + } + return Literal::CreateFromProto(response.literal()); +} + StatusOr Client::LoadSnapshot(const SessionModule& module) { LoadComputationSnapshotRequest request; *request.mutable_module() = module; diff --git a/tensorflow/compiler/xla/client/client.h b/tensorflow/compiler/xla/client/client.h index 05d707dab1..14c685d94e 100644 --- a/tensorflow/compiler/xla/client/client.h +++ b/tensorflow/compiler/xla/client/client.h @@ -194,6 +194,27 @@ class Client { const ExecutionOptions* execution_options = nullptr, ExecutionProfile* execution_profile = nullptr); + // Computes the value of the given computation using a non-optimized + // interpreter on the host. + // + // The computation must not depend on any parameters, or on stateful operators + // such as `RngNormal` or `Infeed`. + // + // This functionality can be useful when translating a computation into XLA + // where something that looked dynamic is required by XLA to be specified as a + // constant. E.g. the source computation (outside of XLA) may include a + // dynamic computation of the shape of something and ComputeConstant lets you + // determine what the value of that computation is in the case where the value + // can be determined at compile time. + // + // If output_layout is non-null, then the output of the computation will be + // stored using that layout. + // + // TODO(b/74197823): This is a part of a NOT YET ready refactor. + StatusOr> ComputeConstant( + const XlaComputation& computation, + const Layout* output_layout = nullptr) const; + // Unregister the memory for the given GlobalData on the device. Status Unregister(const GlobalData& data); diff --git a/tensorflow/compiler/xla/client/xla_client/BUILD b/tensorflow/compiler/xla/client/xla_client/BUILD index b1dba16856..31fa1241ee 100644 --- a/tensorflow/compiler/xla/client/xla_client/BUILD +++ b/tensorflow/compiler/xla/client/xla_client/BUILD @@ -44,6 +44,7 @@ cc_library( hdrs = ["xla_builder.h"], deps = [ ":xla_computation", + "//tensorflow/compiler/xla:execution_options_util", "//tensorflow/compiler/xla:literal_util", "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:status_macros", diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc index 170dd59c79..a01be28881 100644 --- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc +++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc @@ -17,12 +17,15 @@ limitations under the License. #include #include +#include #include #include +#include "tensorflow/compiler/xla/execution_options_util.h" #include "tensorflow/compiler/xla/service/hlo_opcode.h" #include "tensorflow/compiler/xla/service/shape_inference.h" #include "tensorflow/compiler/xla/util.h" +#include "tensorflow/core/lib/gtl/flatset.h" #include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/mutex.h" @@ -82,7 +85,7 @@ StatusOr XlaOp::GetShape() const { } XlaBuilder::XlaBuilder(const string& computation_name) - : name_(computation_name), unique_id_(GetUniqueId()) {} + : name_(computation_name) {} XlaBuilder::~XlaBuilder() {} @@ -111,10 +114,11 @@ XlaOp XlaBuilder::NoteErrorOrReturn( return op.ConsumeValueOrDie(); } -StatusOr XlaBuilder::GetProgramShape(int64* root_id) { +StatusOr XlaBuilder::GetProgramShape(int64* root_id) const { TF_RETURN_IF_ERROR(first_error_); TF_RET_CHECK(root_id != nullptr); + ProgramShape program_shape; // Not all instructions can be roots. Walk backwards from the last added @@ -155,9 +159,56 @@ StatusOr XlaBuilder::GetProgramShape(int64* root_id) { return program_shape; } -StatusOr XlaBuilder::GetProgramShape() { - int64 root_id; - return GetProgramShape(&root_id); +StatusOr XlaBuilder::GetProgramShape() const { + int64 root; + return GetProgramShape(&root); +} + +void XlaBuilder::IsConstantVisitor(const int64 op_handle, + std::set* visited, + bool* is_constant) const { + if (visited->count(op_handle) != 0 || !*is_constant) { + return; + } + + CHECK(op_handle < instructions_.size() && op_handle >= 0); + + const HloInstructionProto& instr = instructions_[op_handle]; + const HloOpcode opcode = StringToHloOpcode(instr.opcode()).ValueOrDie(); + switch (opcode) { + default: + for (const int64 operand_id : instr.operand_ids()) { + IsConstantVisitor(operand_id, visited, is_constant); + } + // TODO(b/32495713): We aren't checking the called computations. + break; + + // Non functional ops. + case HloOpcode::kRng: + case HloOpcode::kCrossReplicaSum: + // TODO(b/33009255): Implmement constant folding for cross replica sum. + case HloOpcode::kInfeed: + case HloOpcode::kOutfeed: + case HloOpcode::kHostCompute: + case HloOpcode::kCall: + // TODO(b/32495713): We aren't checking the to_apply computation itself, + // so we conservatively say that computations containing the Call op + // cannot be constant. We cannot set is_functional=false in other similar + // cases since we're already relying on IsConstant to return true. + case HloOpcode::kCustomCall: + case HloOpcode::kWhile: + // TODO(b/32495713): We aren't checking the condition and body + // computations themselves. + case HloOpcode::kSend: + case HloOpcode::kRecv: + case HloOpcode::kParameter: + *is_constant = false; + break; + } + if (!*is_constant) { + VLOG(1) << "Non-constant: " << instr.name(); + } + visited->insert(op_handle); } XlaComputation XlaBuilder::BuildAndNoteError() { @@ -180,21 +231,24 @@ StatusOr XlaBuilder::Build() { } HloComputationProto entry; + entry.set_id(GetUniqueId()); // Give the computation a global unique id. + entry.set_name(StrCat(name_, entry.id())); // Ensure that the name is unique. { int64 root_id; - ProgramShape program_shape; - TF_ASSIGN_OR_RETURN(program_shape, GetProgramShape(&root_id)); - entry.mutable_program_shape()->Swap(&program_shape); + TF_ASSIGN_OR_RETURN(*entry.mutable_program_shape(), + GetProgramShape(&root_id)); entry.set_root_id(root_id); } for (auto& instruction : instructions_) { + // Ensures that the instruction names are unique among the whole graph. + const string& new_name = + StrCat(instruction.name(), ".", entry.id(), ".", instruction.id()); + instruction.set_name(new_name); entry.add_instructions()->Swap(&instruction); } - entry.set_id(unique_id_); - entry.set_name(StrCat(name_, entry.id())); // Ensure that the name is unique. XlaComputation computation(entry.id()); HloModuleProto* module = computation.mutable_proto(); module->set_name(entry.name()); @@ -417,11 +471,10 @@ XlaOp XlaBuilder::Parameter(int64 parameter_number, const Shape& shape, const string& name) { return NoteErrorOrReturn([&]() -> StatusOr { HloInstructionProto instr; - if (parameter_numbers_.find(parameter_number) != parameter_numbers_.end()) { + if (!parameter_numbers_.insert(parameter_number).second) { return InvalidArgument("parameter %lld already registered", parameter_number); } - parameter_numbers_.insert(parameter_number); instr.set_parameter_number(parameter_number); instr.set_name(name); *instr.mutable_shape() = shape; @@ -1262,15 +1315,98 @@ XlaOp XlaBuilder::Recv(const Shape& shape, const ChannelHandle& handle) { }); } -StatusOr XlaBuilder::IsConstant(const XlaOp& operand, - int64 num_parameters) { - return Unimplemented("IsConstant is not implemented."); +StatusOr XlaBuilder::IsConstant(const XlaOp& operand) const { + TF_RETURN_IF_ERROR(first_error_); + + // Verify that the handle is valid. + TF_RETURN_IF_ERROR(LookUpInstruction(operand).status()); + + bool is_constant = true; + std::set visited; + IsConstantVisitor(operand.handle(), &visited, &is_constant); + return is_constant; } -StatusOr> XlaBuilder::ComputeConstant( - const XlaOp& operand, const Layout* output_layout, - tensorflow::gtl::ArraySlice parameters) { - return Unimplemented("ComputeConstant is not implemented"); +StatusOr XlaBuilder::BuildConstantSubGraph( + const XlaOp& root_op) const { + TF_ASSIGN_OR_RETURN(bool is_constant, IsConstant(root_op)); + if (!is_constant) { + auto op_status = LookUpInstruction(root_op); + string op_string = + op_status.ok() ? op_status.ValueOrDie()->name() : ""; + return InvalidArgument( + "Operand to BuildConstantSubGraph depends on a parameter.\n\n" + " op requested for constant subgraph: %s\n\n" + "This is an internal error that typically happens when the XLA user " + "(e.g. TensorFlow) is attempting to determine a value that must be a " + "compile-time constant (e.g. an array dimension) but it is not capable " + "of being evaluated at XLA compile time.\n\n" + "Please file a usability bug with the framework being used (e.g. " + "TensorFlow).", + op_string.c_str()); + } + + TF_ASSIGN_OR_RETURN(const HloInstructionProto* root, + LookUpInstruction(root_op)); + TF_ASSIGN_OR_RETURN(HloOpcode opcode, StringToHloOpcode(root->opcode())); + if (!CanBeRoot(opcode)) { + return InvalidArgument("the operand with opcode %s cannot be root", + root->opcode().c_str()); + } + + HloComputationProto entry; + entry.set_id(GetUniqueId()); // Give the computation a global unique id. + entry.set_name(StrCat(name_, entry.id(), "_compute_constant")); + entry.set_root_id(root->id()); + ProgramShape* program_shape = entry.mutable_program_shape(); + *program_shape->mutable_result() = root->shape(); + + // We use std::set to keep the instruction ids in ascending order (which is + // also a valid denpendency order). The related ops will be added to the + // subgraph in the same order. + std::set related_ops; + tensorflow::gtl::FlatSet related_calls; // Related computations. + std::queue worklist; + worklist.push(root->id()); + related_ops.insert(root->id()); + while (!worklist.empty()) { + int64 node = worklist.front(); + worklist.pop(); + for (int64 id : instructions_[node].operand_ids()) { + if (related_ops.insert(id).second) { + worklist.push(id); + } + } + for (int64 called_id : instructions_[node].called_computation_ids()) { + related_calls.insert(called_id); + } + } + + // Add related ops to the computation. + for (int64 id : related_ops) { + auto* instr = entry.add_instructions(); + *instr = instructions_[id]; + // Ensures that the instruction names are unique among the graph. + const string& new_name = + StrCat(instr->name(), ".", entry.id(), ".", instr->id()); + instr->set_name(new_name); + } + + XlaComputation computation(entry.id()); + HloModuleProto* module = computation.mutable_proto(); + module->set_name(entry.name()); + module->set_id(entry.id()); + module->set_entry_computation_name(entry.name()); + module->set_entry_computation_id(entry.id()); + *module->mutable_program_shape() = *program_shape; + for (auto& e : embedded_) { + if (related_calls.find(e.second.id()) != related_calls.end()) { + *module->add_computations() = e.second; + } + } + *module->add_computations() = std::move(entry); + + return std::move(computation); } std::unique_ptr XlaBuilder::CreateSubBuilder( @@ -1281,10 +1417,6 @@ std::unique_ptr XlaBuilder::CreateSubBuilder( return sub_builder; } -Status XlaBuilder::SetReturnValue(const XlaOp& operand) { - return Unimplemented("SetReturnValue is not implemented."); -} - /* static */ ConvolutionDimensionNumbers XlaBuilder::CreateDefaultConvDimensionNumbers(int num_spatial_dims) { ConvolutionDimensionNumbers dimension_numbers; @@ -1364,10 +1496,7 @@ StatusOr XlaBuilder::AddInstruction( instr.set_id(handle); instr.set_opcode(HloOpcodeString(opcode)); if (instr.name().empty()) { - instr.set_name(StrCat(instr.opcode(), ".", unique_id_, ".", handle)); - } else { - // Append the handle to make sure the name is unique. - instr.set_name(StrCat(instr.name(), ".", unique_id_, ".", handle)); + instr.set_name(StrCat(instr.opcode())); } for (const auto& operand : operands) { if (operand.builder_ == nullptr) { diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.h b/tensorflow/compiler/xla/client/xla_client/xla_builder.h index 0673b86646..d747691f16 100644 --- a/tensorflow/compiler/xla/client/xla_client/xla_builder.h +++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.h @@ -687,11 +687,12 @@ class XlaBuilder { XlaOp Recv(const Shape& shape, const ChannelHandle& handle); // Returns true if 'operand' is a compile-time constant. A compile-time - // constant does not depend on parameters with index greater than or equal to - // `num_parameters`, or on stateful operators such as `RngNormal` or `Infeed`. - // Unlike `ComputeConstant`, `IsConstant` tests whether a computation is a - // compile-time constant without evaluating the computation. - StatusOr IsConstant(const XlaOp& operand, int64 num_parameters = 0); + // constant does not depend on any parameters, or on stateful operators such + // as `RngNormal` or `Infeed`. + // + // This tests whether a computation is a compile-time constant without + // evaluating the computation. + StatusOr IsConstant(const XlaOp& operand) const; // Normalizes operand across spatial and batch dimensions for each feature. // @@ -731,47 +732,14 @@ class XlaBuilder { const XlaOp& grad_output, float epsilon, int64 feature_index); - // Computes the value of a constant indicated by a XlaOp using a non-optimized - // interpreter on the host. - // - // The operand must represent a constant value, which in this case - // means that it must not statically depend on any parameter of the - // computation that is being built other then the ones specified on the - // parameter list. The parameters in the list will be indexed by their - // parameter id property so the number of parameters specified should be at - // least as many as the largest used parameter index. - // - // `IsConstant` can be used to test whether a computation is a compile-time - // constant without evaluation it. `ComputeConstant` only succeeds for - // computations where `IsConstant` returns true. - // - // This functionality can be useful when translating a computation - // into XLA where something that looked dynamic is required by - // XLA to be specified as a constant. E.g. the source - // computation (outside of XLA) may include a dynamic - // computation of the shape of something and ComputeConstant lets - // you determine what the value of that computation is in the case - // where the value can be determined at compile time. - // - // If output_layout is non-null, then the output of the computation - // will be stored using that layout. - StatusOr> ComputeConstant( - const XlaOp& operand, const Layout* output_layout = nullptr, - tensorflow::gtl::ArraySlice parameters = {}); - // Returns a new XlaBuilder whose resultant Computation is used only by this // XlaBuilder. The sub-XlaBuilder has the same die_immediately_on_error // behavior as the parent. std::unique_ptr CreateSubBuilder(const string& computation_name); - // Modifies the computation being built so that executions of it will return - // the value associated with operand, rather than the last expression enqueued - // on the XlaBuilder. Any subsequent operations added to the XlaBuilder will - // not have any effect unless SetReturnValue is called again. - Status SetReturnValue(const XlaOp& operand); - // Builds the computation with the requested operations, or returns a non-ok - // status. + // status. Note that all ops that have been enqueued will be moved to the + // computation being returned. StatusOr Build(); // Builds the computation with the requested operations, or notes an error in @@ -784,6 +752,12 @@ class XlaBuilder { // instead. XlaComputation BuildAndNoteError(); + // Returns a subgraph that roots on the given root. If the root is not a + // compile-time constant (see `IsConstant`), returns an error. + // + // This will copy the needed ops/computations to the subgraph. + StatusOr BuildConstantSubGraph(const XlaOp& root_op) const; + // Returns the first error that was encountered while building the // computation. When an error is encountered, by default we return a vacuous // XlaOp and inform the user of the error that occurred while @@ -796,7 +770,7 @@ class XlaBuilder { StatusOr GetShape(const XlaOp& op) const; // Returns the (inferred) result for the current computation's shape. - StatusOr GetProgramShape(); + StatusOr GetProgramShape() const; private: StatusOr AddInstruction( @@ -851,10 +825,17 @@ class XlaBuilder { // Returns the (inferred) result for the program shape for the current // computation and fills the root_id in the pointer. - StatusOr GetProgramShape(int64* root_id); + StatusOr GetProgramShape(int64* root_id) const; + + // A visitor which checks whether an operation is a compile-time constant, + // meaning that it doesn't depend on any parameters, or on any stateful + // operation such as `RngNormal` or `Infeed`. The visitor walks the + // computation starting at a given operation and sets is_constant to false iff + // a parameter or stateful operation is encountered. + void IsConstantVisitor(const int64 op_handle, std::set* visited, + bool* is_constant) const; - string name_; // Name to use for the built computation. - int64 unique_id_; // The unique id for the built computation. + string name_; // Name to use for the built computation. // The first error encountered while building the computation. // This is OK until the first error is encountered. diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc index ec883a6cf3..70af1c44ea 100644 --- a/tensorflow/compiler/xla/service/service.cc +++ b/tensorflow/compiler/xla/service/service.cc @@ -1544,6 +1544,50 @@ tensorflow::Status Service::ComputeConstant(const ComputeConstantRequest* arg, // Since the shape_with_output_layout option in ExecutionOption is // non-effective to the Evaluator results, explicit relayout here. + // + // TODO(b/77824332): Make HloEvaluator take care of the re-layout. + if (arg->has_output_layout()) { + result_literal = result_literal->Relayout(arg->output_layout()); + } + *result->mutable_literal() = result_literal->ToProto(); + + return tensorflow::Status::OK(); +} + +tensorflow::Status Service::ComputeConstantGraph( + const ComputeConstantGraphRequest* arg, ComputeConstantResponse* result) { + if (!arg->has_computation()) { + return InvalidArgument("computations may not be empty"); + } + if (!arg->computation().has_program_shape()) { + return InvalidArgument("program shape may not be empty"); + } + if (arg->computation().program_shape().parameters_size() != 0) { + return InvalidArgument( + "constant computation may not depend on any parameters."); + } + + ProgramShape program_shape = arg->computation().program_shape(); + TF_DCHECK_OK(ShapeUtil::ValidateShape(program_shape.result())); + if (arg->has_output_layout()) { + TF_RETURN_IF_ERROR(LayoutUtil::ValidateLayoutForShape( + arg->output_layout(), program_shape.result())); + } + + HloModuleConfig config(program_shape); + + TF_ASSIGN_OR_RETURN(std::unique_ptr module, + HloModule::CreateFromProto(arg->computation(), config)); + + HloEvaluator evaluator; + TF_ASSIGN_OR_RETURN(auto result_literal, + evaluator.Evaluate>( + *module, /*arg_literals=*/{})); + + // Since the result layout is non-effective to the Evaluator results, explicit + // relayout here. + // + // TODO(b/77824332): Make HloEvaluator take care of the re-layout. if (arg->has_output_layout()) { result_literal = result_literal->Relayout(arg->output_layout()); } diff --git a/tensorflow/compiler/xla/service/service.h b/tensorflow/compiler/xla/service/service.h index 9fa72c1b8c..e399f1ac19 100644 --- a/tensorflow/compiler/xla/service/service.h +++ b/tensorflow/compiler/xla/service/service.h @@ -206,6 +206,9 @@ class Service : public ServiceInterface { // Computes the value of a constant expression. tensorflow::Status ComputeConstant(const ComputeConstantRequest* arg, ComputeConstantResponse* result) override; + tensorflow::Status ComputeConstantGraph( + const ComputeConstantGraphRequest* arg, + ComputeConstantResponse* result) override; // Returns the shape (with layout) of an array associated with a given data // handle. diff --git a/tensorflow/compiler/xla/service_interface.h b/tensorflow/compiler/xla/service_interface.h index 32aae64973..5b44c26b7c 100644 --- a/tensorflow/compiler/xla/service_interface.h +++ b/tensorflow/compiler/xla/service_interface.h @@ -112,6 +112,10 @@ class ServiceInterface { virtual tensorflow::Status ComputeConstant( const ComputeConstantRequest* arg, ComputeConstantResponse* result) = 0; + virtual tensorflow::Status ComputeConstantGraph( + const ComputeConstantGraphRequest* arg, + ComputeConstantResponse* result) = 0; + // Methods used by Computation. virtual tensorflow::Status SnapshotComputation( const SnapshotComputationRequest* ag, diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD index 8ecb421780..6c43014b33 100644 --- a/tensorflow/compiler/xla/tests/BUILD +++ b/tensorflow/compiler/xla/tests/BUILD @@ -1551,6 +1551,8 @@ xla_test( "//tensorflow/compiler/xla/client:computation", "//tensorflow/compiler/xla/client:computation_builder", "//tensorflow/compiler/xla/client:global_data", + "//tensorflow/compiler/xla/client/xla_client:xla_builder", + "//tensorflow/compiler/xla/client/xla_client:xla_computation", "//tensorflow/compiler/xla/tests:literal_test_util", "//tensorflow/compiler/xla/tests:test_utils", "//tensorflow/compiler/xla/tests:xla_internal_test_main", diff --git a/tensorflow/compiler/xla/tests/compute_constant_test.cc b/tensorflow/compiler/xla/tests/compute_constant_test.cc index e5a03b49ad..c15d808f1d 100644 --- a/tensorflow/compiler/xla/tests/compute_constant_test.cc +++ b/tensorflow/compiler/xla/tests/compute_constant_test.cc @@ -21,6 +21,8 @@ limitations under the License. #include "tensorflow/compiler/xla/client/computation.h" #include "tensorflow/compiler/xla/client/computation_builder.h" #include "tensorflow/compiler/xla/client/global_data.h" +#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h" +#include "tensorflow/compiler/xla/client/xla_client/xla_computation.h" #include "tensorflow/compiler/xla/layout_util.h" #include "tensorflow/compiler/xla/literal_util.h" #include "tensorflow/compiler/xla/shape_util.h" @@ -31,6 +33,7 @@ limitations under the License. #include "tensorflow/compiler/xla/tests/test_macros.h" #include "tensorflow/compiler/xla/tests/test_utils.h" #include "tensorflow/compiler/xla/xla_data.pb.h" +#include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/platform/types.h" @@ -71,28 +74,35 @@ class ComputeConstantTest : public ::testing::Test { } StatusOr> ComputeConstantLiteral( - Client* client, const ComputationDataHandle& operand, - ComputationBuilder* builder, Layout* output_layout = nullptr, - tensorflow::gtl::ArraySlice parameters = {}) { - TF_ASSIGN_OR_RETURN(auto computed, builder->ComputeConstant( - operand, output_layout, parameters)); + Client* client, const XlaOp& operand, XlaBuilder* builder, + Layout* output_layout = nullptr) { + TF_ASSIGN_OR_RETURN(auto subgraph, builder->BuildConstantSubGraph(operand)); + TF_ASSIGN_OR_RETURN(auto computed, + client->ComputeConstant(subgraph, output_layout)); return std::move(computed); } + template + StatusOr ComputeConstantScalar(Client* client, const XlaOp& operand, + XlaBuilder* builder) { + TF_ASSIGN_OR_RETURN(auto literal, ComputeConstantLiteral(client, operand, + builder, nullptr)); + return literal->Get({}); + } + template StatusOr ComputeConstantScalar( Client* client, const ComputationDataHandle& operand, ComputationBuilder* builder, tensorflow::gtl::ArraySlice parameters = {}) { - TF_ASSIGN_OR_RETURN( - auto literal, - ComputeConstantLiteral(client, operand, builder, nullptr, parameters)); + TF_ASSIGN_OR_RETURN(auto literal, + builder->ComputeConstant( + operand, /*output_layout=*/nullptr, parameters)); return literal->Get({}); } - bool IsConstant(const ComputationDataHandle& operand, - ComputationBuilder* builder, int64 num_parameters = 0) { - StatusOr result = builder->IsConstant(operand, num_parameters); + bool IsConstant(const XlaOp& operand, XlaBuilder* builder) { + StatusOr result = builder->IsConstant(operand); EXPECT_TRUE(result.ok()) << result.status(); return result.ok() ? result.ValueOrDie() : false; } @@ -103,7 +113,7 @@ class ComputeConstantTest : public ::testing::Test { TEST_F(ComputeConstantTest, ScalarInt32Literal) { for (ClientType client_type : client_types) { Client* client = ClientOrDie(platform_, client_type); - ComputationBuilder b(client, TestName()); + XlaBuilder b(TestName()); auto computation = b.ConstantR0(42); EXPECT_TRUE(IsConstant(computation, &b)); @@ -116,7 +126,7 @@ TEST_F(ComputeConstantTest, ScalarInt32Literal) { TEST_F(ComputeConstantTest, ScalarFloatAdd) { for (ClientType client_type : client_types) { Client* client = ClientOrDie(platform_, client_type); - ComputationBuilder b(client, TestName()); + XlaBuilder b(TestName()); auto computation = b.Add(b.ConstantR0(42.5f), b.ConstantR0(1.5f)); EXPECT_TRUE(IsConstant(computation, &b)); @@ -130,7 +140,7 @@ TEST_F(ComputeConstantTest, ScalarFloatAdd) { TEST_F(ComputeConstantTest, ScalarRng) { for (ClientType client_type : client_types) { Client* client = ClientOrDie(platform_, client_type); - ComputationBuilder b(client, TestName()); + XlaBuilder b(TestName()); auto computation = b.RngUniform(b.ConstantR0(1.1f), b.ConstantR0(2.1f), ShapeUtil::MakeShape(F32, {})); @@ -151,19 +161,21 @@ TEST_F(ComputeConstantTest, Param) { std::vector arguments; arguments.push_back(std::move(*Literal::CreateR0(42.5f))); - EXPECT_TRUE(IsConstant(computation, &b, arguments.size())); - - auto value = - ComputeConstantScalar(client, computation, &b, arguments); - ASSERT_TRUE(value.ok()) << value.status(); - EXPECT_EQ(value.ValueOrDie(), 44.0f); + TF_ASSERT_OK_AND_ASSIGN(bool is_constant, + b.IsConstant(computation, arguments.size())); + EXPECT_TRUE(is_constant); + + TF_ASSERT_OK_AND_ASSIGN( + auto value, + ComputeConstantScalar(client, computation, &b, arguments)); + EXPECT_EQ(value, 44.0f); } } TEST_F(ComputeConstantTest, DirectParamMissing) { for (ClientType client_type : client_types) { Client* client = ClientOrDie(platform_, client_type); - ComputationBuilder b(client, TestName()); + XlaBuilder b(TestName()); auto computation = b.Parameter(0, ShapeUtil::MakeShape(F32, {}), "param"); EXPECT_FALSE(IsConstant(computation, &b)); @@ -177,7 +189,7 @@ TEST_F(ComputeConstantTest, DirectParamMissing) { TEST_F(ComputeConstantTest, IndirectParamMissing) { for (ClientType client_type : client_types) { Client* client = ClientOrDie(platform_, client_type); - ComputationBuilder b(client, TestName()); + XlaBuilder b(TestName()); auto computation = b.Add(b.ConstantR0(1.0f), b.Parameter(0, ShapeUtil::MakeShape(F32, {}), "param")); @@ -195,7 +207,7 @@ TEST_F(ComputeConstantTest, IndirectParamMissing) { TEST_F(ComputeConstantTest, UnrelatedParam) { for (ClientType client_type : client_types) { Client* client = ClientOrDie(platform_, client_type); - ComputationBuilder b(client, TestName()); + XlaBuilder b(TestName()); auto param_a = b.Parameter(10, ShapeUtil::MakeShape(F32, {}), "param0"); auto constant_4 = @@ -212,64 +224,64 @@ TEST_F(ComputeConstantTest, UnrelatedParam) { EXPECT_TRUE(IsConstant(constant_13, &b)); - auto value = ComputeConstantScalar(client, constant_13, &b); - ASSERT_TRUE(value.ok()) << value.status(); - EXPECT_EQ(value.ValueOrDie(), 13.0f); + TF_ASSERT_OK_AND_ASSIGN( + auto value, ComputeConstantScalar(client, constant_13, &b)); + EXPECT_EQ(value, 13.0f); } } TEST_F(ComputeConstantTest, NonScalarAdd) { for (ClientType client_type : client_types) { Client* client = ClientOrDie(platform_, client_type); - ComputationBuilder b(client, TestName()); + XlaBuilder b(TestName()); auto computation = b.Add(b.ConstantR1({1, 2}), b.ConstantR1({3, 4})); EXPECT_TRUE(IsConstant(computation, &b)); - auto computed = ComputeConstantLiteral(client, computation, &b); - ASSERT_TRUE(computed.ok()) << computed.status(); + TF_ASSERT_OK_AND_ASSIGN(auto computed, + ComputeConstantLiteral(client, computation, &b)); std::unique_ptr expected_literal = Literal::CreateR1({4, 6}); - LiteralTestUtil::ExpectEqual(*expected_literal, *computed.ValueOrDie()); + LiteralTestUtil::ExpectEqual(*expected_literal, *computed); } } TEST_F(ComputeConstantTest, IntegerDivide) { for (ClientType client_type : client_types) { Client* client = ClientOrDie(platform_, client_type); - ComputationBuilder b(client, TestName()); + XlaBuilder b(TestName()); auto computation = b.Div(b.ConstantR0(15), b.ConstantR0(3)); EXPECT_TRUE(IsConstant(computation, &b)); - auto computed = ComputeConstantLiteral(client, computation, &b); - ASSERT_TRUE(computed.ok()) << computed.status(); + TF_ASSERT_OK_AND_ASSIGN(auto computed, + ComputeConstantLiteral(client, computation, &b)); std::unique_ptr expected_literal = Literal::CreateR0(5); - LiteralTestUtil::ExpectEqual(*expected_literal, *computed.ValueOrDie()); + LiteralTestUtil::ExpectEqual(*expected_literal, *computed); } } XLA_TEST_F(ComputeConstantTest, Layout) { for (ClientType client_type : client_types) { Client* client = ClientOrDie(platform_, client_type); - ComputationBuilder b(client, TestName()); + XlaBuilder b(TestName()); std::vector> layouts = {{0, 1}, {1, 0}}; for (const std::vector& layout : layouts) { auto layout_proto = LayoutUtil::MakeLayout(layout); - auto computed = ComputeConstantLiteral( - client, - b.Add(b.ConstantR2({{1, 2}, {3, 4}}), - b.ConstantR2({{10, 20}, {30, 40}})), - &b, &layout_proto); - ASSERT_TRUE(computed.ok()) << computed.status(); + TF_ASSERT_OK_AND_ASSIGN( + auto computed, ComputeConstantLiteral( + client, + b.Add(b.ConstantR2({{1, 2}, {3, 4}}), + b.ConstantR2({{10, 20}, {30, 40}})), + &b, &layout_proto)); std::unique_ptr expected_literal = Literal::CreateR2WithLayout({{11, 22}, {33, 44}}, LayoutUtil::MakeLayout(layout)); - LiteralTestUtil::AssertEqualShapesAndLayouts( - expected_literal->shape(), computed.ValueOrDie()->shape()); - LiteralTestUtil::ExpectEqual(*expected_literal, *computed.ValueOrDie()); + LiteralTestUtil::AssertEqualShapesAndLayouts(expected_literal->shape(), + computed->shape()); + LiteralTestUtil::ExpectEqual(*expected_literal, *computed); } } } diff --git a/tensorflow/compiler/xla/xla.proto b/tensorflow/compiler/xla/xla.proto index f9943f71d3..b4cbdf3773 100644 --- a/tensorflow/compiler/xla/xla.proto +++ b/tensorflow/compiler/xla/xla.proto @@ -417,6 +417,11 @@ message ComputeConstantRequest { repeated LiteralProto parameters = 4; } +message ComputeConstantGraphRequest { + HloModuleProto computation = 1; + Layout output_layout = 2; +} + message ComputeConstantResponse { // A LiteralProto is returned directly for this request, instead of a // ComputationDataHandle. -- GitLab From 6e027ee0cbb0e389b912306fd88ebefa470d6065 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 9 Apr 2018 23:13:11 -0700 Subject: [PATCH 079/791] [XLA] Redesign: implement and test custom call. PiperOrigin-RevId: 192241311 --- .../compiler/xla/client/xla_client/xla_builder.cc | 13 ++++++++++++- tensorflow/compiler/xla/tests/custom_call_test.cc | 6 +++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc index a01be28881..74d48635eb 100644 --- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc +++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc @@ -887,7 +887,18 @@ void XlaBuilder::Outfeed(const XlaOp& operand, const Shape& shape_with_layout, XlaOp XlaBuilder::CustomCall(const string& call_target_name, tensorflow::gtl::ArraySlice operands, const Shape& shape) { - return UnimplementedOp(); + return NoteErrorOrReturn([&]() -> StatusOr { + HloInstructionProto instr; + if (tensorflow::str_util::StartsWith(call_target_name, "$")) { + return InvalidArgument( + "Invalid custom_call_target \"%s\": Call targets that start with '$' " + "are reserved for internal use.", + call_target_name.c_str()); + } + *instr.mutable_shape() = shape; + instr.set_custom_call_target(call_target_name); + return AddInstruction(std::move(instr), HloOpcode::kCustomCall, operands); + }); } XlaOp XlaBuilder::HostCompute(tensorflow::gtl::ArraySlice operands, diff --git a/tensorflow/compiler/xla/tests/custom_call_test.cc b/tensorflow/compiler/xla/tests/custom_call_test.cc index 2d847a66b0..b43d5c9ff5 100644 --- a/tensorflow/compiler/xla/tests/custom_call_test.cc +++ b/tensorflow/compiler/xla/tests/custom_call_test.cc @@ -134,9 +134,9 @@ class CustomCallClientAPITest : public ClientLibraryTestBase {}; // When using the client API, CustomCall targets can't begin with '$' -- these // are reserved for internal use. XLA_TEST_F(CustomCallClientAPITest, IllegalCustomCallTarget) { - ComputationBuilder builder(client_, TestName()); - auto call = builder.CustomCall("$illegal", /*operands=*/{}, - ShapeUtil::MakeShape(F32, {1})); + XlaBuilder builder(TestName()); + builder.CustomCall("$illegal", /*operands=*/{}, + ShapeUtil::MakeShape(F32, {1})); StatusOr> result = Execute(&builder, /*arguments=*/{}); -- GitLab From 61994c21f5ddee273e0d79b08444b48858e11bfd Mon Sep 17 00:00:00 2001 From: imsheridan Date: Tue, 10 Apr 2018 20:00:22 +0800 Subject: [PATCH 080/791] Remove breaking ``` for math equations --- tensorflow/contrib/optimizer_v2/adam.py | 4 ---- tensorflow/python/training/adam.py | 4 ---- 2 files changed, 8 deletions(-) diff --git a/tensorflow/contrib/optimizer_v2/adam.py b/tensorflow/contrib/optimizer_v2/adam.py index 9bc160c0b9..a38c98f471 100644 --- a/tensorflow/contrib/optimizer_v2/adam.py +++ b/tensorflow/contrib/optimizer_v2/adam.py @@ -40,23 +40,19 @@ class AdamOptimizer(optimizer_v2.OptimizerV2): Initialization: - ``` \\(m_0 <- 0\\) (Initialize initial 1st moment vector) \\(v_0 <- 0\\) (Initialize initial 2nd moment vector) \\(t <- 0\\) (Initialize timestep) - ``` The update rule for `variable` with gradient `g` uses an optimization described at the end of section2 of the paper: - ``` $$t <- t + 1$$ $$lr_t <- \text{learning_rate} * \sqrt{(1 - beta_2^t) / (1 - beta_1^t)}$$ $$m_t <- beta_1 * m_{t-1} + (1 - beta_1) * g$$ $$v_t <- beta_2 * v_{t-1} + (1 - beta_2) * g * g$$ $$variable <- variable - lr_t * m_t / (\sqrt{v_t} + \epsilon)$$ - ``` The default value of 1e-8 for epsilon might not be a good default in general. For example, when training an Inception network on ImageNet a diff --git a/tensorflow/python/training/adam.py b/tensorflow/python/training/adam.py index 1f2c40f18e..dc0f1aba09 100644 --- a/tensorflow/python/training/adam.py +++ b/tensorflow/python/training/adam.py @@ -43,23 +43,19 @@ class AdamOptimizer(optimizer.Optimizer): Initialization: - ``` \\(m_0 <- 0\\) (Initialize initial 1st moment vector) \\(v_0 <- 0\\) (Initialize initial 2nd moment vector) \\(t <- 0\\) (Initialize timestep) - ``` The update rule for `variable` with gradient `g` uses an optimization described at the end of section2 of the paper: - ``` $$t <- t + 1$$ $$lr_t <- \text{learning_rate} * \sqrt{(1 - beta_2^t) / (1 - beta_1^t)}$$ $$m_t <- beta_1 * m_{t-1} + (1 - beta_1) * g$$ $$v_t <- beta_2 * v_{t-1} + (1 - beta_2) * g * g$$ $$variable <- variable - lr_t * m_t / (\sqrt{v_t} + \epsilon)$$ - ``` The default value of 1e-8 for epsilon might not be a good default in general. For example, when training an Inception network on ImageNet a -- GitLab From c6a6253b5d3cd53409e3ae2636c8cb2597353d12 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 07:47:14 -0700 Subject: [PATCH 081/791] Suppress -Wself-assign in self-assignment tests, which triggers in newer clang revisions. PiperOrigin-RevId: 192284946 --- tensorflow/core/lib/gtl/flatset_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/lib/gtl/flatset_test.cc b/tensorflow/core/lib/gtl/flatset_test.cc index 09fbbb1fb6..010b4bb5df 100644 --- a/tensorflow/core/lib/gtl/flatset_test.cc +++ b/tensorflow/core/lib/gtl/flatset_test.cc @@ -252,7 +252,7 @@ TEST(FlatSet, Copy) { NumSet copy2; copy2 = src; EXPECT_EQ(Contents(src), Contents(copy2)); - copy2 = copy2; // Self-assignment + copy2 = *©2; // Self-assignment, avoiding -Wself-assign. EXPECT_EQ(Contents(src), Contents(copy2)); } } -- GitLab From 7a8fb5b32e797565d5ffa8d6f250a33e2210f423 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 08:51:18 -0700 Subject: [PATCH 082/791] Update document PiperOrigin-RevId: 192292160 --- tensorflow/docs_src/programmers_guide/graphs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/docs_src/programmers_guide/graphs.md b/tensorflow/docs_src/programmers_guide/graphs.md index e69b717432..aa72cae766 100644 --- a/tensorflow/docs_src/programmers_guide/graphs.md +++ b/tensorflow/docs_src/programmers_guide/graphs.md @@ -96,7 +96,7 @@ to all API functions in the same context. For example: (See @{$programmers_guide/variables} for more information about variables.) * Calling @{tf.train.Optimizer.minimize} will add operations and tensors to the - default graph that calculate gradients, and return a @{tf.Operation} that, + default graph that calculates gradients, and return a @{tf.Operation} that, when run, will apply those gradients to a set of variables. Most programs rely solely on the default graph. However, -- GitLab From d7e4458c3ca839fd8f5a86b4342905ce511a47eb Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 08:55:24 -0700 Subject: [PATCH 083/791] Added minimum op, better type support in maximum. PiperOrigin-RevId: 192292693 --- tensorflow/contrib/lite/builtin_ops.h | 1 + tensorflow/contrib/lite/kernels/BUILD | 18 ++- .../internal/reference/reference_ops.h | 11 +- .../{maximum.cc => maximum_minimum.cc} | 77 +++++++--- .../lite/kernels/maximum_minimum_test.cc | 143 ++++++++++++++++++ .../contrib/lite/kernels/maximum_test.cc | 95 ------------ tensorflow/contrib/lite/kernels/register.cc | 2 + tensorflow/contrib/lite/model.cc | 3 +- tensorflow/contrib/lite/nnapi_delegate.cc | 1 + tensorflow/contrib/lite/schema/schema.fbs | 5 +- .../contrib/lite/schema/schema_generated.h | 113 +++++++------- tensorflow/contrib/lite/testing/BUILD | 1 + .../contrib/lite/testing/generate_examples.py | 35 +++++ .../testing/generated_examples_zip_test.cc | 1 + .../contrib/lite/toco/tflite/operator.cc | 2 + .../contrib/lite/toco/tflite/operator_test.cc | 2 + 16 files changed, 328 insertions(+), 182 deletions(-) rename tensorflow/contrib/lite/kernels/{maximum.cc => maximum_minimum.cc} (59%) create mode 100644 tensorflow/contrib/lite/kernels/maximum_minimum_test.cc delete mode 100644 tensorflow/contrib/lite/kernels/maximum_test.cc diff --git a/tensorflow/contrib/lite/builtin_ops.h b/tensorflow/contrib/lite/builtin_ops.h index e11c7fb2e4..1ceefafc56 100644 --- a/tensorflow/contrib/lite/builtin_ops.h +++ b/tensorflow/contrib/lite/builtin_ops.h @@ -81,6 +81,7 @@ typedef enum { kTfLiteBuiltinPrelu = 54, kTfLiteBuiltinMaximum = 55, kTfLiteBuiltinArgMax = 56, + kTfLiteBuiltinMinimum = 57, } TfLiteBuiltinOperator; #ifdef __cplusplus diff --git a/tensorflow/contrib/lite/kernels/BUILD b/tensorflow/contrib/lite/kernels/BUILD index b79900623e..f07eca0ba9 100644 --- a/tensorflow/contrib/lite/kernels/BUILD +++ b/tensorflow/contrib/lite/kernels/BUILD @@ -157,7 +157,7 @@ cc_library( "local_response_norm.cc", "lsh_projection.cc", "lstm.cc", - "maximum.cc", + "maximum_minimum.cc", "mean.cc", "mfcc.cc", "mul.cc", @@ -555,9 +555,9 @@ tf_cc_test( ) tf_cc_test( - name = "maximum_test", + name = "maximum_minimum_test", size = "small", - srcs = ["maximum_test.cc"], + srcs = ["maximum_minimum_test.cc"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -941,4 +941,16 @@ tf_cc_test( ], ) +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = [ + "**/METADATA", + "**/OWNERS", + ], + ), + visibility = ["//tensorflow:__subpackages__"], +) + tflite_portable_test_suite() diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h index 31e190e248..410688411e 100644 --- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h +++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h @@ -3419,10 +3419,11 @@ void TensorFlowMaximum(const T* input1_data, const Dims<4>& input1_dims, } } -template -void TensorFlowMaximum(const T* input1_data, const Dims<4>& input1_dims, - const T* input2_data, const Dims<4>& input2_dims, - T* output_data, const Dims<4>& output_dims) { +template +void TensorFlowMaximumMinimum(const T* input1_data, const Dims<4>& input1_dims, + const T* input2_data, const Dims<4>& input2_dims, + T* output_data, const Dims<4>& output_dims, + Op op) { NdArrayDesc<4> desc1; NdArrayDesc<4> desc2; NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2); @@ -3436,7 +3437,7 @@ void TensorFlowMaximum(const T* input1_data, const Dims<4>& input1_dims, auto in2_idx = SubscriptToIndex(desc2, c, x, y, b); auto in1_val = input1_data[in1_idx]; auto in2_val = input2_data[in2_idx]; - output_data[out_idx] = in1_val > in2_val ? in1_val : in2_val; + output_data[out_idx] = op(in1_val, in2_val); } } } diff --git a/tensorflow/contrib/lite/kernels/maximum.cc b/tensorflow/contrib/lite/kernels/maximum_minimum.cc similarity index 59% rename from tensorflow/contrib/lite/kernels/maximum.cc rename to tensorflow/contrib/lite/kernels/maximum_minimum.cc index 13c40603ce..5a28d663c9 100644 --- a/tensorflow/contrib/lite/kernels/maximum.cc +++ b/tensorflow/contrib/lite/kernels/maximum_minimum.cc @@ -24,9 +24,9 @@ limitations under the License. namespace tflite { namespace ops { namespace builtin { -namespace maximum { +namespace maximum_minimum { -// This file has a reference implemenation of TFMaximum. +// This file has a reference implemenation of TFMaximum/TFMinimum. enum KernelType { kReference, }; @@ -35,8 +35,8 @@ constexpr int kInputTensor1 = 0; constexpr int kInputTensor2 = 1; constexpr int kOutputTensor = 0; -struct MaximumContext { - MaximumContext(TfLiteContext* context, TfLiteNode* node) { +struct OpContext { + OpContext(TfLiteContext* context, TfLiteNode* node) { input1 = GetInput(context, node, kInputTensor1); input2 = GetInput(context, node, kInputTensor2); output = GetOutput(context, node, kOutputTensor); @@ -50,7 +50,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_EQ(context, NumInputs(node), 2); TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); - MaximumContext op_context(context, node); + OpContext op_context(context, node); TF_LITE_ENSURE_EQ(context, op_context.input1->type, op_context.input2->type); op_context.output->type = op_context.input1->type; @@ -69,23 +69,49 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { return context->ResizeTensor(context, op_context.output, output_size); } -template -TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { - MaximumContext op_context(context, node); +struct MaximumOp { + template + static data_type op(data_type el1, data_type el2) { + return el1 > el2 ? el1 : el2; + } +}; + +struct MinimumOp { + template + static data_type op(data_type el1, data_type el2) { + return el1 < el2 ? el1 : el2; + } +}; + +template +void TFLiteOperation(TfLiteContext* context, TfLiteNode* node, + const OpContext& op_context) { + reference_ops::TensorFlowMaximumMinimum( + GetTensorData(op_context.input1), + GetTensorDims(op_context.input1), + GetTensorData(op_context.input2), + GetTensorDims(op_context.input2), + GetTensorData(op_context.output), + GetTensorDims(op_context.output), op_type::template op); +} -#define TF_LITE_MAXIMUM(kernel_type, data_type) \ - kernel_type::TensorFlowMaximum( \ - GetTensorData(op_context.input1), \ - GetTensorDims(op_context.input1), \ - GetTensorData(op_context.input2), \ - GetTensorDims(op_context.input2), \ - GetTensorData(op_context.output), \ - GetTensorDims(op_context.output)) +template +TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { + OpContext op_context(context, node); if (kernel_type == kReference) { switch (op_context.output->type) { case kTfLiteFloat32: - TF_LITE_MAXIMUM(reference_ops, float); + TFLiteOperation(context, node, op_context); + break; + case kTfLiteUInt8: + TFLiteOperation(context, node, op_context); + break; + case kTfLiteInt32: + TFLiteOperation(context, node, op_context); + break; + case kTfLiteInt64: + TFLiteOperation(context, node, op_context); break; default: context->ReportError(context, @@ -99,19 +125,28 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { op_context.output->type); return kTfLiteError; } -#undef TF_LITE_MAXIMUM return kTfLiteOk; } -} // namespace maximum +} // namespace maximum_minimum TfLiteRegistration* Register_MAXIMUM_REF() { - static TfLiteRegistration r = {nullptr, nullptr, maximum::Prepare, - maximum::Eval}; + static TfLiteRegistration r = { + nullptr, nullptr, maximum_minimum::Prepare, + maximum_minimum::Eval}; return &r; } +TfLiteRegistration* Register_MINIMUM_REF() { + static TfLiteRegistration r = { + nullptr, nullptr, maximum_minimum::Prepare, + maximum_minimum::Eval}; + return &r; +} TfLiteRegistration* Register_MAXIMUM() { return Register_MAXIMUM_REF(); } +TfLiteRegistration* Register_MINIMUM() { return Register_MINIMUM_REF(); } } // namespace builtin } // namespace ops diff --git a/tensorflow/contrib/lite/kernels/maximum_minimum_test.cc b/tensorflow/contrib/lite/kernels/maximum_minimum_test.cc new file mode 100644 index 0000000000..0752aa1804 --- /dev/null +++ b/tensorflow/contrib/lite/kernels/maximum_minimum_test.cc @@ -0,0 +1,143 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include "tensorflow/contrib/lite/interpreter.h" +#include "tensorflow/contrib/lite/kernels/register.h" +#include "tensorflow/contrib/lite/kernels/test_util.h" +#include "tensorflow/contrib/lite/model.h" + +namespace tflite { +namespace { + +using ::testing::ElementsAreArray; + +class MaxMinOpModel : public SingleOpModel { + public: + MaxMinOpModel(tflite::BuiltinOperator op, const TensorData& input1, + const TensorData& input2, const TensorType& output) { + input1_ = AddInput(input1); + input2_ = AddInput(input2); + output_ = AddOutput(output); + SetBuiltinOp(op, BuiltinOptions_MaximumMinimumOptions, + CreateMaximumMinimumOptions(builder_).Union()); + BuildInterpreter({GetShape(input1_), GetShape(input2_)}); + } + + template + void SetInput1(std::initializer_list data) { + PopulateTensor(input1_, data); + } + + template + void SetInput2(std::initializer_list data) { + PopulateTensor(input2_, data); + } + + template + std::vector GetOutput() { + return ExtractVector(output_); + } + std::vector GetOutputShape() { return GetTensorShape(output_); } + + protected: + int input1_; + int input2_; + int output_; +}; + +template +void TestModel(tflite::BuiltinOperator op, const TensorData& input1, + const TensorData& input2, const TensorData& output, + std::initializer_list input1_values, + std::initializer_list input2_values, + std::initializer_list output_values) { + MaxMinOpModel m(op, input1, input2, output.type); + m.SetInput1(input1_values); + m.SetInput2(input2_values); + m.Invoke(); + EXPECT_THAT(m.GetOutputShape(), ElementsAreArray(output.shape)); + EXPECT_THAT(m.GetOutput(), ElementsAreArray(output_values)); +} + +template <> +void TestModel(tflite::BuiltinOperator op, const TensorData& input1, + const TensorData& input2, const TensorData& output, + std::initializer_list input1_values, + std::initializer_list input2_values, + std::initializer_list output_values) { + MaxMinOpModel m(op, input1, input2, output.type); + m.SetInput1(input1_values); + m.SetInput2(input2_values); + m.Invoke(); + EXPECT_THAT(m.GetOutputShape(), ElementsAreArray(output.shape)); + EXPECT_THAT(m.GetOutput(), + ElementsAreArray(ArrayFloatNear(output_values))); +} + +TEST(MaximumOpTest, FloatTest) { + std::initializer_list data1 = {1.0, 0.0, -1.0, 11.0, -2.0, -1.44}; + std::initializer_list data2 = {-1.0, 0.0, 1.0, 12.0, -3.0, -1.43}; + TestModel(BuiltinOperator_MAXIMUM, {TensorType_FLOAT32, {3, 1, 2}}, + {TensorType_FLOAT32, {3, 1, 2}}, + {TensorType_FLOAT32, {3, 1, 2}}, data1, data2, + {1.0, 0.0, 1.0, 12.0, -2.0, -1.43}); + TestModel(BuiltinOperator_MINIMUM, {TensorType_FLOAT32, {3, 1, 2}}, + {TensorType_FLOAT32, {3, 1, 2}}, + {TensorType_FLOAT32, {3, 1, 2}}, data1, data2, + {-1.0, 0.0, -1.0, 11.0, -3.0, -1.44}); +} + +TEST(MaxMinOpTest, Uint8Test) { + std::initializer_list data1 = {1, 0, 2, 11, 2, 23}; + std::initializer_list data2 = {0, 0, 1, 12, 255, 1}; + TestModel(BuiltinOperator_MAXIMUM, {TensorType_UINT8, {3, 1, 2}}, + {TensorType_UINT8, {3, 1, 2}}, + {TensorType_UINT8, {3, 1, 2}}, data1, data2, + {1, 0, 2, 12, 255, 23}); + TestModel(BuiltinOperator_MINIMUM, {TensorType_UINT8, {3, 1, 2}}, + {TensorType_UINT8, {3, 1, 2}}, + {TensorType_UINT8, {3, 1, 2}}, data1, data2, + {0, 0, 1, 11, 2, 1}); +} + +TEST(MaximumOpTest, FloatWithBroadcastTest) { + std::initializer_list data1 = {1.0, 0.0, -1.0, -2.0, -1.44, 11.0}; + std::initializer_list data2 = {0.5, 2.0}; + TestModel(BuiltinOperator_MAXIMUM, {TensorType_FLOAT32, {3, 1, 2}}, + {TensorType_FLOAT32, {2}}, {TensorType_FLOAT32, {3, 1, 2}}, + data1, data2, {1.0, 2.0, 0.5, 2.0, 0.5, 11.0}); + TestModel(BuiltinOperator_MINIMUM, {TensorType_FLOAT32, {3, 1, 2}}, + {TensorType_FLOAT32, {2}}, {TensorType_FLOAT32, {3, 1, 2}}, + data1, data2, {0.5, 0.0, -1.0, -2.0, -1.44, 2.0}); +} + +TEST(MaximumOpTest, Int32WithBroadcastTest) { + std::initializer_list data1 = {1, 0, -1, -2, 3, 11}; + std::initializer_list data2 = {2}; + TestModel(BuiltinOperator_MAXIMUM, {TensorType_INT32, {3, 1, 2}}, + {TensorType_INT32, {1}}, {TensorType_INT32, {3, 1, 2}}, + data1, data2, {2, 2, 2, 2, 3, 11}); + TestModel(BuiltinOperator_MINIMUM, {TensorType_INT32, {3, 1, 2}}, + {TensorType_INT32, {1}}, {TensorType_INT32, {3, 1, 2}}, + data1, data2, {1, 0, -1, -2, 2, 2}); +} +} // namespace +} // namespace tflite + +int main(int argc, char** argv) { + ::tflite::LogToStderr(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tensorflow/contrib/lite/kernels/maximum_test.cc b/tensorflow/contrib/lite/kernels/maximum_test.cc deleted file mode 100644 index df2bf29c20..0000000000 --- a/tensorflow/contrib/lite/kernels/maximum_test.cc +++ /dev/null @@ -1,95 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ -#include -#include "tensorflow/contrib/lite/interpreter.h" -#include "tensorflow/contrib/lite/kernels/register.h" -#include "tensorflow/contrib/lite/kernels/test_util.h" -#include "tensorflow/contrib/lite/model.h" - -namespace tflite { -namespace { - -using ::testing::ElementsAreArray; - -class MaximumOpModel : public SingleOpModel { - public: - MaximumOpModel(const TensorData& input1, const TensorData& input2, - const TensorType& output) { - input1_ = AddInput(input1); - input2_ = AddInput(input2); - output_ = AddOutput(output); - SetBuiltinOp(BuiltinOperator_MAXIMUM, BuiltinOptions_MaximumOptions, - CreateMaximumOptions(builder_).Union()); - BuildInterpreter({GetShape(input1_), GetShape(input2_)}); - } - - template - void SetInput1(std::initializer_list data) { - PopulateTensor(input1_, data); - } - - template - void SetInput2(std::initializer_list data) { - PopulateTensor(input2_, data); - } - - template - std::vector GetOutput() { - return ExtractVector(output_); - } - std::vector GetOutputShape() { return GetTensorShape(output_); } - - protected: - int input1_; - int input2_; - int output_; -}; - -TEST(MaximumOpTest, FloatTest) { - std::initializer_list data1 = {1.0, 0.0, -1.0, 11.0, -2.0, -1.44}; - std::initializer_list data2 = {-1.0, 0.0, 1.0, 12.0, -3.0, -1.43}; - MaximumOpModel m({TensorType_FLOAT32, {3, 1, 2}}, - {TensorType_FLOAT32, {3, 1, 2}}, TensorType_FLOAT32); - m.SetInput1(data1); - m.SetInput2(data2); - m.Invoke(); - EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({3, 1, 2})); - EXPECT_THAT( - m.GetOutput(), - ElementsAreArray(ArrayFloatNear({1.0, 0.0, 1.0, 12.0, -2.0, -1.43}))); -} - -TEST(MaximumOpTest, FloatWithBroadcastTest) { - std::initializer_list data1 = {1.0, 0.0, -1.0, -2.0, -1.44, 11.0}; - std::initializer_list data2 = {0.5, 2.0}; - MaximumOpModel m({TensorType_FLOAT32, {3, 1, 2}}, {TensorType_FLOAT32, {2}}, - TensorType_FLOAT32); - m.SetInput1(data1); - m.SetInput2(data2); - m.Invoke(); - EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({3, 1, 2})); - EXPECT_THAT( - m.GetOutput(), - ElementsAreArray(ArrayFloatNear({1.0, 2.0, 0.5, 2.0, 0.5, 11.0}))); -} - -} // namespace -} // namespace tflite - -int main(int argc, char** argv) { - ::tflite::LogToStderr(); - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/tensorflow/contrib/lite/kernels/register.cc b/tensorflow/contrib/lite/kernels/register.cc index 384e1afaa4..67ba8d0f39 100644 --- a/tensorflow/contrib/lite/kernels/register.cc +++ b/tensorflow/contrib/lite/kernels/register.cc @@ -77,6 +77,7 @@ TfLiteRegistration* Register_CAST(); TfLiteRegistration* Register_DEQUANTIZE(); TfLiteRegistration* Register_PRELU(); TfLiteRegistration* Register_MAXIMUM(); +TfLiteRegistration* Register_MINIMUM(); TfLiteRegistration* Register_ARG_MAX(); BuiltinOpResolver::BuiltinOpResolver() { @@ -136,6 +137,7 @@ BuiltinOpResolver::BuiltinOpResolver() { AddBuiltin(BuiltinOperator_DEQUANTIZE, Register_DEQUANTIZE()); AddBuiltin(BuiltinOperator_PRELU, Register_PRELU()); AddBuiltin(BuiltinOperator_MAXIMUM, Register_MAXIMUM()); + AddBuiltin(BuiltinOperator_MINIMUM, Register_MINIMUM()); AddBuiltin(BuiltinOperator_ARG_MAX, Register_ARG_MAX()); // TODO(andrewharp, ahentz): Move these somewhere more appropriate so that diff --git a/tensorflow/contrib/lite/model.cc b/tensorflow/contrib/lite/model.cc index 921c139e30..13e5532909 100644 --- a/tensorflow/contrib/lite/model.cc +++ b/tensorflow/contrib/lite/model.cc @@ -650,7 +650,8 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, builtin_data = reinterpret_cast(params); break; } - case BuiltinOperator_MAXIMUM: { + case BuiltinOperator_MAXIMUM: + case BuiltinOperator_MINIMUM: { break; } case BuiltinOperator_ARG_MAX: { diff --git a/tensorflow/contrib/lite/nnapi_delegate.cc b/tensorflow/contrib/lite/nnapi_delegate.cc index 04d53d955a..08fb820767 100644 --- a/tensorflow/contrib/lite/nnapi_delegate.cc +++ b/tensorflow/contrib/lite/nnapi_delegate.cc @@ -351,6 +351,7 @@ void AddOpsAndParams(tflite::Interpreter* interpreter, case tflite::BuiltinOperator_CAST: case tflite::BuiltinOperator_PRELU: case tflite::BuiltinOperator_MAXIMUM: + case tflite::BuiltinOperator_MINIMUM: case tflite::BuiltinOperator_ARG_MAX: FATAL("Op code %d is currently not delegated to NNAPI", builtin); nn_op_type = -1; // set to invalid diff --git a/tensorflow/contrib/lite/schema/schema.fbs b/tensorflow/contrib/lite/schema/schema.fbs index 238a406af5..357493755d 100644 --- a/tensorflow/contrib/lite/schema/schema.fbs +++ b/tensorflow/contrib/lite/schema/schema.fbs @@ -133,6 +133,7 @@ enum BuiltinOperator : byte { PRELU = 54, MAXIMUM = 55, ARG_MAX = 56, + MINIMUM = 57, } // Options for the builtin operators. @@ -175,7 +176,7 @@ union BuiltinOptions { LogSoftmaxOptions, CastOptions, DequantizeOptions, - MaximumOptions, + MaximumMinimumOptions, ArgMaxOptions, } @@ -390,7 +391,7 @@ table CastOptions { table DequantizeOptions { } -table MaximumOptions { +table MaximumMinimumOptions { } table ArgMaxOptions { diff --git a/tensorflow/contrib/lite/schema/schema_generated.h b/tensorflow/contrib/lite/schema/schema_generated.h index 8b355b0dc6..c638daf66e 100755 --- a/tensorflow/contrib/lite/schema/schema_generated.h +++ b/tensorflow/contrib/lite/schema/schema_generated.h @@ -145,8 +145,8 @@ struct CastOptionsT; struct DequantizeOptions; struct DequantizeOptionsT; -struct MaximumOptions; -struct MaximumOptionsT; +struct MaximumMinimumOptions; +struct MaximumMinimumOptionsT; struct ArgMaxOptions; struct ArgMaxOptionsT; @@ -263,11 +263,12 @@ enum BuiltinOperator { BuiltinOperator_PRELU = 54, BuiltinOperator_MAXIMUM = 55, BuiltinOperator_ARG_MAX = 56, + BuiltinOperator_MINIMUM = 57, BuiltinOperator_MIN = BuiltinOperator_ADD, - BuiltinOperator_MAX = BuiltinOperator_ARG_MAX + BuiltinOperator_MAX = BuiltinOperator_MINIMUM }; -inline BuiltinOperator (&EnumValuesBuiltinOperator())[55] { +inline BuiltinOperator (&EnumValuesBuiltinOperator())[56] { static BuiltinOperator values[] = { BuiltinOperator_ADD, BuiltinOperator_AVERAGE_POOL_2D, @@ -323,7 +324,8 @@ inline BuiltinOperator (&EnumValuesBuiltinOperator())[55] { BuiltinOperator_CAST, BuiltinOperator_PRELU, BuiltinOperator_MAXIMUM, - BuiltinOperator_ARG_MAX + BuiltinOperator_ARG_MAX, + BuiltinOperator_MINIMUM }; return values; } @@ -387,6 +389,7 @@ inline const char **EnumNamesBuiltinOperator() { "PRELU", "MAXIMUM", "ARG_MAX", + "MINIMUM", nullptr }; return names; @@ -437,7 +440,7 @@ enum BuiltinOptions { BuiltinOptions_LogSoftmaxOptions = 36, BuiltinOptions_CastOptions = 37, BuiltinOptions_DequantizeOptions = 38, - BuiltinOptions_MaximumOptions = 39, + BuiltinOptions_MaximumMinimumOptions = 39, BuiltinOptions_ArgMaxOptions = 40, BuiltinOptions_MIN = BuiltinOptions_NONE, BuiltinOptions_MAX = BuiltinOptions_ArgMaxOptions @@ -484,7 +487,7 @@ inline BuiltinOptions (&EnumValuesBuiltinOptions())[41] { BuiltinOptions_LogSoftmaxOptions, BuiltinOptions_CastOptions, BuiltinOptions_DequantizeOptions, - BuiltinOptions_MaximumOptions, + BuiltinOptions_MaximumMinimumOptions, BuiltinOptions_ArgMaxOptions }; return values; @@ -531,7 +534,7 @@ inline const char **EnumNamesBuiltinOptions() { "LogSoftmaxOptions", "CastOptions", "DequantizeOptions", - "MaximumOptions", + "MaximumMinimumOptions", "ArgMaxOptions", nullptr }; @@ -699,8 +702,8 @@ template<> struct BuiltinOptionsTraits { static const BuiltinOptions enum_value = BuiltinOptions_DequantizeOptions; }; -template<> struct BuiltinOptionsTraits { - static const BuiltinOptions enum_value = BuiltinOptions_MaximumOptions; +template<> struct BuiltinOptionsTraits { + static const BuiltinOptions enum_value = BuiltinOptions_MaximumMinimumOptions; }; template<> struct BuiltinOptionsTraits { @@ -1042,13 +1045,13 @@ struct BuiltinOptionsUnion { return type == BuiltinOptions_DequantizeOptions ? reinterpret_cast(value) : nullptr; } - MaximumOptionsT *AsMaximumOptions() { - return type == BuiltinOptions_MaximumOptions ? - reinterpret_cast(value) : nullptr; + MaximumMinimumOptionsT *AsMaximumMinimumOptions() { + return type == BuiltinOptions_MaximumMinimumOptions ? + reinterpret_cast(value) : nullptr; } - const MaximumOptionsT *AsMaximumOptions() const { - return type == BuiltinOptions_MaximumOptions ? - reinterpret_cast(value) : nullptr; + const MaximumMinimumOptionsT *AsMaximumMinimumOptions() const { + return type == BuiltinOptions_MaximumMinimumOptions ? + reinterpret_cast(value) : nullptr; } ArgMaxOptionsT *AsArgMaxOptions() { return type == BuiltinOptions_ArgMaxOptions ? @@ -3827,45 +3830,45 @@ inline flatbuffers::Offset CreateDequantizeOptions( flatbuffers::Offset CreateDequantizeOptions(flatbuffers::FlatBufferBuilder &_fbb, const DequantizeOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); -struct MaximumOptionsT : public flatbuffers::NativeTable { - typedef MaximumOptions TableType; - MaximumOptionsT() { +struct MaximumMinimumOptionsT : public flatbuffers::NativeTable { + typedef MaximumMinimumOptions TableType; + MaximumMinimumOptionsT() { } }; -struct MaximumOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { - typedef MaximumOptionsT NativeTableType; +struct MaximumMinimumOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + typedef MaximumMinimumOptionsT NativeTableType; bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && verifier.EndTable(); } - MaximumOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; - void UnPackTo(MaximumOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const; - static flatbuffers::Offset Pack(flatbuffers::FlatBufferBuilder &_fbb, const MaximumOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); + MaximumMinimumOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(MaximumMinimumOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const; + static flatbuffers::Offset Pack(flatbuffers::FlatBufferBuilder &_fbb, const MaximumMinimumOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); }; -struct MaximumOptionsBuilder { +struct MaximumMinimumOptionsBuilder { flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; - explicit MaximumOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) + explicit MaximumMinimumOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); } - MaximumOptionsBuilder &operator=(const MaximumOptionsBuilder &); - flatbuffers::Offset Finish() { + MaximumMinimumOptionsBuilder &operator=(const MaximumMinimumOptionsBuilder &); + flatbuffers::Offset Finish() { const auto end = fbb_.EndTable(start_); - auto o = flatbuffers::Offset(end); + auto o = flatbuffers::Offset(end); return o; } }; -inline flatbuffers::Offset CreateMaximumOptions( +inline flatbuffers::Offset CreateMaximumMinimumOptions( flatbuffers::FlatBufferBuilder &_fbb) { - MaximumOptionsBuilder builder_(_fbb); + MaximumMinimumOptionsBuilder builder_(_fbb); return builder_.Finish(); } -flatbuffers::Offset CreateMaximumOptions(flatbuffers::FlatBufferBuilder &_fbb, const MaximumOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +flatbuffers::Offset CreateMaximumMinimumOptions(flatbuffers::FlatBufferBuilder &_fbb, const MaximumMinimumOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); struct ArgMaxOptionsT : public flatbuffers::NativeTable { typedef ArgMaxOptions TableType; @@ -4152,8 +4155,8 @@ struct Operator FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { const DequantizeOptions *builtin_options_as_DequantizeOptions() const { return builtin_options_type() == BuiltinOptions_DequantizeOptions ? static_cast(builtin_options()) : nullptr; } - const MaximumOptions *builtin_options_as_MaximumOptions() const { - return builtin_options_type() == BuiltinOptions_MaximumOptions ? static_cast(builtin_options()) : nullptr; + const MaximumMinimumOptions *builtin_options_as_MaximumMinimumOptions() const { + return builtin_options_type() == BuiltinOptions_MaximumMinimumOptions ? static_cast(builtin_options()) : nullptr; } const ArgMaxOptions *builtin_options_as_ArgMaxOptions() const { return builtin_options_type() == BuiltinOptions_ArgMaxOptions ? static_cast(builtin_options()) : nullptr; @@ -4336,8 +4339,8 @@ template<> inline const DequantizeOptions *Operator::builtin_options_as inline const MaximumOptions *Operator::builtin_options_as() const { - return builtin_options_as_MaximumOptions(); +template<> inline const MaximumMinimumOptions *Operator::builtin_options_as() const { + return builtin_options_as_MaximumMinimumOptions(); } template<> inline const ArgMaxOptions *Operator::builtin_options_as() const { @@ -5878,26 +5881,26 @@ inline flatbuffers::Offset CreateDequantizeOptions(flatbuffer _fbb); } -inline MaximumOptionsT *MaximumOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { - auto _o = new MaximumOptionsT(); +inline MaximumMinimumOptionsT *MaximumMinimumOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { + auto _o = new MaximumMinimumOptionsT(); UnPackTo(_o, _resolver); return _o; } -inline void MaximumOptions::UnPackTo(MaximumOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { +inline void MaximumMinimumOptions::UnPackTo(MaximumMinimumOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { (void)_o; (void)_resolver; } -inline flatbuffers::Offset MaximumOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const MaximumOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) { - return CreateMaximumOptions(_fbb, _o, _rehasher); +inline flatbuffers::Offset MaximumMinimumOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const MaximumMinimumOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) { + return CreateMaximumMinimumOptions(_fbb, _o, _rehasher); } -inline flatbuffers::Offset CreateMaximumOptions(flatbuffers::FlatBufferBuilder &_fbb, const MaximumOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) { +inline flatbuffers::Offset CreateMaximumMinimumOptions(flatbuffers::FlatBufferBuilder &_fbb, const MaximumMinimumOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) { (void)_rehasher; (void)_o; - struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const MaximumOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; - return tflite::CreateMaximumOptions( + struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const MaximumMinimumOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + return tflite::CreateMaximumMinimumOptions( _fbb); } @@ -6259,8 +6262,8 @@ inline bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *ob auto ptr = reinterpret_cast(obj); return verifier.VerifyTable(ptr); } - case BuiltinOptions_MaximumOptions: { - auto ptr = reinterpret_cast(obj); + case BuiltinOptions_MaximumMinimumOptions: { + auto ptr = reinterpret_cast(obj); return verifier.VerifyTable(ptr); } case BuiltinOptions_ArgMaxOptions: { @@ -6437,8 +6440,8 @@ inline void *BuiltinOptionsUnion::UnPack(const void *obj, BuiltinOptions type, c auto ptr = reinterpret_cast(obj); return ptr->UnPack(resolver); } - case BuiltinOptions_MaximumOptions: { - auto ptr = reinterpret_cast(obj); + case BuiltinOptions_MaximumMinimumOptions: { + auto ptr = reinterpret_cast(obj); return ptr->UnPack(resolver); } case BuiltinOptions_ArgMaxOptions: { @@ -6603,9 +6606,9 @@ inline flatbuffers::Offset BuiltinOptionsUnion::Pack(flatbuffers::FlatBuff auto ptr = reinterpret_cast(value); return CreateDequantizeOptions(_fbb, ptr, _rehasher).Union(); } - case BuiltinOptions_MaximumOptions: { - auto ptr = reinterpret_cast(value); - return CreateMaximumOptions(_fbb, ptr, _rehasher).Union(); + case BuiltinOptions_MaximumMinimumOptions: { + auto ptr = reinterpret_cast(value); + return CreateMaximumMinimumOptions(_fbb, ptr, _rehasher).Union(); } case BuiltinOptions_ArgMaxOptions: { auto ptr = reinterpret_cast(value); @@ -6769,8 +6772,8 @@ inline BuiltinOptionsUnion::BuiltinOptionsUnion(const BuiltinOptionsUnion &u) FL value = new DequantizeOptionsT(*reinterpret_cast(u.value)); break; } - case BuiltinOptions_MaximumOptions: { - value = new MaximumOptionsT(*reinterpret_cast(u.value)); + case BuiltinOptions_MaximumMinimumOptions: { + value = new MaximumMinimumOptionsT(*reinterpret_cast(u.value)); break; } case BuiltinOptions_ArgMaxOptions: { @@ -6974,8 +6977,8 @@ inline void BuiltinOptionsUnion::Reset() { delete ptr; break; } - case BuiltinOptions_MaximumOptions: { - auto ptr = reinterpret_cast(value); + case BuiltinOptions_MaximumMinimumOptions: { + auto ptr = reinterpret_cast(value); delete ptr; break; } diff --git a/tensorflow/contrib/lite/testing/BUILD b/tensorflow/contrib/lite/testing/BUILD index 386cfdb524..9f0ba43252 100644 --- a/tensorflow/contrib/lite/testing/BUILD +++ b/tensorflow/contrib/lite/testing/BUILD @@ -39,6 +39,7 @@ gen_zipped_test_files( "max_pool.zip", "maximum.zip", "mean.zip", + "minimum.zip", "mul.zip", "pad.zip", "prelu.zip", diff --git a/tensorflow/contrib/lite/testing/generate_examples.py b/tensorflow/contrib/lite/testing/generate_examples.py index 42aa92c1bb..672158aa2f 100644 --- a/tensorflow/contrib/lite/testing/generate_examples.py +++ b/tensorflow/contrib/lite/testing/generate_examples.py @@ -910,6 +910,41 @@ def make_maximum_tests(zip_path): make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs) +def make_minimum_tests(zip_path): + """Make a set of tests to do minimum.""" + + test_parameters = [{ + "input_dtype": [tf.float32], + "input_shape_1": [[3], [1, 100], [4, 2, 3], [5, 224, 224, 3]], + "input_shape_2": [[3], [1, 100], [4, 2, 3], [5, 224, 224, 3]], + }] + + def build_graph(parameters): + """Build the minimum op testing graph.""" + input_tensor_1 = tf.placeholder( + dtype=parameters["input_dtype"], + name="input_1", + shape=parameters["input_shape_1"]) + input_tensor_2 = tf.placeholder( + dtype=parameters["input_dtype"], + name="input_2", + shape=parameters["input_shape_2"]) + + out = tf.minimum(input_tensor_1, input_tensor_2) + return [input_tensor_1, input_tensor_2], [out] + + def build_inputs(parameters, sess, inputs, outputs): + values = [ + create_tensor_data(parameters["input_dtype"], + parameters["input_shape_1"]), + create_tensor_data(parameters["input_dtype"], + parameters["input_shape_2"]) + ] + return values, sess.run(outputs, feed_dict=dict(zip(inputs, values))) + + make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs) + + def make_binary_op_tests_func(binary_operator): """Return a function that does a test on a binary operator.""" return lambda zip_path: make_binary_op_tests(zip_path, binary_operator) diff --git a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc index 291c974545..7426ab56af 100644 --- a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc +++ b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc @@ -262,6 +262,7 @@ INSTANTIATE_TESTS(log_softmax) INSTANTIATE_TESTS(maximum) INSTANTIATE_TESTS(max_pool) INSTANTIATE_TESTS(mean) +INSTANTIATE_TESTS(minimum) INSTANTIATE_TESTS(mul) INSTANTIATE_TESTS(pad) INSTANTIATE_TESTS(relu) diff --git a/tensorflow/contrib/lite/toco/tflite/operator.cc b/tensorflow/contrib/lite/toco/tflite/operator.cc index 4df16827b4..e015108120 100644 --- a/tensorflow/contrib/lite/toco/tflite/operator.cc +++ b/tensorflow/contrib/lite/toco/tflite/operator.cc @@ -890,6 +890,8 @@ std::vector> BuildOperatorList() { "LOG_SOFTMAX", OperatorType::kLogSoftmax)); ops.emplace_back(new SimpleOperator( "MAXIMUM", OperatorType::kTensorFlowMaximum)); + ops.emplace_back(new SimpleOperator( + "MINIMUM", OperatorType::kTensorFlowMinimum)); return ops; } diff --git a/tensorflow/contrib/lite/toco/tflite/operator_test.cc b/tensorflow/contrib/lite/toco/tflite/operator_test.cc index 5546bda696..24ba71e459 100644 --- a/tensorflow/contrib/lite/toco/tflite/operator_test.cc +++ b/tensorflow/contrib/lite/toco/tflite/operator_test.cc @@ -111,6 +111,8 @@ TEST_F(OperatorTest, SimpleOperators) { OperatorType::kLogSoftmax); CheckSimpleOperator( "MAXIMUM", OperatorType::kTensorFlowMaximum); + CheckSimpleOperator( + "MINIMUM", OperatorType::kTensorFlowMinimum); } TEST_F(OperatorTest, BuiltinAdd) { -- GitLab From 40f85affa388288a66d5fb9b2295155216e68b94 Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Tue, 10 Apr 2018 09:26:59 -0700 Subject: [PATCH 084/791] [XLA:GPU] Add infrastructure for unrolling kernels to improve bandwidth utilization. We often have simple kernels that do very little actual work, duplicating that can increase the used bandwidth. This change introduces flags and infrastructure for unrolling kernels, it doesn't include any cost heuristics and is disabled by default. Based on code written by Bixia Zheng. PiperOrigin-RevId: 192296781 --- .../xla/legacy_flags/debug_options_flags.cc | 7 ++- .../xla/service/cpu/parallel_loop_emitter.cc | 5 +- .../xla/service/cpu/parallel_loop_emitter.h | 2 +- .../xla/service/gpu/ir_emitter_unnested.cc | 37 ++++++++++++-- .../xla/service/gpu/ir_emitter_unnested.h | 6 ++- .../compiler/xla/service/gpu/kernel_thunk.cc | 6 ++- .../compiler/xla/service/gpu/kernel_thunk.h | 8 +++- .../xla/service/gpu/parallel_loop_emitter.cc | 48 ++++++++++++++----- .../xla/service/gpu/parallel_loop_emitter.h | 10 ++-- .../xla/service/gpu/partition_assignment.cc | 6 ++- .../xla/service/gpu/partition_assignment.h | 3 +- .../xla/service/llvm_ir/loop_emitter.cc | 12 +++-- .../xla/service/llvm_ir/loop_emitter.h | 7 +-- tensorflow/compiler/xla/xla.proto | 3 ++ 14 files changed, 121 insertions(+), 39 deletions(-) diff --git a/tensorflow/compiler/xla/legacy_flags/debug_options_flags.cc b/tensorflow/compiler/xla/legacy_flags/debug_options_flags.cc index f037663e3f..70ae95bf47 100644 --- a/tensorflow/compiler/xla/legacy_flags/debug_options_flags.cc +++ b/tensorflow/compiler/xla/legacy_flags/debug_options_flags.cc @@ -43,7 +43,7 @@ void SetDebugOptionsDefaults(DebugOptions* flags) { #ifdef INTEL_MKL flags->set_xla_cpu_use_mkl_dnn(true); #endif // INTEL_MKL - + flags->set_xla_gpu_max_kernel_unroll_factor(1); // Set cudnn batchnorm off by default; it does not provide a performance win // on average. flags->set_xla_gpu_use_cudnn_batchnorm(false); @@ -223,6 +223,11 @@ void AllocateFlags() { bool_setter_for(&DebugOptions::set_xla_gpu_disable_multi_streaming), flag_values->xla_gpu_disable_multi_streaming(), "If true, multi-streaming in the GPU backend is disabled."), + tensorflow::Flag( + "xla_gpu_max_kernel_unroll_factor", + int32_setter_for(&DebugOptions::set_xla_gpu_max_kernel_unroll_factor), + flag_values->xla_gpu_max_kernel_unroll_factor(), + "Specify the maximum kernel unroll factor for the GPU backend."), tensorflow::Flag( "xla_dump_optimized_hlo_proto_to", flag_values->mutable_xla_dump_optimized_hlo_proto_to(), diff --git a/tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.cc b/tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.cc index 1e439cde11..54af40506d 100644 --- a/tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.cc +++ b/tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.cc @@ -29,7 +29,8 @@ ParallelLoopEmitter::ParallelLoopEmitter( : LoopEmitter(target_element_generator, target_array, ir_builder), dynamic_loop_bounds_(dynamic_loop_bounds) {} -llvm_ir::IrArray::Index ParallelLoopEmitter::EmitIndexAndSetExitBasicBlock( +std::vector +ParallelLoopEmitter::EmitIndexAndSetExitBasicBlock( tensorflow::StringPiece loop_name) { CHECK(!ShapeUtil::IsTuple(shape_)); CHECK(!ShapeUtil::IsScalar(shape_)); @@ -69,7 +70,7 @@ llvm_ir::IrArray::Index ParallelLoopEmitter::EmitIndexAndSetExitBasicBlock( exit_bb_ = loop_nest.GetOuterLoopExitBasicBlock(); CHECK(exit_bb_ != nullptr); - return array_index; + return {array_index}; } } // namespace cpu diff --git a/tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.h b/tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.h index ce92e36a94..755715634a 100644 --- a/tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.h +++ b/tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.h @@ -60,7 +60,7 @@ class ParallelLoopEmitter : public llvm_ir::LoopEmitter { ParallelLoopEmitter& operator=(const ParallelLoopEmitter&) = delete; ~ParallelLoopEmitter() override = default; - llvm_ir::IrArray::Index EmitIndexAndSetExitBasicBlock( + std::vector EmitIndexAndSetExitBasicBlock( tensorflow::StringPiece loop_name) override; private: diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc index d29cc21ab1..26e497762f 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc @@ -536,7 +536,27 @@ Status IrEmitterUnnested::HandleFusion(HloInstruction* fusion) { thunk_sequence_->emplace_back(BuildGemmThunk(fusion)); return Status::OK(); } - thunk_sequence_->emplace_back(BuildKernelThunk(fusion)); + + int max_unroll_factor = fusion->GetModule() + ->config() + .debug_options() + .xla_gpu_max_kernel_unroll_factor(); + + // Find the largest possible power of two to unroll by. + // TODO(kramerb): Make this smarter. + int unroll_factor = 1; + if (!fusion->IsMultiOutputFusion()) { + CHECK(fusion->fusion_kind() == HloInstruction::FusionKind::kLoop); + int64 num_elements = ShapeUtil::ElementsIn(fusion->shape()); + for (int i = max_unroll_factor; i > 1; i /= 2) { + if (num_elements % i == 0) { + unroll_factor = i; + break; + } + } + } + + thunk_sequence_->emplace_back(BuildKernelThunk(fusion, unroll_factor)); return IrEmitter::HandleFusion(fusion); } @@ -2021,7 +2041,7 @@ Status IrEmitterUnnested::HandleGather(HloInstruction* gather) { } std::unique_ptr IrEmitterUnnested::BuildKernelThunk( - const HloInstruction* inst) { + const HloInstruction* inst, int unroll_factor) { const BufferAssignment& buffer_assn = ir_emitter_context_->buffer_assignment(); @@ -2113,7 +2133,7 @@ std::unique_ptr IrEmitterUnnested::BuildKernelThunk( } return MakeUnique(buffers, llvm_ir::AsString(kernel->getName()), - inst); + inst, unroll_factor); } std::unique_ptr IrEmitterUnnested::BuildHostToDeviceCopyThunk( @@ -2485,21 +2505,28 @@ std::unique_ptr IrEmitterUnnested::BuildConditionalThunk( Status IrEmitterUnnested::EmitTargetElementLoopInThunk( const HloInstruction& hlo, const llvm_ir::ElementGenerator& element_generator, KernelThunk* thunk) { + int unroll_factor = thunk->unroll_factor(); VLOG(3) << bindings_.ToString(); const Shape& element_shape = hlo.IsMultiOutputFusion() ? ShapeUtil::GetSubshape(hlo.shape(), {0}) : hlo.shape(); + VLOG(3) << "EmitTargetElementLoopInThunk " + << ShapeUtil::HumanStringWithLayout(hlo.shape()) + << " for unroll_factor " << unroll_factor; LaunchDimensions launch_dimensions = CalculateLaunchDimensions( - element_shape, ir_emitter_context_->device_description()); + element_shape, ir_emitter_context_->device_description(), unroll_factor); UpdateLaunchDimensions(launch_dimensions, thunk, ir_emitter_context_->llvm_module()); if (!hlo.IsMultiOutputFusion()) { return ParallelLoopEmitter(element_generator, GetIrArray(hlo, hlo), - launch_dimensions, &ir_builder_) + launch_dimensions, &ir_builder_, unroll_factor) .EmitLoop(IrName(&hlo)); } + CHECK_EQ(unroll_factor, 1) + << "multi-output fusion does not support unrolling"; + // For multiple outputs fusion, we need to emit each operand and the root. std::vector output_arrays; for (int64 i = 0; i < ShapeUtil::TupleElementCount(hlo.shape()); ++i) { diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h index 66c62e2d2d..b842f480c6 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h @@ -150,8 +150,10 @@ class IrEmitterUnnested : public IrEmitter { // Returns a KernelThunk that invokes the kernel emitted for `inst`. The // caller needs to make sure `inst` outlives the lifetime of the returned - // Thunk object. - std::unique_ptr BuildKernelThunk(const HloInstruction* inst); + // Thunk object. The kernel implementation will be unrolled if unroll_factor + // is greater than one. + std::unique_ptr BuildKernelThunk(const HloInstruction* inst, + int unroll_factor = 1); // Returns a FftThunk that calls cuFFT to implement `inst`. std::unique_ptr BuildFftThunk(const HloInstruction* inst); diff --git a/tensorflow/compiler/xla/service/gpu/kernel_thunk.cc b/tensorflow/compiler/xla/service/gpu/kernel_thunk.cc index c20a781a33..c24dc1457f 100644 --- a/tensorflow/compiler/xla/service/gpu/kernel_thunk.cc +++ b/tensorflow/compiler/xla/service/gpu/kernel_thunk.cc @@ -30,10 +30,12 @@ namespace gpu { KernelThunk::KernelThunk( tensorflow::gtl::ArraySlice args, - const string& kernel_name, const HloInstruction* hlo_instruction) + const string& kernel_name, const HloInstruction* hlo_instruction, + int unroll_factor) : Thunk(Kind::kKernel, hlo_instruction), args_(args.begin(), args.end()), - kernel_name_(kernel_name) {} + kernel_name_(kernel_name), + unroll_factor_(unroll_factor) {} tensorflow::Status KernelThunk::Initialize(const GpuExecutable& executable) { tensorflow::mutex_lock lock(mutex_); diff --git a/tensorflow/compiler/xla/service/gpu/kernel_thunk.h b/tensorflow/compiler/xla/service/gpu/kernel_thunk.h index 9ae455e2fc..df8971b083 100644 --- a/tensorflow/compiler/xla/service/gpu/kernel_thunk.h +++ b/tensorflow/compiler/xla/service/gpu/kernel_thunk.h @@ -47,12 +47,14 @@ class KernelThunk : public Thunk { // // `hlo_instruction` is as in Thunk. Other arguments are as the class members. KernelThunk(tensorflow::gtl::ArraySlice args, - const string& kernel_name, const HloInstruction* hlo_instruction); + const string& kernel_name, const HloInstruction* hlo_instruction, + int unroll_factor); KernelThunk(const KernelThunk&) = delete; KernelThunk& operator=(const KernelThunk&) = delete; ~KernelThunk() override = default; const string& kernel_name() const { return kernel_name_; } + int unroll_factor() const { return unroll_factor_; } void SetLaunchDimensions(const LaunchDimensions& launch_dims); tensorflow::Status Initialize(const GpuExecutable& executable) override; @@ -69,6 +71,10 @@ class KernelThunk : public Thunk { // Entry kernel name for the computation. const string kernel_name_; + // The number of times this kernel should be unrolled. This works as a + // multiplier on the number of elements produced by a GPU thread. + const int unroll_factor_; + // The thread and block dimension used to launch the kernel. // Will be set by IrEmitterUnnested. LaunchDimensions launch_dimensions_; diff --git a/tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.cc b/tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.cc index 388dcc008b..d8c07dc311 100644 --- a/tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.cc +++ b/tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.cc @@ -32,25 +32,32 @@ namespace gpu { ParallelLoopEmitter::ParallelLoopEmitter( BodyEmitter body_emitter, const Shape& shape, - const LaunchDimensions& launch_dimensions, llvm::IRBuilder<>* ir_builder) + const LaunchDimensions& launch_dimensions, llvm::IRBuilder<>* ir_builder, + int unroll_factor) : LoopEmitter(body_emitter, shape, ir_builder), - launch_dimensions_(launch_dimensions) {} + launch_dimensions_(launch_dimensions), + unroll_factor_(unroll_factor) {} ParallelLoopEmitter::ParallelLoopEmitter( const llvm_ir::ElementGenerator& target_element_generator, tensorflow::gtl::ArraySlice target_arrays, - const LaunchDimensions& launch_dimensions, llvm::IRBuilder<>* ir_builder) + const LaunchDimensions& launch_dimensions, llvm::IRBuilder<>* ir_builder, + int unroll_factor) : LoopEmitter(target_element_generator, target_arrays, ir_builder), - launch_dimensions_(launch_dimensions) {} + launch_dimensions_(launch_dimensions), + unroll_factor_(unroll_factor) {} ParallelLoopEmitter::ParallelLoopEmitter( const llvm_ir::ElementGenerator& target_element_generator, const llvm_ir::IrArray& target_array, - const LaunchDimensions& launch_dimensions, llvm::IRBuilder<>* ir_builder) + const LaunchDimensions& launch_dimensions, llvm::IRBuilder<>* ir_builder, + int unroll_factor) : LoopEmitter(target_element_generator, target_array, ir_builder), - launch_dimensions_(launch_dimensions) {} + launch_dimensions_(launch_dimensions), + unroll_factor_(unroll_factor) {} -llvm_ir::IrArray::Index ParallelLoopEmitter::EmitIndexAndSetExitBasicBlock( +std::vector +ParallelLoopEmitter::EmitIndexAndSetExitBasicBlock( tensorflow::StringPiece loop_name) { // Emit the following code in LLVM IR: // linear_index = blockIdx.x * blockDim.x + threadIdx.x; @@ -63,6 +70,9 @@ llvm_ir::IrArray::Index ParallelLoopEmitter::EmitIndexAndSetExitBasicBlock( // "It is guaranteed that [...] 0 <= %ctaid.x < %nctaid.x" // // %nctaid.x is currently specified as 2147483647. + VLOG(3) << "EmitIndexAndSetExitBasicBlock unroll_factor " << unroll_factor_; + std::vector array_indices; + llvm::Value* block_id = llvm_ir::EmitCallToIntrinsic( llvm::Intrinsic::nvvm_read_ptx_sreg_ctaid_x, {}, {}, ir_builder_); llvm_ir::AddRangeMetadata(0, launch_dimensions_.block_count(), @@ -81,7 +91,7 @@ llvm_ir::IrArray::Index ParallelLoopEmitter::EmitIndexAndSetExitBasicBlock( thread_id = ir_builder_->CreateZExt(thread_id, ir_builder_->getInt64Ty(), "thread_id"); - llvm::Value* linear_index = ir_builder_->CreateAdd( + llvm::Value* linear_index_base = ir_builder_->CreateAdd( ir_builder_->CreateMul( block_id, ir_builder_->getInt64(launch_dimensions_.threads_per_block()), "", @@ -99,15 +109,30 @@ llvm_ir::IrArray::Index ParallelLoopEmitter::EmitIndexAndSetExitBasicBlock( llvm_ir::EmitCallToIntrinsic( llvm::Intrinsic::assume, {ir_builder_->CreateICmpULT( - linear_index, + linear_index_base, ir_builder_->getInt64(launch_dimensions_.threads_per_block() * launch_dimensions_.block_count()), "linear_index_in_range")}, {}, ir_builder_); + if (unroll_factor_ > 1) { + linear_index_base = ir_builder_->CreateMul( + linear_index_base, ir_builder_->getInt64(unroll_factor_), + "linear_index_base", /*HasNUW=*/true, /*HasNSW=*/true); + } + + array_indices.emplace_back(linear_index_base, shape_, ir_builder_); + for (int i = 1; i < unroll_factor_; ++i) { + llvm::Value* linear_index = ir_builder_->CreateAdd( + linear_index_base, ir_builder_->getInt64(i), "linear_index", + /*HasNUW=*/true, /*HasNSW=*/true); + array_indices.emplace_back(linear_index, shape_, ir_builder_); + } + auto if_in_bounds = llvm_ir::EmitIfThenElse( ir_builder_->CreateICmpULT( - linear_index, ir_builder_->getInt64(ShapeUtil::ElementsIn(shape_))), + linear_index_base, + ir_builder_->getInt64(ShapeUtil::ElementsIn(shape_))), llvm_ir::IrName(loop_name, "in_bounds"), ir_builder_, false); // Set exit_bb_ to the exit block of the if structure. @@ -116,7 +141,8 @@ llvm_ir::IrArray::Index ParallelLoopEmitter::EmitIndexAndSetExitBasicBlock( // Set IR builder insertion point to the body of the if structure. llvm_ir::SetToFirstInsertPoint(if_in_bounds.true_block, ir_builder_); - return llvm_ir::IrArray::Index(linear_index, shape_, ir_builder_); + + return array_indices; } } // namespace gpu diff --git a/tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.h b/tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.h index 8ed63a854a..25318b3bed 100644 --- a/tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.h +++ b/tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.h @@ -34,13 +34,13 @@ class ParallelLoopEmitter : public llvm_ir::LoopEmitter { // The meanings of other parameters are the same as LoopEmitter. ParallelLoopEmitter(BodyEmitter body_emitter, const Shape& shape, const LaunchDimensions& launch_dimensions, - llvm::IRBuilder<>* ir_builder); + llvm::IRBuilder<>* ir_builder, int unroll_factor = 1); // Constructs a ParallelLoopEmitter from an element generator that generates // each element of the given target array. ParallelLoopEmitter(const llvm_ir::ElementGenerator& target_element_generator, const llvm_ir::IrArray& target_array, const LaunchDimensions& launch_dimensions, - llvm::IRBuilder<>* ir_builder); + llvm::IRBuilder<>* ir_builder, int unroll_factor = 1); // Constructs a loop emitter for a loop that generates on element of each of N // arrays on each iteration. @@ -50,18 +50,20 @@ class ParallelLoopEmitter : public llvm_ir::LoopEmitter { ParallelLoopEmitter( const llvm_ir::ElementGenerator& target_element_generator, tensorflow::gtl::ArraySlice target_arrays, - const LaunchDimensions& launch_dimensions, llvm::IRBuilder<>* ir_builder); + const LaunchDimensions& launch_dimensions, llvm::IRBuilder<>* ir_builder, + int unroll_factor = 1); ParallelLoopEmitter(const ParallelLoopEmitter&) = delete; ParallelLoopEmitter& operator=(const ParallelLoopEmitter&) = delete; ~ParallelLoopEmitter() override = default; - llvm_ir::IrArray::Index EmitIndexAndSetExitBasicBlock( + std::vector EmitIndexAndSetExitBasicBlock( tensorflow::StringPiece loop_name) override; private: // The thread and block dimension to parallelize the loop on. const LaunchDimensions launch_dimensions_; + const int unroll_factor_; }; } // namespace gpu diff --git a/tensorflow/compiler/xla/service/gpu/partition_assignment.cc b/tensorflow/compiler/xla/service/gpu/partition_assignment.cc index 6cf280df05..5283d51cd1 100644 --- a/tensorflow/compiler/xla/service/gpu/partition_assignment.cc +++ b/tensorflow/compiler/xla/service/gpu/partition_assignment.cc @@ -44,12 +44,16 @@ std::ostream& operator<<(std::ostream& out, // Calculates the launch dimensions used to invoke `hlo`. LaunchDimensions CalculateLaunchDimensions( - const Shape& shape, const se::DeviceDescription& device_desc) { + const Shape& shape, const se::DeviceDescription& device_desc, + int unroll_factor) { int64 num_elements = ShapeUtil::ElementsIn(shape); if (num_elements <= 1) { return LaunchDimensions(); } + CHECK_EQ(num_elements % unroll_factor, 0); + num_elements = num_elements / unroll_factor; + // Since we don't do any inter-warp communication, we're free to choose any // block size we want, subject to hardware constraints. We choose the // smallest block size that allows the GPU to reach full occupancy (assuming diff --git a/tensorflow/compiler/xla/service/gpu/partition_assignment.h b/tensorflow/compiler/xla/service/gpu/partition_assignment.h index 0bf463a6ef..42d2d2af2e 100644 --- a/tensorflow/compiler/xla/service/gpu/partition_assignment.h +++ b/tensorflow/compiler/xla/service/gpu/partition_assignment.h @@ -58,7 +58,8 @@ std::ostream& operator<<(std::ostream& out, LaunchDimensions CalculateLaunchDimensions( const Shape& shape, - const perftools::gputools::DeviceDescription& device_desc); + const perftools::gputools::DeviceDescription& device_desc, + int unroll_factor = 1); } // namespace gpu } // namespace xla diff --git a/tensorflow/compiler/xla/service/llvm_ir/loop_emitter.cc b/tensorflow/compiler/xla/service/llvm_ir/loop_emitter.cc index b6b918ec78..3978acc132 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/loop_emitter.cc +++ b/tensorflow/compiler/xla/service/llvm_ir/loop_emitter.cc @@ -88,12 +88,12 @@ LoopEmitter::LoopEmitter(const ElementGenerator& target_element_generator, } } -IrArray::Index LoopEmitter::EmitIndexAndSetExitBasicBlock( +std::vector LoopEmitter::EmitIndexAndSetExitBasicBlock( tensorflow::StringPiece loop_name) { if (ShapeUtil::IsScalar(shape_)) { // No loop needed, so set exit_bb_ to nullptr. exit_bb_ = nullptr; - return IrArray::Index(); + return {IrArray::Index()}; } // Create loop nest with one for-loop for each dimension of the target shape. @@ -121,12 +121,14 @@ IrArray::Index LoopEmitter::EmitIndexAndSetExitBasicBlock( exit_bb_ = loop_nest.GetOuterLoopExitBasicBlock(); CHECK_NOTNULL(exit_bb_); - return array_index; + return {array_index}; } tensorflow::Status LoopEmitter::EmitLoop(tensorflow::StringPiece loop_name) { - IrArray::Index array_index = EmitIndexAndSetExitBasicBlock(loop_name); - TF_RETURN_IF_ERROR(body_emitter_(array_index)); + for (const IrArray::Index& array_index : + EmitIndexAndSetExitBasicBlock(loop_name)) { + TF_RETURN_IF_ERROR(body_emitter_(array_index)); + } // Set the insertion point of ir_builder_ to the loop exit, so that // code emitted for later instructions will be correctly placed. diff --git a/tensorflow/compiler/xla/service/llvm_ir/loop_emitter.h b/tensorflow/compiler/xla/service/llvm_ir/loop_emitter.h index 0fc528439a..9ff497aecd 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/loop_emitter.h +++ b/tensorflow/compiler/xla/service/llvm_ir/loop_emitter.h @@ -63,11 +63,12 @@ class LoopEmitter { // Emits a loop nest (with a yet-to-be-filled loop body) that iterates through // every element in the given shape. Returns the multi-dimensional index that - // specifies the element. - IrArray::Index EmitIndexAndSetExitBasicBlock() { + // specifies the element, will return multiple indices if the loop is + // unrolled. + std::vector EmitIndexAndSetExitBasicBlock() { return EmitIndexAndSetExitBasicBlock(/*loop_name=*/""); } - virtual IrArray::Index EmitIndexAndSetExitBasicBlock( + virtual std::vector EmitIndexAndSetExitBasicBlock( tensorflow::StringPiece loop_name); // Emits a complete loop nest for every element in the given shape. diff --git a/tensorflow/compiler/xla/xla.proto b/tensorflow/compiler/xla/xla.proto index b4cbdf3773..f619b8dc24 100644 --- a/tensorflow/compiler/xla/xla.proto +++ b/tensorflow/compiler/xla/xla.proto @@ -192,6 +192,9 @@ message DebugOptions { // Generate calls to MKL-DNN in the CPU backend. bool xla_cpu_use_mkl_dnn = 97; + // Maximum kernel unroll factor for the GPU backend. + int32 xla_gpu_max_kernel_unroll_factor = 98; + // Extra options to pass to the compilation backend; specific interpretation // of these values is left to the backend. map xla_backend_extra_options = 500; -- GitLab From 934e383628a94f9994e7da2dea706a92e8eeffb3 Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Tue, 10 Apr 2018 09:40:25 -0700 Subject: [PATCH 085/791] Fix capitalization (on master) PiperOrigin-RevId: 192298563 --- tensorflow/docs_src/programmers_guide/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/docs_src/programmers_guide/index.md b/tensorflow/docs_src/programmers_guide/index.md index 017db0e8cb..648d001bd3 100644 --- a/tensorflow/docs_src/programmers_guide/index.md +++ b/tensorflow/docs_src/programmers_guide/index.md @@ -5,7 +5,7 @@ works. The units are as follows: ## High Level APIs - * @{$programmers_guide/eager}, which is the easiest way to use tensorflow. + * @{$programmers_guide/eager}, which is the easiest way to use TensorFlow. * @{$programmers_guide/estimators}, which introduces a high-level TensorFlow API that greatly simplifies ML programming. * @{$programmers_guide/datasets}, which explains how to -- GitLab From a0410fb4c5be8fd455640507146cbcd0b0b7a2f1 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 09:41:45 -0700 Subject: [PATCH 086/791] Remove manifest_merger from tensorflow_demo. This is an internal-only attribute that is being removed from bazel. PiperOrigin-RevId: 192298746 --- tensorflow/examples/android/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/examples/android/BUILD b/tensorflow/examples/android/BUILD index a088d7cf2f..aa594a63c6 100644 --- a/tensorflow/examples/android/BUILD +++ b/tensorflow/examples/android/BUILD @@ -76,7 +76,6 @@ android_binary( custom_package = "org.tensorflow.demo", inline_constants = 1, manifest = "AndroidManifest.xml", - manifest_merger = "legacy", resource_files = glob(["res/**"]), tags = [ "manual", -- GitLab From 62b6b3009b6806d27536da58b11373cf97cdab7d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 09:50:02 -0700 Subject: [PATCH 087/791] Fix markdown in a couple of tf.estimator docstrings. PiperOrigin-RevId: 192299871 --- tensorflow/python/estimator/run_config.py | 2 +- tensorflow/python/estimator/training.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/estimator/run_config.py b/tensorflow/python/estimator/run_config.py index f62c9cece6..dab442aeda 100644 --- a/tensorflow/python/estimator/run_config.py +++ b/tensorflow/python/estimator/run_config.py @@ -316,7 +316,7 @@ class RunConfig(object): a list of task addresses. `task` has two attributes: `type` and `index`, where `type` can be any of - the task types in `cluster`. ` When `TF_CONFIG` contains said information, + the task types in `cluster`. When `TF_CONFIG` contains said information, the following properties are set on this class: * `cluster_spec` is parsed from `TF_CONFIG['cluster']`. Defaults to {}. If diff --git a/tensorflow/python/estimator/training.py b/tensorflow/python/estimator/training.py index e38b765da5..9d271758f6 100644 --- a/tensorflow/python/estimator/training.py +++ b/tensorflow/python/estimator/training.py @@ -137,7 +137,7 @@ class TrainSpec( * A tuple (features, labels): Where features is a `Tensor` or a dictionary of string feature name to `Tensor` and labels is a `Tensor` or a dictionary of string label name to `Tensor`. - + max_steps: Int. Positive number of total steps for which to train model. If `None`, train forever. The training `input_fn` is not expected to generate `OutOfRangeError` or `StopIteration` exceptions. See the @@ -334,7 +334,8 @@ def train_and_evaluate(estimator, train_spec, eval_spec): can read and write). The only extra work to do is setting the environment variable `TF_CONFIG` properly for each worker correspondingly. - Also see: https://www.tensorflow.org/deploy/distributed + Also see + [Distributed TensorFlow](https://www.tensorflow.org/deploy/distributed). Setting environment variable depends on the platform. For example, on Linux, it can be done as follows (`$` is the shell prompt): -- GitLab From 6b0ec4bab215169c8cde893b022288f8bf7c8835 Mon Sep 17 00:00:00 2001 From: cclauss Date: Tue, 10 Apr 2018 18:56:52 +0200 Subject: [PATCH 088/791] raw_input() was removed in Python 3 (#16440) --- .../python/keras/_impl/keras/utils/io_utils.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tensorflow/python/keras/_impl/keras/utils/io_utils.py b/tensorflow/python/keras/_impl/keras/utils/io_utils.py index bbf1d2a3d9..f82e3277de 100644 --- a/tensorflow/python/keras/_impl/keras/utils/io_utils.py +++ b/tensorflow/python/keras/_impl/keras/utils/io_utils.py @@ -19,9 +19,9 @@ from __future__ import division from __future__ import print_function from collections import defaultdict -import sys import numpy as np +import six from tensorflow.python.util.tf_export import tf_export @@ -160,13 +160,11 @@ def ask_to_proceed_with_overwrite(filepath): Returns: True if we can proceed with overwrite, False otherwise. """ - get_input = input - if sys.version_info[:2] <= (2, 7): - get_input = raw_input - overwrite = get_input('[WARNING] %s already exists - overwrite? ' - '[y/n]' % (filepath)) - while overwrite not in ['y', 'n']: - overwrite = get_input('Enter "y" (overwrite) or "n" (cancel).') + overwrite = six.moves.input('[WARNING] %s already exists - overwrite? ' + '[y/n]' % (filepath)).strip().lower() + while overwrite not in ('y', 'n'): + overwrite = six.moves.input('Enter "y" (overwrite) or "n" ' + '(cancel).').strip().lower() if overwrite == 'n': return False print('[TIP] Next time specify overwrite=True!') -- GitLab From bd718c410478d066ed1c41d5ffe31970075b808a Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Tue, 10 Apr 2018 10:07:31 -0700 Subject: [PATCH 089/791] Place data format op on CPU:0. PiperOrigin-RevId: 192302833 --- tensorflow/core/grappler/optimizers/layout_optimizer.cc | 1 + tensorflow/core/grappler/optimizers/layout_optimizer_test.cc | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer.cc b/tensorflow/core/grappler/optimizers/layout_optimizer.cc index 561226f945..8fb30d116d 100644 --- a/tensorflow/core/grappler/optimizers/layout_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/layout_optimizer.cc @@ -919,6 +919,7 @@ class NodeProcessor : public GraphProcessor { ParseNodeName(input_name, &port); if (IsHostMemory(*input, port)) { parsed_name.type = "CPU"; + parsed_name.id = 0; device = DeviceNameUtils::ParsedNameToString(parsed_name); } } diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc b/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc index 260347b0e8..b913f2b004 100644 --- a/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc +++ b/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc @@ -36,7 +36,7 @@ class LayoutOptimizerTest : public ::testing::Test { DeviceProperties device_properties; device_properties.set_type("GPU"); device_properties.mutable_environment()->insert({"architecture", "6"}); - virtual_cluster_.reset(new VirtualCluster({{"/GPU:0", device_properties}})); + virtual_cluster_.reset(new VirtualCluster({{"/GPU:1", device_properties}})); } Output SimpleConv2D(tensorflow::Scope* s, int input_size, int filter_size, -- GitLab From 049dfd5e070cfa84c82eea71c6c746a70cba4a3f Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 10 Apr 2018 10:34:32 -0700 Subject: [PATCH 090/791] Add python built-in types support for `tf.as_dtype` (#17652) * Add python built-in types support for `tf.as_dtype` This fix tries to address the issue raised in 17641 where it was not possible to use `tf.as_dtype(float)` the same way as numpy `np.dtype(float)`. This fix adds the built-in types support for `tf.as_dtype`, so that it is possible to specify: ``` dtypes.as_dtype(float) # dtypes.float64 dtypes.as_dtype(int) # dtypes.int32 dtypes.as_dtype(long) # dtypes.int64 dtypes.as_dtype(complex) # dtypes.complex128 dtypes.as_dtype(bool) # dtypes.bool ``` This fix fixes 17641. Signed-off-by: Yong Tang * Add test cases for built-in types support with `tf.as_dtype` Signed-off-by: Yong Tang * Fix failed test cases with added built-in types support of tf.as_dtype Signed-off-by: Yong Tang * Fix python 3 build Signed-off-by: Yong Tang * Restrict the changes to float and bool based on review feedback Signed-off-by: Yong Tang --- tensorflow/python/framework/dtypes.py | 9 +++++++++ tensorflow/python/framework/dtypes_test.py | 3 +++ 2 files changed, 12 insertions(+) diff --git a/tensorflow/python/framework/dtypes.py b/tensorflow/python/framework/dtypes.py index a31c424263..6d918f8b89 100644 --- a/tensorflow/python/framework/dtypes.py +++ b/tensorflow/python/framework/dtypes.py @@ -648,6 +648,10 @@ QUANTIZED_DTYPES = frozenset([ ]) tf_export("QUANTIZED_DTYPES").export_constant(__name__, "QUANTIZED_DTYPES") +_PYTHON_TO_TF = { + float: float32, + bool: bool, +} @tf_export("as_dtype") def as_dtype(type_value): @@ -679,6 +683,11 @@ def as_dtype(type_value): except KeyError: pass + try: + return _PYTHON_TO_TF[type_value] + except KeyError: + pass + if isinstance(type_value, np.dtype): # The numpy dtype for strings is variable length. We can not compare # dtype with a single constant (np.string does not exist) to decide diff --git a/tensorflow/python/framework/dtypes_test.py b/tensorflow/python/framework/dtypes_test.py index e49e2fda5d..478733e389 100644 --- a/tensorflow/python/framework/dtypes_test.py +++ b/tensorflow/python/framework/dtypes_test.py @@ -295,6 +295,9 @@ class TypesTest(test_util.TensorFlowTestCase): self.assertNotEqual(dtypes.int32, int) self.assertNotEqual(dtypes.float64, 2.1) + def testPythonTypesConversion(self): + self.assertIs(dtypes.float32, dtypes.as_dtype(float)) + self.assertIs(dtypes.bool, dtypes.as_dtype(bool)) if __name__ == "__main__": googletest.main() -- GitLab From 36a07c59954b8ace54879b8732b6a7ae2dce6450 Mon Sep 17 00:00:00 2001 From: Asim Shankar Date: Tue, 10 Apr 2018 10:43:14 -0700 Subject: [PATCH 091/791] Simplify test_util.run_in_graph_and_eager_modes - Get rid of unnecessary options - Update various resource variable tests so that they correctly exercise the cases where the variables are placed on GPU (these "with tf.device('/cpu:0')" blocks that were added for eager execution are no longer necessary) PiperOrigin-RevId: 192309109 --- .../eager/python/checkpointable_utils_test.py | 10 +- .../contrib/optimizer_v2/momentum_test.py | 24 +- tensorflow/python/framework/test_util.py | 90 +++-- .../_impl/keras/layers/embeddings_test.py | 2 +- .../keras/_impl/keras/layers/pooling_test.py | 18 +- .../resource_variable_ops_test.py | 340 ++++++++---------- tensorflow/python/training/momentum_test.py | 24 +- 7 files changed, 256 insertions(+), 252 deletions(-) diff --git a/tensorflow/contrib/eager/python/checkpointable_utils_test.py b/tensorflow/contrib/eager/python/checkpointable_utils_test.py index e6498ddb06..3ec5c3de39 100644 --- a/tensorflow/contrib/eager/python/checkpointable_utils_test.py +++ b/tensorflow/contrib/eager/python/checkpointable_utils_test.py @@ -719,8 +719,9 @@ class CheckpointingTests(test.TestCase): checkpoint_directory = self.get_temp_dir() root = checkpointable.Checkpointable() - root.var = checkpointable_utils.add_variable( - root, name="var", initializer=0.) + with ops.device("/cpu:0"): + root.var = checkpointable_utils.add_variable( + root, name="var", initializer=0.) optimizer = adam.AdamOptimizer(0.1) if context.executing_eagerly(): optimizer.minimize(root.var.read_value) @@ -750,8 +751,9 @@ class CheckpointingTests(test.TestCase): new_root).restore(no_slots_path) with self.assertRaises(AssertionError): no_slot_status.assert_consumed() - new_root.var = checkpointable_utils.add_variable( - new_root, name="var", shape=[]) + with ops.device("/cpu:0"): + new_root.var = checkpointable_utils.add_variable( + new_root, name="var", shape=[]) no_slot_status.assert_consumed() no_slot_status.run_restore_ops() self.assertEqual(12., self.evaluate(new_root.var)) diff --git a/tensorflow/contrib/optimizer_v2/momentum_test.py b/tensorflow/contrib/optimizer_v2/momentum_test.py index f37eb48181..26724f66c2 100644 --- a/tensorflow/contrib/optimizer_v2/momentum_test.py +++ b/tensorflow/contrib/optimizer_v2/momentum_test.py @@ -237,7 +237,17 @@ class MomentumOptimizerTest(test.TestCase): @test_util.run_in_graph_and_eager_modes(reset_test=True) def testMinimizeSparseResourceVariable(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype) + # This test invokes the ResourceSparseApplyMomentum operation, which + # did not have a registered GPU kernel as of April 2018. With graph + # execution, the placement algorithm notices this and automatically + # places the variable in CPU (host) memory. With eager execution, + # the variable would be placed in GPU memory if available, which + # would then conflict with the future invocation of the + # ResourceSparseApplyMomentum operation. + # To work around this discrepancy, for now we force the variable + # to be placed on CPU. + with ops.device("/cpu:0"): + var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype) # pylint: disable=cell-var-from-loop def loss(): @@ -256,7 +266,17 @@ class MomentumOptimizerTest(test.TestCase): @test_util.run_in_graph_and_eager_modes(reset_test=True) def testMinimizeWith2DIndiciesForEmbeddingLookup(self): - var0 = resource_variable_ops.ResourceVariable(array_ops.ones([2, 2])) + # This test invokes the ResourceSparseApplyMomentum operation, which + # did not have a registered GPU kernel as of April 2018. With graph + # execution, the placement algorithm notices this and automatically + # places the variable in CPU (host) memory. With eager execution, + # the variable would be placed in GPU memory if available, which + # would then conflict with the future invocation of the + # ResourceSparseApplyMomentum operation. + # To work around this discrepancy, for now we force the variable + # to be placed on CPU. + with ops.device("/cpu:0"): + var0 = resource_variable_ops.ResourceVariable(array_ops.ones([2, 2])) def loss(): return math_ops.reduce_sum(embedding_ops.embedding_lookup(var0, [[1]])) diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py index bf00fa6439..eea27d76c6 100644 --- a/tensorflow/python/framework/test_util.py +++ b/tensorflow/python/framework/test_util.py @@ -615,45 +615,68 @@ def assert_no_garbage_created(f): def run_in_graph_and_eager_modes(__unused__=None, - graph=None, config=None, - use_gpu=False, - force_gpu=False, + use_gpu=True, reset_test=True, assert_no_eager_garbage=False): - """Runs the test in both graph and eager modes. + """Execute the decorated test with and without enabling eager execution. + + This function returns a decorator intended to be applied to test methods in + a @{tf.test.TestCase} class. Doing so will cause the contents of the test + method to be executed twice - once normally, and once with eager execution + enabled. This allows unittests to confirm the equivalence between eager + and graph execution (see @{tf.enable_eager_execution}). + + For example, consider the following unittest: + + ```python + class MyTests(tf.test.TestCase): + + @run_in_graph_and_eager_modes() + def test_foo(self): + x = tf.constant([1, 2]) + y = tf.constant([3, 4]) + z = tf.add(x, y) + self.assertAllEqual([4, 6], self.evaluate(z)) + + if __name__ == "__main__": + tf.test.main() + ``` + + This test validates that `tf.add()` has the same behavior when computed with + eager execution enabled as it does when constructing a TensorFlow graph and + executing the `z` tensor in a session. + Args: __unused__: Prevents sliently skipping tests. - graph: Optional graph to use during the returned session. config: An optional config_pb2.ConfigProto to use to configure the - session. - use_gpu: If True, attempt to run as many ops as possible on GPU. - force_gpu: If True, pin all ops to `/device:GPU:0`. - reset_test: If True, tearDown and SetUp the test case again. + session when executing graphs. + use_gpu: If True, attempt to run as many operations as possible on GPU. + reset_test: If True, tearDown and SetUp the test case between the two + executions of the test (once with and once without eager execution). assert_no_eager_garbage: If True, sets DEBUG_SAVEALL on the garbage collector and asserts that no extra garbage has been created when running - the test in eager mode. This will fail if there are reference cycles - (e.g. a = []; a.append(a)). Off by default because some tests may create - garbage for legitimate reasons (e.g. they define a class which inherits - from `object`), and because DEBUG_SAVEALL is sticky in some Python - interpreters (meaning that tests which rely on objects being collected - elsewhere in the unit test file will not work). Additionally, checks that - nothing still has a reference to Tensors that the test allocated. + the test with eager execution enabled. This will fail if there are + reference cycles (e.g. a = []; a.append(a)). Off by default because some + tests may create garbage for legitimate reasons (e.g. they define a class + which inherits from `object`), and because DEBUG_SAVEALL is sticky in some + Python interpreters (meaning that tests which rely on objects being + collected elsewhere in the unit test file will not work). Additionally, + checks that nothing still has a reference to Tensors that the test + allocated. Returns: - Returns a decorator that will run the decorated test function - using both a graph and using eager execution. + Returns a decorator that will run the decorated test method twice: + once by constructing and executing a graph in a session and once with + eager execution enabled. """ assert not __unused__, "Add () after run_in_graph_and_eager_modes." def decorator(f): - """Test method decorator.""" - def decorated(self, **kwargs): - """Decorated the test method.""" with context.graph_mode(): - with self.test_session(graph, config, use_gpu, force_gpu): + with self.test_session(use_gpu=use_gpu): f(self, **kwargs) if reset_test: @@ -663,27 +686,20 @@ def run_in_graph_and_eager_modes(__unused__=None, self._tempdir = None self.setUp() - def run_eager_mode(self, **kwargs): - if force_gpu: - gpu_name = gpu_device_name() - if not gpu_name: - gpu_name = "/device:GPU:0" - with context.device(gpu_name): - f(self) - elif use_gpu: - # TODO(xpan): Support softplacement and gpu by default when available. - f(self, **kwargs) - else: - with context.device("/device:CPU:0"): + def run_eagerly(self, **kwargs): + if not use_gpu: + with ops.device("/cpu:0"): f(self, **kwargs) + else: + f(self, **kwargs) if assert_no_eager_garbage: - run_eager_mode = assert_no_new_tensors( - assert_no_garbage_created(run_eager_mode)) + run_eagerly = assert_no_new_tensors( + assert_no_garbage_created(run_eagerly)) with context.eager_mode(): with ops.Graph().as_default(): - run_eager_mode(self, **kwargs) + run_eagerly(self, **kwargs) return decorated diff --git a/tensorflow/python/keras/_impl/keras/layers/embeddings_test.py b/tensorflow/python/keras/_impl/keras/layers/embeddings_test.py index 26fd1f1c11..9f6793eac8 100644 --- a/tensorflow/python/keras/_impl/keras/layers/embeddings_test.py +++ b/tensorflow/python/keras/_impl/keras/layers/embeddings_test.py @@ -26,7 +26,7 @@ from tensorflow.python.platform import test class EmbeddingTest(test.TestCase): - @tf_test_util.run_in_graph_and_eager_modes() + @tf_test_util.run_in_graph_and_eager_modes(use_gpu=False) def test_embedding(self): testing_utils.layer_test( keras.layers.Embedding, diff --git a/tensorflow/python/keras/_impl/keras/layers/pooling_test.py b/tensorflow/python/keras/_impl/keras/layers/pooling_test.py index bb003c1ddd..2c08b647ea 100644 --- a/tensorflow/python/keras/_impl/keras/layers/pooling_test.py +++ b/tensorflow/python/keras/_impl/keras/layers/pooling_test.py @@ -27,14 +27,14 @@ from tensorflow.python.platform import test class GlobalPoolingTest(test.TestCase): - @tf_test_util.run_in_graph_and_eager_modes(use_gpu=True) + @tf_test_util.run_in_graph_and_eager_modes() def test_globalpooling_1d(self): testing_utils.layer_test(keras.layers.pooling.GlobalMaxPooling1D, input_shape=(3, 4, 5)) testing_utils.layer_test( keras.layers.pooling.GlobalAveragePooling1D, input_shape=(3, 4, 5)) - @tf_test_util.run_in_graph_and_eager_modes(use_gpu=True) + @tf_test_util.run_in_graph_and_eager_modes() def test_globalpooling_2d(self): testing_utils.layer_test( keras.layers.pooling.GlobalMaxPooling2D, @@ -53,7 +53,7 @@ class GlobalPoolingTest(test.TestCase): kwargs={'data_format': 'channels_last'}, input_shape=(3, 5, 6, 4)) - @tf_test_util.run_in_graph_and_eager_modes(use_gpu=True) + @tf_test_util.run_in_graph_and_eager_modes() def test_globalpooling_3d(self): testing_utils.layer_test( keras.layers.pooling.GlobalMaxPooling3D, @@ -75,7 +75,7 @@ class GlobalPoolingTest(test.TestCase): class Pooling2DTest(test.TestCase): - @tf_test_util.run_in_graph_and_eager_modes(use_gpu=True) + @tf_test_util.run_in_graph_and_eager_modes() def test_maxpooling_2d(self): pool_size = (3, 3) for strides in [(1, 1), (2, 2)]: @@ -88,7 +88,7 @@ class Pooling2DTest(test.TestCase): }, input_shape=(3, 5, 6, 4)) - @tf_test_util.run_in_graph_and_eager_modes(use_gpu=True) + @tf_test_util.run_in_graph_and_eager_modes() def test_averagepooling_2d(self): testing_utils.layer_test( keras.layers.AveragePooling2D, @@ -122,7 +122,7 @@ class Pooling2DTest(test.TestCase): class Pooling3DTest(test.TestCase): - @tf_test_util.run_in_graph_and_eager_modes(use_gpu=True) + @tf_test_util.run_in_graph_and_eager_modes() def test_maxpooling_3d(self): pool_size = (3, 3, 3) testing_utils.layer_test( @@ -141,7 +141,7 @@ class Pooling3DTest(test.TestCase): }, input_shape=(3, 4, 11, 12, 10)) - @tf_test_util.run_in_graph_and_eager_modes(use_gpu=True) + @tf_test_util.run_in_graph_and_eager_modes() def test_averagepooling_3d(self): pool_size = (3, 3, 3) testing_utils.layer_test( @@ -163,7 +163,7 @@ class Pooling3DTest(test.TestCase): class Pooling1DTest(test.TestCase): - @tf_test_util.run_in_graph_and_eager_modes(use_gpu=True) + @tf_test_util.run_in_graph_and_eager_modes() def test_maxpooling_1d(self): for padding in ['valid', 'same']: for stride in [1, 2]: @@ -173,7 +173,7 @@ class Pooling1DTest(test.TestCase): 'padding': padding}, input_shape=(3, 5, 4)) - @tf_test_util.run_in_graph_and_eager_modes(use_gpu=True) + @tf_test_util.run_in_graph_and_eager_modes() def test_averagepooling_1d(self): for padding in ['valid', 'same']: for stride in [1, 2]: diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py index edc63264a3..6d33086936 100644 --- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py +++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py @@ -174,215 +174,161 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)) self.assertEqual(read, 2) - @test_util.run_in_graph_and_eager_modes(use_gpu=True) + @test_util.run_in_graph_and_eager_modes() def testScatterAdd(self): - with ops.device("cpu:0"): - handle = resource_variable_ops.var_handle_op( - dtype=dtypes.int32, shape=[1, 1]) - self.evaluate(resource_variable_ops.assign_variable_op( - handle, constant_op.constant([[1]], dtype=dtypes.int32))) - self.evaluate(resource_variable_ops.resource_scatter_add( - handle, [0], constant_op.constant([[2]], dtype=dtypes.int32))) - read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertEqual(self.evaluate(read), [[3]]) - - @test_util.run_in_graph_and_eager_modes(use_gpu=True) + handle = resource_variable_ops.var_handle_op( + dtype=dtypes.int32, shape=[1, 1]) + self.evaluate( + resource_variable_ops.assign_variable_op( + handle, constant_op.constant([[1]], dtype=dtypes.int32))) + self.evaluate( + resource_variable_ops.resource_scatter_add( + handle, [0], constant_op.constant([[2]], dtype=dtypes.int32))) + read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) + self.assertEqual(self.evaluate(read), [[3]]) + + @test_util.run_in_graph_and_eager_modes() def testScatterSub(self): - with ops.device("cpu:0"): - handle = resource_variable_ops.var_handle_op( - dtype=dtypes.int32, shape=[1, 1]) - self.evaluate( - resource_variable_ops.assign_variable_op(handle, - constant_op.constant( - [[1]], - dtype=dtypes.int32))) - self.evaluate( - resource_variable_ops.resource_scatter_sub(handle, [0], - constant_op.constant( - [[2]], - dtype=dtypes.int32))) - read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertEqual(self.evaluate(read), [[-1]]) - - @test_util.run_in_graph_and_eager_modes(use_gpu=True) + handle = resource_variable_ops.var_handle_op( + dtype=dtypes.int32, shape=[1, 1]) + self.evaluate( + resource_variable_ops.assign_variable_op( + handle, constant_op.constant([[1]], dtype=dtypes.int32))) + self.evaluate( + resource_variable_ops.resource_scatter_sub( + handle, [0], constant_op.constant([[2]], dtype=dtypes.int32))) + read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) + self.assertEqual(self.evaluate(read), [[-1]]) + + @test_util.run_in_graph_and_eager_modes() def testScatterMul(self): - with ops.device("cpu:0"): - handle = resource_variable_ops.var_handle_op( - dtype=dtypes.int32, shape=[1, 1]) - self.evaluate( - resource_variable_ops.assign_variable_op(handle, - constant_op.constant( - [[1]], - dtype=dtypes.int32))) - self.evaluate( - resource_variable_ops.resource_scatter_mul(handle, [0], - constant_op.constant( - [[5]], - dtype=dtypes.int32))) - read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertEqual(self.evaluate(read), [[5]]) - - @test_util.run_in_graph_and_eager_modes(use_gpu=True) + handle = resource_variable_ops.var_handle_op( + dtype=dtypes.int32, shape=[1, 1]) + self.evaluate( + resource_variable_ops.assign_variable_op( + handle, constant_op.constant([[1]], dtype=dtypes.int32))) + self.evaluate( + resource_variable_ops.resource_scatter_mul( + handle, [0], constant_op.constant([[5]], dtype=dtypes.int32))) + read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) + self.assertEqual(self.evaluate(read), [[5]]) + + @test_util.run_in_graph_and_eager_modes() def testScatterDiv(self): - with ops.device("cpu:0"): - handle = resource_variable_ops.var_handle_op( - dtype=dtypes.int32, shape=[1, 1]) - self.evaluate( - resource_variable_ops.assign_variable_op(handle, - constant_op.constant( - [[6]], - dtype=dtypes.int32))) - self.evaluate( - resource_variable_ops.resource_scatter_div(handle, [0], - constant_op.constant( - [[3]], - dtype=dtypes.int32))) - read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertEqual(self.evaluate(read), [[2]]) - - @test_util.run_in_graph_and_eager_modes(use_gpu=True) + handle = resource_variable_ops.var_handle_op( + dtype=dtypes.int32, shape=[1, 1]) + self.evaluate( + resource_variable_ops.assign_variable_op( + handle, constant_op.constant([[6]], dtype=dtypes.int32))) + self.evaluate( + resource_variable_ops.resource_scatter_div( + handle, [0], constant_op.constant([[3]], dtype=dtypes.int32))) + read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) + self.assertEqual(self.evaluate(read), [[2]]) + + @test_util.run_in_graph_and_eager_modes() def testScatterMin(self): - with ops.device("cpu:0"): - handle = resource_variable_ops.var_handle_op( - dtype=dtypes.int32, shape=[1, 1]) - self.evaluate( - resource_variable_ops.assign_variable_op(handle, - constant_op.constant( - [[6]], - dtype=dtypes.int32))) - self.evaluate( - resource_variable_ops.resource_scatter_min(handle, [0], - constant_op.constant( - [[3]], - dtype=dtypes.int32))) - read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertEqual(self.evaluate(read), [[3]]) - - @test_util.run_in_graph_and_eager_modes(use_gpu=True) + handle = resource_variable_ops.var_handle_op( + dtype=dtypes.int32, shape=[1, 1]) + self.evaluate( + resource_variable_ops.assign_variable_op( + handle, constant_op.constant([[6]], dtype=dtypes.int32))) + self.evaluate( + resource_variable_ops.resource_scatter_min( + handle, [0], constant_op.constant([[3]], dtype=dtypes.int32))) + read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) + self.assertEqual(self.evaluate(read), [[3]]) + + @test_util.run_in_graph_and_eager_modes() def testScatterMax(self): - with ops.device("cpu:0"): - handle = resource_variable_ops.var_handle_op( - dtype=dtypes.int32, shape=[1, 1]) - self.evaluate( - resource_variable_ops.assign_variable_op(handle, - constant_op.constant( - [[6]], - dtype=dtypes.int32))) - self.evaluate( - resource_variable_ops.resource_scatter_max(handle, [0], - constant_op.constant( - [[3]], - dtype=dtypes.int32))) - read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertEqual(self.evaluate(read), [[6]]) - - @test_util.run_in_graph_and_eager_modes(use_gpu=True) + handle = resource_variable_ops.var_handle_op( + dtype=dtypes.int32, shape=[1, 1]) + self.evaluate( + resource_variable_ops.assign_variable_op( + handle, constant_op.constant([[6]], dtype=dtypes.int32))) + self.evaluate( + resource_variable_ops.resource_scatter_max( + handle, [0], constant_op.constant([[3]], dtype=dtypes.int32))) + read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) + self.assertEqual(self.evaluate(read), [[6]]) + + @test_util.run_in_graph_and_eager_modes() def testScatterAddScalar(self): - with ops.device("cpu:0"): - handle = resource_variable_ops.var_handle_op( - dtype=dtypes.int32, shape=[1, 1]) - self.evaluate( - resource_variable_ops.assign_variable_op(handle, - constant_op.constant( - [[1]], - dtype=dtypes.int32))) - self.evaluate( - resource_variable_ops.resource_scatter_add(handle, [0], - constant_op.constant( - 2, - dtype=dtypes.int32))) - read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertEqual(self.evaluate(read), [[3]]) - - @test_util.run_in_graph_and_eager_modes(use_gpu=True) + handle = resource_variable_ops.var_handle_op( + dtype=dtypes.int32, shape=[1, 1]) + self.evaluate( + resource_variable_ops.assign_variable_op( + handle, constant_op.constant([[1]], dtype=dtypes.int32))) + self.evaluate( + resource_variable_ops.resource_scatter_add( + handle, [0], constant_op.constant(2, dtype=dtypes.int32))) + read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) + self.assertEqual(self.evaluate(read), [[3]]) + + @test_util.run_in_graph_and_eager_modes() def testScatterSubScalar(self): - with ops.device("cpu:0"): - handle = resource_variable_ops.var_handle_op( - dtype=dtypes.int32, shape=[1, 1]) - self.evaluate( - resource_variable_ops.assign_variable_op(handle, - constant_op.constant( - [[1]], - dtype=dtypes.int32))) - self.evaluate( - resource_variable_ops.resource_scatter_sub(handle, [0], - constant_op.constant( - 2, - dtype=dtypes.int32))) - read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertEqual(self.evaluate(read), [[-1]]) - - @test_util.run_in_graph_and_eager_modes(use_gpu=True) + handle = resource_variable_ops.var_handle_op( + dtype=dtypes.int32, shape=[1, 1]) + self.evaluate( + resource_variable_ops.assign_variable_op( + handle, constant_op.constant([[1]], dtype=dtypes.int32))) + self.evaluate( + resource_variable_ops.resource_scatter_sub( + handle, [0], constant_op.constant(2, dtype=dtypes.int32))) + read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) + self.assertEqual(self.evaluate(read), [[-1]]) + + @test_util.run_in_graph_and_eager_modes() def testScatterMulScalar(self): - with ops.device("cpu:0"): - handle = resource_variable_ops.var_handle_op( - dtype=dtypes.int32, shape=[1, 1]) - self.evaluate( - resource_variable_ops.assign_variable_op(handle, - constant_op.constant( - [[1]], - dtype=dtypes.int32))) - self.evaluate( - resource_variable_ops.resource_scatter_mul(handle, [0], - constant_op.constant( - 5, - dtype=dtypes.int32))) - read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertEqual(self.evaluate(read), [[5]]) - - @test_util.run_in_graph_and_eager_modes(use_gpu=True) + handle = resource_variable_ops.var_handle_op( + dtype=dtypes.int32, shape=[1, 1]) + self.evaluate( + resource_variable_ops.assign_variable_op( + handle, constant_op.constant([[1]], dtype=dtypes.int32))) + self.evaluate( + resource_variable_ops.resource_scatter_mul( + handle, [0], constant_op.constant(5, dtype=dtypes.int32))) + read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) + self.assertEqual(self.evaluate(read), [[5]]) + + @test_util.run_in_graph_and_eager_modes() def testScatterDivScalar(self): - with ops.device("cpu:0"): - handle = resource_variable_ops.var_handle_op( - dtype=dtypes.int32, shape=[1, 1]) - self.evaluate( - resource_variable_ops.assign_variable_op(handle, - constant_op.constant( - [[6]], - dtype=dtypes.int32))) - self.evaluate( - resource_variable_ops.resource_scatter_div(handle, [0], - constant_op.constant( - 3, - dtype=dtypes.int32))) - read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertEqual(self.evaluate(read), [[2]]) - - @test_util.run_in_graph_and_eager_modes(use_gpu=True) + handle = resource_variable_ops.var_handle_op( + dtype=dtypes.int32, shape=[1, 1]) + self.evaluate( + resource_variable_ops.assign_variable_op( + handle, constant_op.constant([[6]], dtype=dtypes.int32))) + self.evaluate( + resource_variable_ops.resource_scatter_div( + handle, [0], constant_op.constant(3, dtype=dtypes.int32))) + read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) + self.assertEqual(self.evaluate(read), [[2]]) + + @test_util.run_in_graph_and_eager_modes() def testScatterMinScalar(self): - with ops.device("cpu:0"): - handle = resource_variable_ops.var_handle_op( - dtype=dtypes.int32, shape=[1, 1]) - self.evaluate( - resource_variable_ops.assign_variable_op(handle, - constant_op.constant( - [[6]], - dtype=dtypes.int32))) - self.evaluate( - resource_variable_ops.resource_scatter_min(handle, [0], - constant_op.constant( - 3, - dtype=dtypes.int32))) - read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertEqual(self.evaluate(read), [[3]]) - - @test_util.run_in_graph_and_eager_modes(use_gpu=True) + handle = resource_variable_ops.var_handle_op( + dtype=dtypes.int32, shape=[1, 1]) + self.evaluate( + resource_variable_ops.assign_variable_op( + handle, constant_op.constant([[6]], dtype=dtypes.int32))) + self.evaluate( + resource_variable_ops.resource_scatter_min( + handle, [0], constant_op.constant(3, dtype=dtypes.int32))) + read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) + self.assertEqual(self.evaluate(read), [[3]]) + + @test_util.run_in_graph_and_eager_modes() def testScatterMaxScalar(self): - with ops.device("cpu:0"): - handle = resource_variable_ops.var_handle_op( - dtype=dtypes.int32, shape=[1, 1]) - self.evaluate( - resource_variable_ops.assign_variable_op(handle, - constant_op.constant( - [[6]], - dtype=dtypes.int32))) - self.evaluate( - resource_variable_ops.resource_scatter_max(handle, [0], - constant_op.constant( - 3, - dtype=dtypes.int32))) - read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertEqual(self.evaluate(read), [[6]]) + handle = resource_variable_ops.var_handle_op( + dtype=dtypes.int32, shape=[1, 1]) + self.evaluate( + resource_variable_ops.assign_variable_op( + handle, constant_op.constant([[6]], dtype=dtypes.int32))) + self.evaluate( + resource_variable_ops.resource_scatter_max( + handle, [0], constant_op.constant(3, dtype=dtypes.int32))) + read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) + self.assertEqual(self.evaluate(read), [[6]]) def testScatterUpdateString(self): handle = resource_variable_ops.var_handle_op( diff --git a/tensorflow/python/training/momentum_test.py b/tensorflow/python/training/momentum_test.py index 297a8bbde5..7bd57ad3d8 100644 --- a/tensorflow/python/training/momentum_test.py +++ b/tensorflow/python/training/momentum_test.py @@ -237,7 +237,17 @@ class MomentumOptimizerTest(test.TestCase): @test_util.run_in_graph_and_eager_modes(reset_test=True) def testMinimizeSparseResourceVariable(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype) + # This test invokes the ResourceSparseApplyMomentum operation, which + # did not have a registered GPU kernel as of April 2018. With graph + # execution, the placement algorithm notices this and automatically + # places the variable in CPU (host) memory. With eager execution, + # the variable would be placed in GPU memory if available, which + # would then conflict with the future invocation of the + # ResourceSparseApplyMomentum operation. + # To work around this discrepancy, for now we force the variable + # to be placed on CPU. + with ops.device("/cpu:0"): + var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype) # pylint: disable=cell-var-from-loop def loss(): @@ -256,7 +266,17 @@ class MomentumOptimizerTest(test.TestCase): @test_util.run_in_graph_and_eager_modes(reset_test=True) def testMinimizeWith2DIndiciesForEmbeddingLookup(self): - var0 = resource_variable_ops.ResourceVariable(array_ops.ones([2, 2])) + # This test invokes the ResourceSparseApplyMomentum operation, which + # did not have a registered GPU kernel as of April 2018. With graph + # execution, the placement algorithm notices this and automatically + # places the variable in CPU (host) memory. With eager execution, + # the variable would be placed in GPU memory if available, which + # would then conflict with the future invocation of the + # ResourceSparseApplyMomentum operation. + # To work around this discrepancy, for now we force the variable + # to be placed on CPU. + with ops.device("/cpu:0"): + var0 = resource_variable_ops.ResourceVariable(array_ops.ones([2, 2])) def loss(): return math_ops.reduce_sum(embedding_ops.embedding_lookup(var0, [[1]])) -- GitLab From c276b8314cd3161c5626d845edcfb6697cefd043 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Tue, 10 Apr 2018 10:52:15 -0700 Subject: [PATCH 092/791] [TF:XLA] fix a segfault in MakeFakeArguments, and add a test case. PiperOrigin-RevId: 192310749 --- tensorflow/compiler/xla/tests/BUILD | 13 +++++ tensorflow/compiler/xla/tests/test_utils.cc | 2 +- .../compiler/xla/tests/test_utils_test.cc | 57 +++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 tensorflow/compiler/xla/tests/test_utils_test.cc diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD index 6c43014b33..699b077d80 100644 --- a/tensorflow/compiler/xla/tests/BUILD +++ b/tensorflow/compiler/xla/tests/BUILD @@ -1969,3 +1969,16 @@ tf_cc_test( "//tensorflow/core:test", ], ) + +xla_test( + name = "test_utils_test", + srcs = ["test_utils_test.cc"], + deps = [ + ":local_client_test_base", + ":test_utils", + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla/client:computation_builder", + "//tensorflow/compiler/xla/tests:xla_internal_test_main", + "//tensorflow/core:test", + ], +) diff --git a/tensorflow/compiler/xla/tests/test_utils.cc b/tensorflow/compiler/xla/tests/test_utils.cc index 68f75d50cb..e30d115fae 100644 --- a/tensorflow/compiler/xla/tests/test_utils.cc +++ b/tensorflow/compiler/xla/tests/test_utils.cc @@ -165,7 +165,7 @@ enum class ConstantType { kUnknown, kZero, kOne }; // Return the constant type required by this computation, if known. ConstantType GetInitValue(const HloComputation& computation) { const HloInstruction* const root = computation.root_instruction(); - if (computation.num_parameters() != 2 || + if (computation.num_parameters() != 2 || root->operand_count() != 2 || root->operand(0)->opcode() != HloOpcode::kParameter || root->operand(1)->opcode() != HloOpcode::kParameter || root->operand(0) == root->operand(1)) { diff --git a/tensorflow/compiler/xla/tests/test_utils_test.cc b/tensorflow/compiler/xla/tests/test_utils_test.cc new file mode 100644 index 0000000000..e8efc6e2a8 --- /dev/null +++ b/tensorflow/compiler/xla/tests/test_utils_test.cc @@ -0,0 +1,57 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/tests/test_utils.h" + +#include "tensorflow/compiler/xla/client/computation_builder.h" +#include "tensorflow/compiler/xla/shape_util.h" +#include "tensorflow/compiler/xla/tests/local_client_test_base.h" +#include "tensorflow/compiler/xla/tests/test_macros.h" +#include "tensorflow/core/lib/core/status_test_util.h" + +namespace xla { +namespace { + +// A test fixture is used because we need a client for our computation builder. +class TestUtilsTest : public LocalClientTestBase {}; + +XLA_TEST_F(TestUtilsTest, UnusedParam) { + ComputationBuilder builder(local_client_, TestName()); + // Make the reduction lambda. + Shape single_float = ShapeUtil::MakeShape(F32, {}); + builder.Parameter(0, single_float, "unused"); + builder.Parameter(1, single_float, "used"); + auto computation_status = builder.Build(); + TF_ASSERT_OK(computation_status.status()); + + // Make the reduction. + Shape pair_float = ShapeUtil::MakeShape(F32, {2}); + builder.Reduce(builder.Parameter(0, pair_float, "operand"), + builder.Parameter(1, single_float, "init"), + computation_status.ValueOrDie(), {0}); + computation_status = builder.Build(); + TF_ASSERT_OK(computation_status.status()); + + auto executable_status = local_client_->Compile( + computation_status.ValueOrDie(), {&pair_float, &single_float}, + ExecutableBuildOptions()); + TF_ASSERT_OK(executable_status.status()); + HloModule& module = const_cast( + executable_status.ValueOrDie()->executable()->module()); + TF_ASSERT_OK(MakeFakeArguments(&module).status()); +} + +} // namespace +} // namespace xla -- GitLab From afa17984849881f39fb56c6e3500d866539924d5 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 11:09:37 -0700 Subject: [PATCH 093/791] Adds support for hoisting out common denominator in arithmetic_optimizer PiperOrigin-RevId: 192314177 --- .../optimizers/arithmetic_optimizer.cc | 103 +++++++++++++----- .../optimizers/arithmetic_optimizer_test.cc | 85 ++++++++++++++- 2 files changed, 161 insertions(+), 27 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc index fa0f7c1c6e..463c332858 100644 --- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc @@ -695,15 +695,20 @@ class AddOpsRewriteStage : public ArithmeticNodesGroupOptimizerStage { } }; -// Use the commutativity and (left- and right-) distributive property of -// multiplication over addition to hoist common factors out of aggregate nodes -// where all the inputs are Mul nodes. This pattern occurs frequently in -// regularization terms for the gradients during training. +// Use the distributive property of multiplication and division over addition, +// along with commutativity of the former, to hoist common factors/denominators +// out of aggregate nodes where ALL the inputs are Mul/Div nodes. +// This pattern occurs frequently in regularization terms for the gradients +// during training. // // For example, we can rewrite an expression of the form: // AddN(Mul(x, y1), Mul(y2, x), Mul(x, y3), ... Mul(x, yn)) // to the following: // Mul(x, AddN(y1, y2, y3, ... yn)) +// For division, we can rewrite +// AddN(Div(y1, x), Div(y2, x), Div(y3, x), ... Div(yn, x)) +// to: +// Div(AddN(y1, y2, y3, ... yn), x) class HoistCommonFactorOutOfAggregation : public ArithmeticOptimizerStage { public: explicit HoistCommonFactorOutOfAggregation( @@ -720,9 +725,11 @@ class HoistCommonFactorOutOfAggregation : public ArithmeticOptimizerStage { Status TrySimplify(NodeDef* node, string* simplified_node_name) override { TF_RETURN_IF_ERROR(EnsureNodeIsSupported(node)); + bool common_factor_is_denominator = false; std::set common_factors; std::vector ctrl_deps; - TF_RETURN_IF_ERROR(GetCommonFactors(node, &common_factors, &ctrl_deps)); + TF_RETURN_IF_ERROR(GetCommonFactors( + node, &common_factors, &common_factor_is_denominator, &ctrl_deps)); if (common_factors.size() == 1) { const string& common_factor = *common_factors.begin(); @@ -730,24 +737,31 @@ class HoistCommonFactorOutOfAggregation : public ArithmeticOptimizerStage { // Gather up the non-shared factors bool shapes_match = true; std::vector unique_factors; - TF_RETURN_IF_ERROR(GetUniqueFactors(node, common_factor, &shapes_match, - &unique_factors)); + TF_RETURN_IF_ERROR(GetUniqueFactors(node, common_factor, + common_factor_is_denominator, + &shapes_match, &unique_factors)); if (shapes_match) { NodeDef* input_0; TF_RETURN_IF_ERROR(GetInputNode(node->input(0), &input_0)); - // Use a copy of the first Mul node for the outer multiplication. - NodeDef* new_mul_node = AddCopyNode(OuterMulNodeName(node), input_0); + // Use a copy of the first node for the outer multiplication/division. + NodeDef* new_outer_node = AddCopyNode( + OuterNodeName(node, common_factor_is_denominator), input_0); // And a copy of aggregation node as one of the inner operands NodeDef* new_add_node = AddCopyNode(InnerAddNodeName(node), node); - new_mul_node->set_device(node->device()); - new_mul_node->set_input(0, common_factor); - new_mul_node->set_input(1, new_add_node->name()); + new_outer_node->set_device(node->device()); + if (common_factor_is_denominator) { + new_outer_node->set_input(0, new_add_node->name()); + new_outer_node->set_input(1, common_factor); + } else { + new_outer_node->set_input(0, common_factor); + new_outer_node->set_input(1, new_add_node->name()); + } - ctx_.node_map->AddOutput(common_factor, new_mul_node->name()); - ctx_.node_map->AddOutput(new_add_node->name(), new_mul_node->name()); + ctx_.node_map->AddOutput(common_factor, new_outer_node->name()); + ctx_.node_map->AddOutput(new_add_node->name(), new_outer_node->name()); // Hoist non-shared factors up into the new AddN node. for (int i = 0; i < unique_factors.size(); ++i) { @@ -766,17 +780,18 @@ class HoistCommonFactorOutOfAggregation : public ArithmeticOptimizerStage { AddToOptimizationQueue(new_add_node); // do not optimize the same node twice rewritten_nodes_.insert(node->name()); - *simplified_node_name = new_mul_node->name(); + *simplified_node_name = new_outer_node->name(); } } return Status::OK(); } private: - // Get a name for new outer Mul node - string OuterMulNodeName(const NodeDef* node) const { + // Get a name for new outer node + string OuterNodeName(const NodeDef* node, bool is_div) const { auto scope_and_name = ParseNodeScopeAndName(node->name()); - return OptimizedNodeName(scope_and_name, "Mul"); + return is_div ? OptimizedNodeName(scope_and_name, "Div") + : OptimizedNodeName(scope_and_name, "Mul"); } // Get a name new inner Add node @@ -785,11 +800,17 @@ class HoistCommonFactorOutOfAggregation : public ArithmeticOptimizerStage { return OptimizedNodeName(scope_and_name, "Add"); } - // Determine the set of common factors if the input nodes are all Mul nodes. + // Determine the set of common factors if the input nodes are all Mul or + // Div nodes. Status GetCommonFactors(const NodeDef* node, std::set* common_factors, + bool* common_factor_is_denominator, std::vector* ctrl_deps) const { CHECK(common_factors->empty()); + CHECK_NOTNULL(common_factor_is_denominator); + *common_factor_is_denominator = false; + bool has_mul = false; + bool has_div = false; for (int i = 0; i < node->input_size(); ++i) { if (i > 0 && common_factors->empty()) break; if (IsControlInput(node->input(i))) { @@ -799,12 +820,36 @@ class HoistCommonFactorOutOfAggregation : public ArithmeticOptimizerStage { NodeDef* input; TF_RETURN_IF_ERROR(GetInputNode(node->input(i), &input)); - if (!IsMul(*input)) { + if ((!IsMul(*input) && !IsAnyDiv(*input)) || (IsMul(*input) && has_div) || + (IsAnyDiv(*input) && has_mul)) { + // Break if input is neither a Mul or Div, or if there are both Mul & + // Div Ops. common_factors->clear(); break; + } else if (IsAnyDiv(*input)) { + has_div = true; + // In case of possible common dividers, we avoid hoisting out if any + // input is not float/double, since integer division is not distributive + // over addition. + OpInfo::TensorProperties properties0, properties1; + TF_RETURN_IF_ERROR(GetTensorProperties(input->input(0), &properties0)); + TF_RETURN_IF_ERROR(GetTensorProperties(input->input(1), &properties1)); + if (properties0.dtype() != DT_FLOAT && + properties0.dtype() != DT_DOUBLE && + properties1.dtype() != DT_FLOAT && + properties1.dtype() != DT_DOUBLE) { + common_factors->clear(); + break; + } + } else if (IsMul(*input)) { + has_mul = true; } - std::set factors_i{input->input(0), input->input(1)}; + // We only focus on common factors from denominators if any Op is a + // Div. + std::set factors_i = + has_mul ? std::set{input->input(0), input->input(1)} + : std::set{input->input(1)}; if (i == 0) { std::swap(*common_factors, factors_i); } else { @@ -819,6 +864,8 @@ class HoistCommonFactorOutOfAggregation : public ArithmeticOptimizerStage { ctrl_deps->push_back(input->input(i)); } } + + *common_factor_is_denominator = has_div; return Status::OK(); } @@ -827,6 +874,7 @@ class HoistCommonFactorOutOfAggregation : public ArithmeticOptimizerStage { // have the same shape since the other aggregation ops do not support // broadcasting. Status GetUniqueFactors(const NodeDef* node, const string& common_factor, + const bool common_factor_is_denominator, bool* shapes_match, std::vector* unique_factors) const { *shapes_match = true; @@ -837,11 +885,13 @@ class HoistCommonFactorOutOfAggregation : public ArithmeticOptimizerStage { if (IsControlInput(input)) { break; } - NodeDef* mul_node; - TF_RETURN_IF_ERROR(GetInputNode(input, &mul_node)); + NodeDef* inner_node; + TF_RETURN_IF_ERROR(GetInputNode(input, &inner_node)); const int unique_factor_index = - mul_node->input(0) == common_factor ? 1 : 0; - unique_factors->push_back(mul_node->input(unique_factor_index)); + common_factor_is_denominator + ? 0 + : (inner_node->input(0) == common_factor ? 1 : 0); + unique_factors->push_back(inner_node->input(unique_factor_index)); if (i > 0 && !IsAdd(*node)) { OpInfo::TensorProperties lhs; OpInfo::TensorProperties rhs; @@ -857,7 +907,8 @@ class HoistCommonFactorOutOfAggregation : public ArithmeticOptimizerStage { // if graph rewrite happens in multiple passes without graph pruning between // them, it's possible that rewritten node already exists in a graph return rewritten_nodes_.find(node->name()) != rewritten_nodes_.end() || - ctx_.node_map->NodeExists(OuterMulNodeName(node)); + ctx_.node_map->NodeExists(OuterNodeName(node, false)) || + ctx_.node_map->NodeExists(OuterNodeName(node, true)); } // keep names of the nodes that were optimized by this stage diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc index 9677175d2e..e639812858 100644 --- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc +++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc @@ -31,6 +31,9 @@ namespace grappler { namespace { +constexpr char kHoistFactorOptimizerDiv[] = + "ArithmeticOptimizer/HoistCommonFactor_Div_"; + constexpr char kHoistFactorOptimizerMul[] = "ArithmeticOptimizer/HoistCommonFactor_Mul_"; @@ -42,6 +45,11 @@ string HoistMulName(const string& name) { return AddPrefixToNodeName(name, kHoistFactorOptimizerMul, ""); } +// Optimized name of outer Div node by HoistCommonFactorOutOfAggregation +string HoistDivName(const string& name) { + return AddPrefixToNodeName(name, kHoistFactorOptimizerDiv, ""); +} + // Optimized name of inner Add node by HoistCommonFactorOutOfAggregation string HoistAddName(const string& name) { return AddPrefixToNodeName(name, kHoistFactorOptimizerAdd, ""); @@ -558,7 +566,7 @@ TEST_F(ArithmeticOptimizerTest, TrivialSumsRepeatedAdd) { EXPECT_EQ("^Placeholder", add_1_const_node->input(0)); } -TEST_F(ArithmeticOptimizerTest, HoistFactor) { +TEST_F(ArithmeticOptimizerTest, HoistFactorMul) { for (bool matching_shapes : {true, false}) { for (bool use_addn : {true, false}) { tensorflow::Scope s = tensorflow::Scope::NewRootScope(); @@ -625,6 +633,81 @@ TEST_F(ArithmeticOptimizerTest, HoistFactor) { } } +TEST_F(ArithmeticOptimizerTest, HoistFactorDiv) { + for (bool matching_shapes : {true, false}) { + for (bool use_addn : {true, false}) { + for (bool use_ints : {true, false}) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + Output x = use_ints + ? ops::Const(s.WithOpName("x"), {1, 2}, {1, 2}) + : ops::Const(s.WithOpName("x"), {1.0f, 2.0f}, {1, 2}); + Output y1 = use_ints + ? ops::Const(s.WithOpName("y1"), {3, 4}, {1, 2}) + : ops::Const(s.WithOpName("y1"), {3.0f, 4.0f}, {1, 2}); + Output y2; + if (matching_shapes) { + y2 = use_ints ? ops::Const(s.WithOpName("y2"), {5, 6}, {1, 2}) + : ops::Const(s.WithOpName("y2"), {5.0f, 6.0f}, {1, 2}); + } else { + y2 = use_ints ? ops::Const(s.WithOpName("y2"), {5}, {1, 1}) + : ops::Const(s.WithOpName("y2"), {5.0f}, {1, 1}); + } + Output div1 = ops::Div(s.WithOpName("div1"), y1, x); + Output div2 = ops::Div(s.WithOpName("div2"), y2, x); + Output id = + use_addn + ? ops::Identity(s.WithOpName("id"), + ops::AddN(s.WithOpName("add"), {div1, div2})) + : ops::Identity(s.WithOpName("id"), + ops::Add(s.WithOpName("add"), div1, div2)); + + GrapplerItem item; + item.fetch = {"id"}; + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + + ArithmeticOptimizer optimizer; + EnableOnlyHoistCommonFactor(&optimizer); + + GraphDef output; + OptimizeTwice(&optimizer, &item, &output); + + // We expect the following rewrite(s) to occur: + // + // Add Div + // / \ / \ + // Div Div -> Add x + // / \ / \ / \ + // y1 x y2 x y1 y2 + // + // If "root" op is AddN and shapes does not match, this rewrite is not + // possible and graph should stay intact. + NodeMap node_map(&output); + + if ((use_addn && !matching_shapes) || use_ints) { + VerifyGraphsMatch(item.graph, output, __LINE__); + } else { + EXPECT_EQ(9, output.node_size()); + + const NodeDef* new_add_node = node_map.GetNode(HoistAddName("add")); + ASSERT_TRUE(new_add_node != nullptr) << "Hoisted Add node not found"; + EXPECT_EQ("y1", new_add_node->input(0)); + EXPECT_EQ("y2", new_add_node->input(1)); + + const NodeDef* new_div_node = node_map.GetNode(HoistDivName("add")); + ASSERT_TRUE(new_div_node != nullptr) << "Hoisted Div node not found"; + EXPECT_EQ(new_add_node->name(), new_div_node->input(0)); + EXPECT_EQ("x", new_div_node->input(1)); + + const NodeDef* id_node = node_map.GetNode("id"); + ASSERT_TRUE(id_node != nullptr) << "Id node not found"; + EXPECT_EQ("id", id_node->name()); + EXPECT_EQ(HoistDivName("add"), id_node->input(0)); + } + } + } + } +} + TEST_F(ArithmeticOptimizerTest, FuseConjAndTranspose) { tensorflow::Scope s = tensorflow::Scope::NewRootScope(); Output re = ops::Const(s.WithOpName("re"), {1.0f, 2.0f, 3.0f, 4.0f}, {2, 2}); -- GitLab From 0fd3c5c450a844573f9c417994ae87035119d2b4 Mon Sep 17 00:00:00 2001 From: Amit Patankar Date: Tue, 10 Apr 2018 11:19:26 -0700 Subject: [PATCH 094/791] Adding the python symlink command for devel packages too. --- tensorflow/tools/docker/Dockerfile.devel | 2 ++ tensorflow/tools/docker/Dockerfile.devel-gpu | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tensorflow/tools/docker/Dockerfile.devel b/tensorflow/tools/docker/Dockerfile.devel index 11f476d12c..c4f6b24e5c 100644 --- a/tensorflow/tools/docker/Dockerfile.devel +++ b/tensorflow/tools/docker/Dockerfile.devel @@ -38,6 +38,8 @@ RUN pip --no-cache-dir install \ && \ python -m ipykernel.kernelspec +# RUN ln -s /usr/bin/python3 /usr/bin/python# + # Set up our notebook config. COPY jupyter_notebook_config.py /root/.jupyter/ diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu index 1fcb6428b2..5aea47e582 100644 --- a/tensorflow/tools/docker/Dockerfile.devel-gpu +++ b/tensorflow/tools/docker/Dockerfile.devel-gpu @@ -47,6 +47,8 @@ RUN pip --no-cache-dir install \ && \ python -m ipykernel.kernelspec +# RUN ln -s /usr/bin/python3 /usr/bin/python# + # Set up our notebook config. COPY jupyter_notebook_config.py /root/.jupyter/ -- GitLab From 17a320fa107905b335a6fb944eaf323e868a2470 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 11:20:38 -0700 Subject: [PATCH 095/791] [XLA] Fix the size of the data returned from Literal for sparse literals. PiperOrigin-RevId: 192315888 --- tensorflow/compiler/xla/literal_util.h | 13 +++++++++---- tensorflow/compiler/xla/literal_util_test.cc | 4 +--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/tensorflow/compiler/xla/literal_util.h b/tensorflow/compiler/xla/literal_util.h index 66ff39ecbb..a6a3dffeb7 100644 --- a/tensorflow/compiler/xla/literal_util.h +++ b/tensorflow/compiler/xla/literal_util.h @@ -741,7 +741,13 @@ class Literal { int64 size_bytes() const { return ShapeUtil::ByteSizeOf(subshape()); } // Returns the number of elements in this piece's array. - int64 element_count() const { return ShapeUtil::ElementsIn(subshape()); } + int64 element_count() const { + // If this is a sparse array, use the number of elements represented by + // the indices in the associated SparseIndexArray. + return LayoutUtil::IsSparseArray(subshape()) + ? sparse_indices()->index_count() + : ShapeUtil::ElementsIn(subshape()); + } // Copy the data from 'src' into this piece's buffer. Shapes of this piece // and src must be compatible. @@ -853,8 +859,7 @@ tensorflow::gtl::ArraySlice Literal::Piece::data() const { << " type, but literal element type is " << PrimitiveType_Name(subshape().element_type()); return tensorflow::gtl::ArraySlice( - reinterpret_cast(buffer()), - ShapeUtil::ElementsIn(subshape())); + reinterpret_cast(buffer()), element_count()); } template @@ -867,7 +872,7 @@ tensorflow::gtl::MutableArraySlice Literal::Piece::data() { << " type, but literal element type is " << PrimitiveType_Name(subshape().element_type()); return tensorflow::gtl::MutableArraySlice( - reinterpret_cast(buffer()), ShapeUtil::ElementsIn(subshape())); + reinterpret_cast(buffer()), element_count()); } template diff --git a/tensorflow/compiler/xla/literal_util_test.cc b/tensorflow/compiler/xla/literal_util_test.cc index be4f2bc5ce..61046784e0 100644 --- a/tensorflow/compiler/xla/literal_util_test.cc +++ b/tensorflow/compiler/xla/literal_util_test.cc @@ -218,9 +218,7 @@ TEST_F(LiteralUtilTest, CreateSparse) { EXPECT_EQ(literal->sparse_indices()->data(), ArraySlice(expected_indices.data(), expected_indices.num_elements())); - EXPECT_EQ( - ArraySlice(literal->data().data(), expected_values.size()), - ArraySlice(expected_values)); + EXPECT_EQ(literal->data(), ArraySlice(expected_values)); } TEST_F(LiteralUtilTest, LiteralR4F32ProjectedStringifies) { -- GitLab From 1ab0cc3548f330fda61cf01c524e3f85a00d8485 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 11:33:45 -0700 Subject: [PATCH 096/791] Fix bug in TFLite Interpreter python interface PiperOrigin-RevId: 192318426 --- tensorflow/contrib/lite/python/interpreter.py | 4 ++-- tensorflow/contrib/lite/python/interpreter_test.py | 3 +++ .../lite/python/interpreter_wrapper/interpreter_wrapper.cc | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/lite/python/interpreter.py b/tensorflow/contrib/lite/python/interpreter.py index b8638007f7..cb9c0d3121 100644 --- a/tensorflow/contrib/lite/python/interpreter.py +++ b/tensorflow/contrib/lite/python/interpreter.py @@ -121,8 +121,8 @@ class Interpreter(object): Raises: ValueError: If the interpreter could not resize the input tensor. """ - if not self.ResizeInputTensor.SetTensor(input_index, tensor_size): - raise ValueError('Failed to set input') + if not self._interpreter.ResizeInputTensor(input_index, tensor_size): + raise ValueError('Failed to resize input') def get_output_details(self): """Gets model output details. diff --git a/tensorflow/contrib/lite/python/interpreter_test.py b/tensorflow/contrib/lite/python/interpreter_test.py index cd2386f526..f802edf020 100644 --- a/tensorflow/contrib/lite/python/interpreter_test.py +++ b/tensorflow/contrib/lite/python/interpreter_test.py @@ -81,6 +81,9 @@ class InterpreterTest(test_util.TensorFlowTestCase): test_input = np.array([[1, 2, 3, 4]], dtype=np.uint8) expected_output = np.array([[4, 3, 2, 1]], dtype=np.uint8) + interpreter.resize_tensor_input(input_details[0]['index'], + np.array(test_input.shape, dtype=np.int32)) + interpreter.allocate_tensors() interpreter.set_tensor(input_details[0]['index'], test_input) interpreter.invoke() diff --git a/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.cc b/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.cc index 35ad226b78..4b34969356 100644 --- a/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.cc +++ b/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.cc @@ -186,7 +186,7 @@ bool InterpreterWrapper::ResizeInputTensor(int i, PyObject* value) { std::vector dims(PyArray_SHAPE(array)[0]); memcpy(dims.data(), PyArray_BYTES(array), dims.size() * sizeof(int)); - return interpreter_->ResizeInputTensor(i, dims); + return (interpreter_->ResizeInputTensor(i, dims) == kTfLiteOk); } std::string InterpreterWrapper::TensorName(int i) const { -- GitLab From 2177a2306ab43b758630180ca93b84602c73dfc6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 11:57:40 -0700 Subject: [PATCH 097/791] Enable loop-invariant node motion in the Grappler loop optimizer. Thanks to the team at Alibaba, who contributed the original version of this code. PiperOrigin-RevId: 192322484 --- tensorflow/core/grappler/optimizers/loop_optimizer.h | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/loop_optimizer.h b/tensorflow/core/grappler/optimizers/loop_optimizer.h index 83c499bbe7..a422505d23 100644 --- a/tensorflow/core/grappler/optimizers/loop_optimizer.h +++ b/tensorflow/core/grappler/optimizers/loop_optimizer.h @@ -52,14 +52,11 @@ class LoopOptimizer : public GraphOptimizer { // Granular control for loop optimizer stages. struct LoopOptimizerOptions { - bool enable_loop_invariant_node_motion = false; + bool enable_loop_invariant_node_motion = true; bool enable_stack_push_removal = true; static LoopOptimizerOptions Default(RewriterConfig::Toggle opt_level) { LoopOptimizerOptions options; - if (opt_level == RewriterConfig::AGGRESSIVE) { - options.enable_loop_invariant_node_motion = true; - } return options; } }; -- GitLab From 199b8ade22550ca3e5ccc6c744914b3ef614d232 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 12:05:06 -0700 Subject: [PATCH 098/791] Expand list of value-preserving ops. This will increase the number of graphs where we can apply the involution and mul->conv fusion optimizations. PiperOrigin-RevId: 192323712 --- tensorflow/core/grappler/op_types.cc | 33 +++++++++++++++++++++++----- tensorflow/core/grappler/op_types.h | 4 ++++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc index 1fb1711f54..9c45aed62f 100644 --- a/tensorflow/core/grappler/op_types.cc +++ b/tensorflow/core/grappler/op_types.cc @@ -456,15 +456,38 @@ bool IsInvolution(const NodeDef& node) { return involution_ops.count(node.op()) > 0; } -bool IsValuePreserving(const NodeDef& node) { +bool IsValueAndOrderPreserving(const NodeDef& node) { if (NumNonControlInputs(node) == 1 && IsAggregate(node)) { return true; } + const std::unordered_set value_and_order_preserving_ops{ + "CheckNumerics", + "DebugGradientIdentity", + "DeepCopy" + "Enter", + "Exit", + "ExpandDims", + "Identity", + "IdentityN", + "PreventGradient", + "Print", + "Reshape", + "Snapshot", + "Squeeze", + "StopGradient", + }; + return value_and_order_preserving_ops.count(node.op()) > 0; +} + +bool IsValuePreserving(const NodeDef& node) { const std::unordered_set value_preserving_ops{ - "Transpose", "Reshape", "Identity", "InvertPermutation", - "Reverse", "StopGradient", "PreventGradient", "CheckNumerics", - "ExpandDims", "Squeeze"}; - return value_preserving_ops.count(node.op()) > 0; + "InvertPermutation", + "Reverse", + "Roll", + "Transpose", + }; + return IsValueAndOrderPreserving(node) || + value_preserving_ops.count(node.op()) > 0; } bool HasOpDef(const NodeDef& node) { diff --git a/tensorflow/core/grappler/op_types.h b/tensorflow/core/grappler/op_types.h index d516baebf3..79fd05e187 100644 --- a/tensorflow/core/grappler/op_types.h +++ b/tensorflow/core/grappler/op_types.h @@ -168,6 +168,10 @@ bool ModifiesInputsInPlace(const NodeDef& node); // own inverse such that f(f(x)) == x. bool IsInvolution(const NodeDef& node); +// Returns true if the op preserves the order and value of elements in its +// first input tensor and possible changes its shape. +bool IsValueAndOrderPreserving(const NodeDef& node); + // Returns true if the op in node only rearranges the order of elements in its // first input tensor and possible changes its shape. More precisely, this // function returns true if the op commutes with all element-wise operations. -- GitLab From dc7883afb6220e5a105d8fea6e0cfdaf92839db3 Mon Sep 17 00:00:00 2001 From: Noah Eisen Date: Tue, 10 Apr 2018 12:28:04 -0700 Subject: [PATCH 099/791] Upgrade gRPC version and fix file duplication This bumps the gRPC version used in OSS Tensorflow to pick up grpc/grpc#14541, which exposes gRPC serialization classes which were previously hidden in an internal namespace. Using these files eliminates files duplicated from gRPC repo PiperOrigin-RevId: 192327358 --- tensorflow/contrib/cmake/external/grpc.cmake | 2 +- tensorflow/core/distributed_runtime/rpc/BUILD | 12 - .../rpc/grpc_master_service_impl.h | 10 - .../rpc/grpc_serialization_traits.h | 217 ------------------ .../rpc/grpc_worker_service_impl.h | 28 +-- tensorflow/workspace.bzl | 8 +- 6 files changed, 14 insertions(+), 263 deletions(-) delete mode 100644 tensorflow/core/distributed_runtime/rpc/grpc_serialization_traits.h diff --git a/tensorflow/contrib/cmake/external/grpc.cmake b/tensorflow/contrib/cmake/external/grpc.cmake index abfc69243e..bec8177a3f 100644 --- a/tensorflow/contrib/cmake/external/grpc.cmake +++ b/tensorflow/contrib/cmake/external/grpc.cmake @@ -17,7 +17,7 @@ include (ExternalProject) set(GRPC_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/include) set(GRPC_URL https://github.com/grpc/grpc.git) set(GRPC_BUILD ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc) -set(GRPC_TAG bd6bdf93279a39a8cd92978fd7c9d14eccd98fc2) +set(GRPC_TAG 09386db3939cae1ac12e5f09b735adfa8958c68e) if(WIN32) if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*") diff --git a/tensorflow/core/distributed_runtime/rpc/BUILD b/tensorflow/core/distributed_runtime/rpc/BUILD index d3478dfc38..fa0f8c9b52 100644 --- a/tensorflow/core/distributed_runtime/rpc/BUILD +++ b/tensorflow/core/distributed_runtime/rpc/BUILD @@ -189,7 +189,6 @@ cc_library( srcs = ["grpc_worker_service_impl.cc"], hdrs = ["grpc_worker_service_impl.h"], deps = [ - ":grpc_serialization_traits", "//tensorflow/core:worker_proto_cc", "//tensorflow/core/distributed_runtime:tensor_coding", "@grpc//:grpc++_unsecure", @@ -235,22 +234,11 @@ cc_library( srcs = ["grpc_master_service_impl.cc"], hdrs = ["grpc_master_service_impl.h"], deps = [ - ":grpc_serialization_traits", "//tensorflow/core:master_proto_cc", "@grpc//:grpc++_unsecure", ], ) -cc_library( - name = "grpc_serialization_traits", - srcs = [], - hdrs = ["grpc_serialization_traits.h"], - deps = [ - "@grpc//:grpc++_unsecure", - "@grpc//:grpc_unsecure", - ], -) - cc_library( name = "rpc_rendezvous_mgr", srcs = ["rpc_rendezvous_mgr.cc"], diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.h b/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.h index 3c382738c4..8f1b589698 100644 --- a/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.h +++ b/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.h @@ -25,18 +25,8 @@ limitations under the License. #include "grpc++/impl/codegen/stub_options.h" #include "grpc++/impl/codegen/sync_stream.h" -#include "tensorflow/core/distributed_runtime/rpc/grpc_serialization_traits.h" #include "tensorflow/core/protobuf/master.pb.h" -// Contains potentially large GraphDef. -TF_GRPC_ALLOW_UNLIMITED_MESSAGE_SIZE(tensorflow::CreateSessionRequest); -// Contains potentially large GraphDef. -TF_GRPC_ALLOW_UNLIMITED_MESSAGE_SIZE(tensorflow::ExtendSessionRequest); -// Contains potentially large TensorProto. -TF_GRPC_ALLOW_UNLIMITED_MESSAGE_SIZE(tensorflow::RunStepRequest); -// Contains potentially large StepStats, TensorProto. -TF_GRPC_ALLOW_UNLIMITED_MESSAGE_SIZE(tensorflow::RunStepResponse); - namespace grpc { class CompletionQueue; class Channel; diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_serialization_traits.h b/tensorflow/core/distributed_runtime/rpc/grpc_serialization_traits.h deleted file mode 100644 index e7f5fb0c6a..0000000000 --- a/tensorflow/core/distributed_runtime/rpc/grpc_serialization_traits.h +++ /dev/null @@ -1,217 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_SERIALIZATION_TRAITS_H_ -#define TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_SERIALIZATION_TRAITS_H_ - -#include "grpc++/impl/codegen/proto_utils.h" -#include "grpc++/support/slice.h" -#include "grpc/grpc.h" - -namespace grpc { - -namespace tensorflow_helper { - -const int kGrpcBufferWriterMaxBufferLength = 8192; - -class GrpcBufferWriter final - : public ::grpc::protobuf::io::ZeroCopyOutputStream { - public: - explicit GrpcBufferWriter(grpc_byte_buffer** bp, int block_size) - : block_size_(block_size), byte_count_(0), have_backup_(false) { - *bp = grpc_raw_byte_buffer_create(NULL, 0); - slice_buffer_ = &(*bp)->data.raw.slice_buffer; - } - - ~GrpcBufferWriter() override { - if (have_backup_) { - grpc_slice_unref(backup_slice_); - } - } - - bool Next(void** data, int* size) override { - if (have_backup_) { - slice_ = backup_slice_; - have_backup_ = false; - } else { - slice_ = grpc_slice_malloc(block_size_); - } - *data = GRPC_SLICE_START_PTR(slice_); - // On win x64, int is only 32bit - GPR_CODEGEN_ASSERT(GRPC_SLICE_LENGTH(slice_) <= INT_MAX); - byte_count_ += * size = (int)GRPC_SLICE_LENGTH(slice_); - grpc_slice_buffer_add(slice_buffer_, slice_); - return true; - } - - void BackUp(int count) override { - grpc_slice_buffer_pop(slice_buffer_); - if (count == block_size_) { - backup_slice_ = slice_; - } else { - backup_slice_ = - grpc_slice_split_tail(&slice_, GRPC_SLICE_LENGTH(slice_) - count); - grpc_slice_buffer_add(slice_buffer_, slice_); - } - // It's dangerous to keep an inlined grpc_slice as the backup slice, since - // on a following Next() call, a reference will be returned to this slice - // via GRPC_SLICE_START_PTR, which will not be an address held by - // slice_buffer_. - have_backup_ = backup_slice_.refcount != NULL; - byte_count_ -= count; - } - - grpc::protobuf::int64 ByteCount() const override { return byte_count_; } - - private: - const int block_size_; - int64_t byte_count_; - grpc_slice_buffer* slice_buffer_; - bool have_backup_; - grpc_slice backup_slice_; - grpc_slice slice_; -}; - -class GrpcBufferReader final - : public ::grpc::protobuf::io::ZeroCopyInputStream { - public: - explicit GrpcBufferReader(grpc_byte_buffer* buffer) - : byte_count_(0), backup_count_(0) { - (void)grpc_byte_buffer_reader_init(&reader_, buffer); - } - ~GrpcBufferReader() override { grpc_byte_buffer_reader_destroy(&reader_); } - - bool Next(const void** data, int* size) override { - if (backup_count_ > 0) { - *data = GRPC_SLICE_START_PTR(slice_) + GRPC_SLICE_LENGTH(slice_) - - backup_count_; - GPR_CODEGEN_ASSERT(backup_count_ <= INT_MAX); - *size = (int)backup_count_; - backup_count_ = 0; - return true; - } - if (!grpc_byte_buffer_reader_next(&reader_, &slice_)) { - return false; - } - grpc_slice_unref(slice_); - *data = GRPC_SLICE_START_PTR(slice_); - // On win x64, int is only 32bit - GPR_CODEGEN_ASSERT(GRPC_SLICE_LENGTH(slice_) <= INT_MAX); - byte_count_ += * size = (int)GRPC_SLICE_LENGTH(slice_); - return true; - } - - void BackUp(int count) override { backup_count_ = count; } - - bool Skip(int count) override { - const void* data; - int size; - while (Next(&data, &size)) { - if (size >= count) { - BackUp(size - count); - return true; - } - // size < count; - count -= size; - } - // error or we have too large count; - return false; - } - - grpc::protobuf::int64 ByteCount() const override { - return byte_count_ - backup_count_; - } - - private: - int64_t byte_count_; - int64_t backup_count_; - grpc_byte_buffer_reader reader_; - grpc_slice slice_; -}; - -} // namespace tensorflow_helper - -// Defines specialized serialization/deserialization routines that -// default to allowing a 2GB max message size. -// -// To instantiate this template for a particular type `T`, use -// `TF_GRPC_ALLOW_UNLIMITED_MESSAGE_SIZE(T)`, as defined below. -template -class UnlimitedSizeProtoSerializationTraits { - public: - static Status Serialize(const T& msg, grpc_byte_buffer** bp, - bool* own_buffer) { - *own_buffer = true; - int byte_size = msg.ByteSize(); - if (byte_size < 0) { - return Status(StatusCode::INTERNAL, "Message length was negative"); - } else if (byte_size <= - tensorflow_helper::kGrpcBufferWriterMaxBufferLength) { - grpc_slice slice = grpc_slice_malloc(byte_size); - GPR_CODEGEN_ASSERT( - GRPC_SLICE_END_PTR(slice) == - msg.SerializeWithCachedSizesToArray(GRPC_SLICE_START_PTR(slice))); - *bp = grpc_raw_byte_buffer_create(&slice, 1); - grpc_slice_unref(slice); - return Status::OK; - } else { - tensorflow_helper::GrpcBufferWriter writer( - bp, tensorflow_helper::kGrpcBufferWriterMaxBufferLength); - return msg.SerializeToZeroCopyStream(&writer) - ? Status::OK - : Status(StatusCode::INTERNAL, "Failed to serialize message"); - } - } - - static Status Deserialize(grpc_byte_buffer* buffer, T* msg, - int max_message_size = INT_MAX) { - if (buffer == nullptr) { - return Status(StatusCode::INTERNAL, "No payload"); - } - Status result = Status::OK; - { - tensorflow_helper::GrpcBufferReader reader(buffer); - ::grpc::protobuf::io::CodedInputStream decoder(&reader); - if (max_message_size == 0) { - // NOTE(mrry): Override maximum message size to 2GB. - decoder.SetTotalBytesLimit(INT_MAX, INT_MAX); - } else { - decoder.SetTotalBytesLimit(max_message_size, max_message_size); - } - if (!msg->ParseFromCodedStream(&decoder)) { - result = Status(StatusCode::INTERNAL, msg->InitializationErrorString()); - } - if (!decoder.ConsumedEntireMessage()) { - result = Status(StatusCode::INTERNAL, "Did not read entire message"); - } - } - grpc_byte_buffer_destroy(buffer); - return result; - } -}; - -} // namespace grpc - -// For the given protobuf message type `MessageType`, specializes the -// gRPC serialization and deserialization such that the default -// maximum message size is 2GB. -#define TF_GRPC_ALLOW_UNLIMITED_MESSAGE_SIZE(MessageType) \ - namespace grpc { \ - template <> \ - class SerializationTraits \ - : public UnlimitedSizeProtoSerializationTraits {}; \ - } // namespace grpc - -#endif // TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_SERIALIZATION_TRAITS_H_ diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h index 2a2f7e3ffb..62b299d5c2 100644 --- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h +++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h @@ -26,24 +26,16 @@ limitations under the License. #include "grpc++/impl/codegen/sync_stream.h" #include "grpc++/support/byte_buffer.h" -#include "tensorflow/core/distributed_runtime/rpc/grpc_serialization_traits.h" #include "tensorflow/core/distributed_runtime/tensor_coding.h" #include "tensorflow/core/protobuf/worker.pb.h" -// Contains potentially large GraphDef. -TF_GRPC_ALLOW_UNLIMITED_MESSAGE_SIZE(tensorflow::RegisterGraphRequest); -// Contains potentially large TensorProto. -TF_GRPC_ALLOW_UNLIMITED_MESSAGE_SIZE(tensorflow::RunGraphRequest); -// Contains potentially large StepStats, TensorProto. -TF_GRPC_ALLOW_UNLIMITED_MESSAGE_SIZE(tensorflow::RunGraphResponse); - namespace tensorflow { class GrpcByteSource : public TensorResponse::Source { public: - explicit GrpcByteSource(grpc_byte_buffer* buffer) : buffer_(buffer) {} + explicit GrpcByteSource(::grpc::ByteBuffer* buffer) : buffer_(buffer) {} ~GrpcByteSource() override { DeleteStream(); } - typedef ::grpc::tensorflow_helper::GrpcBufferReader Reader; + typedef ::grpc::GrpcProtoBufferReader Reader; protobuf::io::ZeroCopyInputStream* contents() override { DeleteStream(); @@ -58,7 +50,7 @@ class GrpcByteSource : public TensorResponse::Source { } } - grpc_byte_buffer* buffer_; // Not owned + ::grpc::ByteBuffer* buffer_; // Not owned Reader* stream_ = nullptr; // Points into space_ if non-nullptr char space_[sizeof(Reader)]; }; @@ -74,17 +66,15 @@ class ServerContext; // Support parsing/unparsing of tensorflow::TensorResponse. // Wire-format is identical to RecvTensorResponse. template <> -class SerializationTraits - : public UnlimitedSizeProtoSerializationTraits { +class SerializationTraits { public: - static Status Serialize(const tensorflow::TensorResponse& msg, - grpc_byte_buffer** bp, bool* own_buffer) { + static Status Serialize(const tensorflow::TensorResponse& msg, ByteBuffer* bp, + bool* own_buffer) { LOG(FATAL) << "TODO(sanjay,jeff): Implement"; return Status(); } - static Status Deserialize(grpc_byte_buffer* buffer, - tensorflow::TensorResponse* msg, - int max_message_size = INT_MAX) { + static Status Deserialize(ByteBuffer* buffer, + tensorflow::TensorResponse* msg) { if (buffer == nullptr) { return Status(StatusCode::INTERNAL, "No payload"); } @@ -98,7 +88,7 @@ class SerializationTraits "TensorResponse parse error", s.ToString())); } } - grpc_byte_buffer_destroy(buffer); + buffer->Clear(); return result; } }; diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index c72aa3e649..52168a89c5 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -438,11 +438,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""): tf_http_archive( name = "grpc", urls = [ - "https://mirror.bazel.build/github.com/grpc/grpc/archive/bd6bdf93279a39a8cd92978fd7c9d14eccd98fc2.tar.gz", - "https://github.com/grpc/grpc/archive/bd6bdf93279a39a8cd92978fd7c9d14eccd98fc2.tar.gz", + "https://mirror.bazel.build/github.com/grpc/grpc/archive/09386db3939cae1ac12e5f09b735adfa8958c68e.tar.gz", + "https://github.com/grpc/grpc/archive/09386db3939cae1ac12e5f09b735adfa8958c68e.tar.gz", ], - sha256 = "0a05bd355e4571b01d813dddffa38e57e689ac41b264dc9b1bd6ec66463ef5d6", - strip_prefix = "grpc-bd6bdf93279a39a8cd92978fd7c9d14eccd98fc2", + sha256 = "b857969c667c14f37faa507afc07a3f39a47fbf73203be889d55925622e7b317", + strip_prefix = "grpc-09386db3939cae1ac12e5f09b735adfa8958c68e", ) -- GitLab From 22a5485a4f0db8d45efc30492499cba79cc1a47e Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 12:28:56 -0700 Subject: [PATCH 100/791] Employ array flat sizes more directly in reference_ops. PiperOrigin-RevId: 192327464 --- .../internal/reference/reference_ops.h | 831 ++++++------------ .../contrib/lite/kernels/internal/types.h | 115 ++- 2 files changed, 401 insertions(+), 545 deletions(-) diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h index 410688411e..4bbec52bf7 100644 --- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h +++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h @@ -635,27 +635,14 @@ void NonGlobalBatchNormalization( const Dims<4>& offset_dims, float* output_data, const Dims<4>& output_dims) { const int batches = MatchingArraySize(input_dims, 3, output_dims, 3); - const int height = - MatchingArraySize(input_dims, 2, mean_dims, 2, multiplier_dims, 2, - offset_dims, 2, output_dims, 2); - const int width = - MatchingArraySize(input_dims, 1, mean_dims, 1, multiplier_dims, 1, - offset_dims, 1, output_dims, 1); - const int depth = - MatchingArraySize(input_dims, 0, mean_dims, 0, multiplier_dims, 0, - offset_dims, 0, output_dims, 0); + const int inner_size = MatchingFlatSizeSkipDim( + input_dims, 3, mean_dims, multiplier_dims, offset_dims, output_dims); for (int b = 0; b < batches; ++b) { - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - for (int c = 0; c < depth; ++c) { - output_data[Offset(output_dims, c, x, y, b)] = ActivationFunction( - (input_data[Offset(input_dims, c, x, y, b)] - - mean_data[Offset(mean_dims, c, x, y, 0)]) * - multiplier_data[Offset(multiplier_dims, c, x, y, 0)] + - offset_data[Offset(offset_dims, c, x, y, 0)]); - } - } + for (int i = 0; i < inner_size; ++i) { + output_data[b * inner_size + i] = ActivationFunction( + (input_data[b * inner_size + i] - mean_data[i]) * multiplier_data[i] + + offset_data[i]); } } } @@ -669,87 +656,52 @@ void GlobalBatchNormalization(const float* input_data, const float* offset_data, const Dims<4>& offset_dims, float* output_data, const Dims<4>& output_dims) { - const int batches = MatchingArraySize(input_dims, 3, output_dims, 3); - const int height = MatchingArraySize(input_dims, 2, output_dims, 2); - const int width = MatchingArraySize(input_dims, 1, output_dims, 1); + const int outer_size = MatchingFlatSizeSkipDim(input_dims, 0, output_dims); const int depth = MatchingArraySize(input_dims, 0, mean_dims, 0, multiplier_dims, 0, offset_dims, 0, output_dims, 0); - for (int b = 0; b < batches; ++b) { - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - for (int c = 0; c < depth; ++c) { - output_data[Offset(output_dims, c, x, y, b)] = ActivationFunction( - (input_data[Offset(input_dims, c, x, y, b)] - - mean_data[Offset(mean_dims, c, 0, 0, 0)]) * - multiplier_data[Offset(multiplier_dims, c, 0, 0, 0)] + - offset_data[Offset(offset_dims, c, 0, 0, 0)]); - } - } + for (int i = 0; i < outer_size; ++i) { + for (int c = 0; c < depth; ++c) { + output_data[depth * i + c] = ActivationFunction( + (input_data[depth * i + c] - mean_data[c]) * multiplier_data[c] + + offset_data[c]); } } } inline void Relu(const float* input_data, const Dims<4>& input_dims, float* output_data, const Dims<4>& output_dims) { - const int batches = MatchingArraySize(input_dims, 3, output_dims, 3); - const int height = MatchingArraySize(input_dims, 2, output_dims, 2); - const int width = MatchingArraySize(input_dims, 1, output_dims, 1); - const int depth = MatchingArraySize(input_dims, 0, output_dims, 0); - for (int b = 0; b < batches; ++b) { - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - for (int c = 0; c < depth; ++c) { - float val = input_data[Offset(input_dims, c, x, y, b)]; - const float lower = 0; - float clamped = val < lower ? lower : val; - output_data[Offset(output_dims, c, x, y, b)] = clamped; - } - } - } + const int flat_size = MatchingFlatSize(input_dims, output_dims); + for (int i = 0; i < flat_size; ++i) { + const float val = input_data[i]; + const float lower = 0; + const float clamped = val < lower ? lower : val; + output_data[i] = clamped; } } inline void Relu1(const float* input_data, const Dims<4>& input_dims, float* output_data, const Dims<4>& output_dims) { - const int batches = MatchingArraySize(input_dims, 3, output_dims, 3); - const int height = MatchingArraySize(input_dims, 2, output_dims, 2); - const int width = MatchingArraySize(input_dims, 1, output_dims, 1); - const int depth = MatchingArraySize(input_dims, 0, output_dims, 0); - for (int b = 0; b < batches; ++b) { - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - for (int c = 0; c < depth; ++c) { - float val = input_data[Offset(input_dims, c, x, y, b)]; - const float upper = 1; - const float lower = -1; - float clamped = val > upper ? upper : val < lower ? lower : val; - output_data[Offset(output_dims, c, x, y, b)] = clamped; - } - } - } + const int flat_size = MatchingFlatSize(input_dims, output_dims); + for (int i = 0; i < flat_size; ++i) { + const float val = input_data[i]; + const float upper = 1; + const float lower = -1; + const float clamped = val > upper ? upper : val < lower ? lower : val; + output_data[i] = clamped; } } inline void Relu6(const float* input_data, const Dims<4>& input_dims, float* output_data, const Dims<4>& output_dims) { - const int batches = MatchingArraySize(input_dims, 3, output_dims, 3); - const int height = MatchingArraySize(input_dims, 2, output_dims, 2); - const int width = MatchingArraySize(input_dims, 1, output_dims, 1); - const int depth = MatchingArraySize(input_dims, 0, output_dims, 0); - for (int b = 0; b < batches; ++b) { - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - for (int c = 0; c < depth; ++c) { - float val = input_data[Offset(input_dims, c, x, y, b)]; - const float upper = 6; - const float lower = 0; - float clamped = val > upper ? upper : val < lower ? lower : val; - output_data[Offset(output_dims, c, x, y, b)] = clamped; - } - } - } + const int flat_size = MatchingFlatSize(input_dims, output_dims); + for (int i = 0; i < flat_size; ++i) { + const float val = input_data[i]; + const float upper = 6; + const float lower = 0; + const float clamped = val > upper ? upper : val < lower ? lower : val; + output_data[i] = clamped; } } @@ -757,24 +709,17 @@ template void L2Normalization(const float* input_data, const Dims<4>& input_dims, float* output_data, const Dims<4>& output_dims) { static_assert(Ac == FusedActivationFunctionType::kNone, ""); - const int batches = MatchingArraySize(input_dims, 3, output_dims, 3); - const int height = MatchingArraySize(input_dims, 2, output_dims, 2); - const int width = MatchingArraySize(input_dims, 1, output_dims, 1); + const int outer_size = MatchingFlatSizeSkipDim(input_dims, 0, output_dims); const int depth = MatchingArraySize(input_dims, 0, output_dims, 0); - for (int b = 0; b < batches; ++b) { - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - float squared_l2_norm = 0; - for (int c = 0; c < depth; ++c) { - float val = input_data[Offset(input_dims, c, x, y, b)]; - squared_l2_norm += val * val; - } - float l2_norm = std::sqrt(squared_l2_norm); - for (int c = 0; c < depth; ++c) { - output_data[Offset(output_dims, c, x, y, b)] = - input_data[Offset(input_dims, c, x, y, b)] / l2_norm; - } - } + for (int i = 0; i < outer_size; ++i) { + float squared_l2_norm = 0; + for (int c = 0; c < depth; ++c) { + const float val = input_data[depth * i + c]; + squared_l2_norm += val * val; + } + const float l2_norm = std::sqrt(squared_l2_norm); + for (int c = 0; c < depth; ++c) { + output_data[depth * i + c] = input_data[depth * i + c] / l2_norm; } } } @@ -859,26 +804,11 @@ inline void Add(const float* input1_data, const Dims<4>& input1_dims, const float* input2_data, const Dims<4>& input2_dims, float output_activation_min, float output_activation_max, float* output_data, const Dims<4>& output_dims) { - const int batches = - MatchingArraySize(input1_dims, 3, input2_dims, 3, output_dims, 3); - const int height = - MatchingArraySize(input1_dims, 2, input2_dims, 2, output_dims, 2); - const int width = - MatchingArraySize(input1_dims, 1, input2_dims, 1, output_dims, 1); - const int depth = - MatchingArraySize(input1_dims, 0, input2_dims, 0, output_dims, 0); - for (int b = 0; b < batches; ++b) { - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - for (int c = 0; c < depth; ++c) { - output_data[Offset(output_dims, c, x, y, b)] = - ActivationFunctionWithMinMax( - input1_data[Offset(input1_dims, c, x, y, b)] + - input2_data[Offset(input2_dims, c, x, y, b)], - output_activation_min, output_activation_max); - } - } - } + const int flat_size = MatchingFlatSize(input1_dims, input2_dims, output_dims); + for (int i = 0; i < flat_size; ++i) { + output_data[i] = ActivationFunctionWithMinMax( + input1_data[i] + input2_data[i], output_activation_min, + output_activation_max); } } @@ -1141,26 +1071,11 @@ inline void Mul(const float* input1_data, const Dims<4>& input1_dims, const float* input2_data, const Dims<4>& input2_dims, float output_activation_min, float output_activation_max, float* output_data, const Dims<4>& output_dims) { - const int batches = - MatchingArraySize(input1_dims, 3, input2_dims, 3, output_dims, 3); - const int height = - MatchingArraySize(input1_dims, 2, input2_dims, 2, output_dims, 2); - const int width = - MatchingArraySize(input1_dims, 1, input2_dims, 1, output_dims, 1); - const int depth = - MatchingArraySize(input1_dims, 0, input2_dims, 0, output_dims, 0); - for (int b = 0; b < batches; ++b) { - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - for (int c = 0; c < depth; ++c) { - output_data[Offset(output_dims, c, x, y, b)] = - ActivationFunctionWithMinMax( - input1_data[Offset(input1_dims, c, x, y, b)] * - input2_data[Offset(input2_dims, c, x, y, b)], - output_activation_min, output_activation_max); - } - } - } + const int flat_size = MatchingFlatSize(input1_dims, input2_dims, output_dims); + for (int i = 0; i < flat_size; ++i) { + output_data[i] = ActivationFunctionWithMinMax( + input1_data[i] * input2_data[i], output_activation_min, + output_activation_max); } } @@ -1384,26 +1299,11 @@ inline void Div(const float* input1_data, const Dims<4>& input1_dims, const float* input2_data, const Dims<4>& input2_dims, float output_activation_min, float output_activation_max, float* output_data, const Dims<4>& output_dims) { - const int batches = - MatchingArraySize(input1_dims, 3, input2_dims, 3, output_dims, 3); - const int height = - MatchingArraySize(input1_dims, 2, input2_dims, 2, output_dims, 2); - const int width = - MatchingArraySize(input1_dims, 1, input2_dims, 1, output_dims, 1); - const int depth = - MatchingArraySize(input1_dims, 0, input2_dims, 0, output_dims, 0); - for (int b = 0; b < batches; ++b) { - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - for (int c = 0; c < depth; ++c) { - output_data[Offset(output_dims, c, x, y, b)] = - ActivationFunctionWithMinMax( - input1_data[Offset(input1_dims, c, x, y, b)] / - input2_data[Offset(input2_dims, c, x, y, b)], - output_activation_min, output_activation_max); - } - } - } + const int flat_size = MatchingFlatSize(input1_dims, input2_dims, output_dims); + for (int i = 0; i < flat_size; ++i) { + output_data[i] = ActivationFunctionWithMinMax( + input1_data[i] / input2_data[i], output_activation_min, + output_activation_max); } } @@ -1411,26 +1311,11 @@ inline void Sub(const float* input1_data, const Dims<4>& input1_dims, const float* input2_data, const Dims<4>& input2_dims, float output_activation_min, float output_activation_max, float* output_data, const Dims<4>& output_dims) { - const int batches = - MatchingArraySize(input1_dims, 3, input2_dims, 3, output_dims, 3); - const int height = - MatchingArraySize(input1_dims, 2, input2_dims, 2, output_dims, 2); - const int width = - MatchingArraySize(input1_dims, 1, input2_dims, 1, output_dims, 1); - const int depth = - MatchingArraySize(input1_dims, 0, input2_dims, 0, output_dims, 0); - for (int b = 0; b < batches; ++b) { - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - for (int c = 0; c < depth; ++c) { - output_data[Offset(output_dims, c, x, y, b)] = - ActivationFunctionWithMinMax( - input1_data[Offset(input1_dims, c, x, y, b)] - - input2_data[Offset(input2_dims, c, x, y, b)], - output_activation_min, output_activation_max); - } - } - } + const int flat_size = MatchingFlatSize(input1_dims, input2_dims, output_dims); + for (int i = 0; i < flat_size; ++i) { + output_data[i] = ActivationFunctionWithMinMax( + input1_data[i] - input2_data[i], output_activation_min, + output_activation_max); } } @@ -1812,15 +1697,9 @@ void LstmCell(const uint8* input_data_uint8, const Dims<4>& input_dims, (void)gemm_context; // only used in optimized code. // Gather dimensions information, and perform consistency checks. - const int batches = - MatchingArraySize(input_dims, 3, prev_activ_dims, 3, prev_state_dims, 3, - output_state_dims, 3, output_activ_dims, 3); - const int height = - MatchingArraySize(input_dims, 2, prev_activ_dims, 2, prev_state_dims, 2, - output_state_dims, 2, output_activ_dims, 2); - const int width = - MatchingArraySize(input_dims, 1, prev_activ_dims, 1, prev_state_dims, 1, - output_state_dims, 1, output_activ_dims, 1); + const int outer_size = + MatchingFlatSizeSkipDim(input_dims, 0, prev_activ_dims, prev_state_dims, + output_state_dims, output_activ_dims); TFLITE_CHECK_EQ(ArraySize(weights_dims, 2), 1); TFLITE_CHECK_EQ(ArraySize(weights_dims, 3), 1); const int input_depth = ArraySize(input_dims, 0); @@ -1836,9 +1715,7 @@ void LstmCell(const uint8* input_data_uint8, const Dims<4>& input_dims, MatchingArraySize(prev_state_dims, 0, prev_activ_dims, 0, output_state_dims, 0, output_activ_dims, 0); TFLITE_CHECK_EQ(output_depth, intern_activ_depth / 4); - const int fc_batches = ArraySize(activ_temp_dims, 1) * - ArraySize(activ_temp_dims, 2) * - ArraySize(activ_temp_dims, 3); + const int fc_batches = FlatSizeSkipDim(activ_temp_dims, 0); const int fc_output_depth = MatchingArraySize(weights_dims, 1, activ_temp_dims, 0); const int fc_accum_depth = ArraySize(weights_dims, 0); @@ -1883,7 +1760,6 @@ void LstmCell(const uint8* input_data_uint8, const Dims<4>& input_dims, // Rest of the LSTM cell: tanh and logistic math functions, and some adds // and muls, all done in 16-bit fixed-point. - const int outer_size = batches * width * height; for (int b = 0; b < outer_size; ++b) { for (int c = 0; c < output_depth; ++c) { // Define the fixed-point data types that we will use here. All use @@ -2418,28 +2294,20 @@ inline void LocalResponseNormalization(const float* input_data, float bias, float alpha, float beta, float* output_data, const Dims<4>& output_dims) { - const int batches = MatchingArraySize(input_dims, 3, output_dims, 3); - const int height = MatchingArraySize(input_dims, 2, output_dims, 2); - const int width = MatchingArraySize(input_dims, 1, output_dims, 1); + const int outer_size = MatchingFlatSizeSkipDim(input_dims, 0, output_dims); const int depth = MatchingArraySize(input_dims, 0, output_dims, 0); - for (int b = 0; b < batches; ++b) { - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - for (int c = 0; c < depth; ++c) { - const int begin_input_c = std::max(0, c - range); - const int end_input_c = std::min(depth, c + range); - float accum = 0.f; - for (int input_c = begin_input_c; input_c < end_input_c; ++input_c) { - const float input_val = - input_data[Offset(input_dims, input_c, x, y, b)]; - accum += input_val * input_val; - } - const float multiplier = std::pow(bias + alpha * accum, -beta); - output_data[Offset(output_dims, c, x, y, b)] = - input_data[Offset(input_dims, c, x, y, b)] * multiplier; - } + for (int i = 0; i < outer_size; ++i) { + for (int c = 0; c < depth; ++c) { + const int begin_input_c = std::max(0, c - range); + const int end_input_c = std::min(depth, c + range); + float accum = 0.f; + for (int input_c = begin_input_c; input_c < end_input_c; ++input_c) { + const float input_val = input_data[i * depth + input_c]; + accum += input_val * input_val; } + const float multiplier = std::pow(bias + alpha * accum, -beta); + output_data[i * depth + c] = input_data[i * depth + c] * multiplier; } } } @@ -2447,37 +2315,28 @@ inline void LocalResponseNormalization(const float* input_data, inline void Softmax(const float* input_data, const Dims<4>& input_dims, float beta, float* output_data, const Dims<4>& output_dims) { - const int batches = MatchingArraySize(input_dims, 3, output_dims, 3); - const int height = MatchingArraySize(input_dims, 2, output_dims, 2); - const int width = MatchingArraySize(input_dims, 1, output_dims, 1); + const int outer_size = MatchingFlatSizeSkipDim(input_dims, 0, output_dims); const int depth = MatchingArraySize(input_dims, 0, output_dims, 0); - for (int b = 0; b < batches; ++b) { - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - // Find max element value which we'll use to ensure numerical stability - // taking advantage of the following equality: - // exp(x[i])/sum(exp(x[i])) == exp(x[i]+C)/sum(exp(x[i]+C)) - float max = std::numeric_limits::lowest(); - for (int c = 0; c < depth; ++c) { - max = std::max(max, input_data[Offset(input_dims, c, x, y, b)]); - } + for (int i = 0; i < outer_size; ++i) { + // Find max element value which we'll use to ensure numerical stability + // taking advantage of the following equality: + // exp(x[i])/sum(exp(x[i])) == exp(x[i]+C)/sum(exp(x[i]+C)) + float max = std::numeric_limits::lowest(); + for (int c = 0; c < depth; ++c) { + max = std::max(max, input_data[i * depth + c]); + } - // Compute sum. - float sum = 0.f; - for (int c = 0; c < depth; ++c) { - sum += std::exp((input_data[Offset(input_dims, c, x, y, b)] - max) * - beta); - } + // Compute sum. + float sum = 0.f; + for (int c = 0; c < depth; ++c) { + sum += std::exp((input_data[i * depth + c] - max) * beta); + } - // Compute result. - for (int c = 0; c < depth; ++c) { - output_data[Offset(output_dims, c, x, y, b)] = - std::exp((input_data[Offset(input_dims, c, x, y, b)] - max) * - beta) / - sum; - } - } + // Compute result. + for (int c = 0; c < depth; ++c) { + output_data[i * depth + c] = + std::exp((input_data[i * depth + c] - max) * beta) / sum; } } } @@ -2498,73 +2357,63 @@ inline void Softmax(const uint8* input_data, const Dims<4>& input_dims, using FixedPointAccum = gemmlowp::FixedPoint; using FixedPoint0 = gemmlowp::FixedPoint; - const int batches = MatchingArraySize(input_dims, 3, output_dims, 3); - const int height = MatchingArraySize(input_dims, 2, output_dims, 2); - const int width = MatchingArraySize(input_dims, 1, output_dims, 1); + const int outer_size = MatchingFlatSizeSkipDim(input_dims, 0, output_dims); const int depth = MatchingArraySize(input_dims, 0, output_dims, 0); - for (int b = 0; b < batches; ++b) { - for (int x = 0; x < width; ++x) { - for (int y = 0; y < height; ++y) { - uint8 max_in_row = 0; - for (int c = 0; c < depth; ++c) { - max_in_row = - std::max(max_in_row, input_data[Offset(input_dims, c, x, y, b)]); - } + for (int i = 0; i < outer_size; ++i) { + uint8 max_in_row = 0; + for (int c = 0; c < depth; ++c) { + max_in_row = std::max(max_in_row, input_data[i * depth + c]); + } + + FixedPointAccum sum_of_exps = FixedPointAccum::Zero(); + for (int c = 0; c < depth; ++c) { + int32 input_diff = + static_cast(input_data[i * depth + c]) - max_in_row; + if (input_diff >= diff_min) { + const int32 input_diff_rescaled = + MultiplyByQuantizedMultiplierGreaterThanOne( + input_diff, input_beta_multiplier, input_beta_left_shift); + const FixedPointScaledDiff scaled_diff_f8 = + FixedPointScaledDiff::FromRaw(input_diff_rescaled); + sum_of_exps = sum_of_exps + gemmlowp::Rescale( + exp_on_negative_values(scaled_diff_f8)); + } + } + + int32 fixed_sum_of_exps = sum_of_exps.raw(); + int headroom_plus_one = + CountLeadingZeros(static_cast(fixed_sum_of_exps)); + // This is the number of bits to the left of the binary point above 1.0. + // Consider fixed_sum_of_exps=1.25. In that case shifted_scale=0.8 and + // no later adjustment will be needed. + int num_bits_over_unit = kAccumulationIntegerBits - headroom_plus_one; + int32 shifted_sum_minus_one = static_cast( + (static_cast(fixed_sum_of_exps) << headroom_plus_one) - + (static_cast(1) << 31)); + + FixedPoint0 shifted_scale = gemmlowp::one_over_one_plus_x_for_x_in_0_1( + FixedPoint0::FromRaw(shifted_sum_minus_one)); + + for (int c = 0; c < depth; ++c) { + int32 input_diff = + static_cast(input_data[i * depth + c]) - max_in_row; + if (input_diff >= diff_min) { + const int32 input_diff_rescaled = + MultiplyByQuantizedMultiplierGreaterThanOne( + input_diff, input_beta_multiplier, input_beta_left_shift); + const FixedPointScaledDiff scaled_diff_f8 = + FixedPointScaledDiff::FromRaw(input_diff_rescaled); + + FixedPoint0 exp_in_0 = exp_on_negative_values(scaled_diff_f8); + int32 unsat_output = gemmlowp::RoundingDivideByPOT( + (shifted_scale * exp_in_0).raw(), num_bits_over_unit + 31 - 8); + + output_data[i * depth + c] = static_cast( + std::max(std::min(unsat_output, static_cast(255)), 0)); - FixedPointAccum sum_of_exps = FixedPointAccum::Zero(); - for (int c = 0; c < depth; ++c) { - int32 input_diff = - static_cast(input_data[Offset(input_dims, c, x, y, b)]) - - max_in_row; - if (input_diff >= diff_min) { - const int32 input_diff_rescaled = - MultiplyByQuantizedMultiplierGreaterThanOne( - input_diff, input_beta_multiplier, input_beta_left_shift); - const FixedPointScaledDiff scaled_diff_f8 = - FixedPointScaledDiff::FromRaw(input_diff_rescaled); - sum_of_exps = - sum_of_exps + gemmlowp::Rescale( - exp_on_negative_values(scaled_diff_f8)); - } - } - - int32 fixed_sum_of_exps = sum_of_exps.raw(); - int headroom_plus_one = - CountLeadingZeros(static_cast(fixed_sum_of_exps)); - // This is the number of bits to the left of the binary point above 1.0. - // Consider fixed_sum_of_exps=1.25. In that case shifted_scale=0.8 and - // no later adjustment will be needed. - int num_bits_over_unit = kAccumulationIntegerBits - headroom_plus_one; - int32 shifted_sum_minus_one = static_cast( - (static_cast(fixed_sum_of_exps) << headroom_plus_one) - - (static_cast(1) << 31)); - - FixedPoint0 shifted_scale = gemmlowp::one_over_one_plus_x_for_x_in_0_1( - FixedPoint0::FromRaw(shifted_sum_minus_one)); - - for (int c = 0; c < depth; ++c) { - int32 input_diff = - static_cast(input_data[Offset(input_dims, c, x, y, b)]) - - max_in_row; - if (input_diff >= diff_min) { - const int32 input_diff_rescaled = - MultiplyByQuantizedMultiplierGreaterThanOne( - input_diff, input_beta_multiplier, input_beta_left_shift); - const FixedPointScaledDiff scaled_diff_f8 = - FixedPointScaledDiff::FromRaw(input_diff_rescaled); - - FixedPoint0 exp_in_0 = exp_on_negative_values(scaled_diff_f8); - int32 unsat_output = gemmlowp::RoundingDivideByPOT( - (shifted_scale * exp_in_0).raw(), num_bits_over_unit + 31 - 8); - - output_data[Offset(output_dims, c, x, y, b)] = static_cast( - std::max(std::min(unsat_output, static_cast(255)), 0)); - - } else { - output_data[Offset(output_dims, c, x, y, b)] = 0; - } - } + } else { + output_data[i * depth + c] = 0; } } } @@ -2572,55 +2421,40 @@ inline void Softmax(const uint8* input_data, const Dims<4>& input_dims, inline void LogSoftmax(const float* input_data, const Dims<4>& input_dims, float* output_data, const Dims<4>& output_dims) { - const int batches = MatchingArraySize(input_dims, 3, output_dims, 3); - const int height = MatchingArraySize(input_dims, 2, output_dims, 2); - const int width = MatchingArraySize(input_dims, 1, output_dims, 1); + const int outer_size = MatchingFlatSizeSkipDim(input_dims, 0, output_dims); const int depth = MatchingArraySize(input_dims, 0, output_dims, 0); - for (int b = 0; b < batches; ++b) { - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - // Find max element value which we'll use to ensure numerical stability - // taking advantage of the following equality: - // log(exp(x[i])/sum(exp(x[i]))) == log(exp(x[i]+C)/sum(exp(x[i]+C))) - float max = std::numeric_limits::lowest(); - for (int c = 0; c < depth; ++c) { - max = std::max(max, input_data[Offset(input_dims, c, x, y, b)]); - } + for (int i = 0; i < outer_size; ++i) { + // Find max element value which we'll use to ensure numerical stability + // taking advantage of the following equality: + // log(exp(x[i])/sum(exp(x[i]))) == log(exp(x[i]+C)/sum(exp(x[i]+C))) + float max = std::numeric_limits::lowest(); + for (int c = 0; c < depth; ++c) { + max = std::max(max, input_data[i * depth + c]); + } - // Compute sum. - float sum = 0.f; - for (int c = 0; c < depth; ++c) { - sum += std::exp(input_data[Offset(input_dims, c, x, y, b)] - max); - } + // Compute sum. + float sum = 0.f; + for (int c = 0; c < depth; ++c) { + sum += std::exp(input_data[i * depth + c] - max); + } - // Compute result. - const float log_sum = std::log(sum); - for (int c = 0; c < depth; ++c) { - output_data[Offset(output_dims, c, x, y, b)] = - input_data[Offset(input_dims, c, x, y, b)] - max - log_sum; - } - } + // Compute result. + const float log_sum = std::log(sum); + for (int c = 0; c < depth; ++c) { + output_data[i * depth + c] = input_data[i * depth + c] - max - log_sum; } } } inline void Logistic(const float* input_data, const Dims<4>& input_dims, float* output_data, const Dims<4>& output_dims) { - const int batches = MatchingArraySize(input_dims, 3, output_dims, 3); - const int height = MatchingArraySize(input_dims, 2, output_dims, 2); - const int width = MatchingArraySize(input_dims, 1, output_dims, 1); - const int depth = MatchingArraySize(input_dims, 0, output_dims, 0); - for (int b = 0; b < batches; ++b) { - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - for (int c = 0; c < depth; ++c) { - float val = input_data[Offset(input_dims, c, x, y, b)]; - float result = 1.f / (1.f + std::exp(-val)); - output_data[Offset(output_dims, c, x, y, b)] = result; - } - } - } + const int flat_size = MatchingFlatSize(output_dims, input_dims); + + for (int i = 0; i < flat_size; i++) { + float val = input_data[i]; + float result = 1.f / (1.f + std::exp(-val)); + output_data[i] = result; } } @@ -2628,53 +2462,43 @@ inline void Logistic(const uint8* input_data, const Dims<4>& input_dims, int32 input_zero_point, int32 input_range_radius, int32 input_multiplier, int input_left_shift, uint8* output_data, const Dims<4>& output_dims) { - const int batches = MatchingArraySize(input_dims, 3, output_dims, 3); - const int height = MatchingArraySize(input_dims, 2, output_dims, 2); - const int width = MatchingArraySize(input_dims, 1, output_dims, 1); - const int depth = MatchingArraySize(input_dims, 0, output_dims, 0); - for (int b = 0; b < batches; ++b) { - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - for (int c = 0; c < depth; ++c) { - const uint8 input_val_u8 = input_data[Offset(input_dims, c, x, y, b)]; - const int32 input_val_centered = - static_cast(input_val_u8) - input_zero_point; - uint8 output_val; - if (input_val_centered <= -input_range_radius) { - output_val = 0; - } else if (input_val_centered >= input_range_radius) { - output_val = 255; - } else { - const int32 input_val_rescaled = - MultiplyByQuantizedMultiplierGreaterThanOne( - input_val_centered, input_multiplier, input_left_shift); - using FixedPoint4 = gemmlowp::FixedPoint; - using FixedPoint0 = gemmlowp::FixedPoint; - const FixedPoint4 input_val_f4 = - FixedPoint4::FromRaw(input_val_rescaled); - const FixedPoint0 output_val_f0 = gemmlowp::logistic(input_val_f4); - // Convert from Q0.31 to Q23.8. - using gemmlowp::RoundingDivideByPOT; - int32 output_val_s32 = RoundingDivideByPOT(output_val_f0.raw(), 23); - if (output_val_s32 == 256) { - output_val_s32 = 255; - } - // Reinterpret as U0.8. - TFLITE_DCHECK_GE(output_val_s32, 0); - TFLITE_DCHECK_LE(output_val_s32, 255); - output_val = static_cast(output_val_s32); - } - output_data[Offset(output_dims, c, x, y, b)] = output_val; - } + const int flat_size = MatchingFlatSize(output_dims, input_dims); + + for (int i = 0; i < flat_size; i++) { + const uint8 input_val_u8 = input_data[i]; + const int32 input_val_centered = + static_cast(input_val_u8) - input_zero_point; + uint8 output_val; + if (input_val_centered <= -input_range_radius) { + output_val = 0; + } else if (input_val_centered >= input_range_radius) { + output_val = 255; + } else { + const int32 input_val_rescaled = + MultiplyByQuantizedMultiplierGreaterThanOne( + input_val_centered, input_multiplier, input_left_shift); + using FixedPoint4 = gemmlowp::FixedPoint; + using FixedPoint0 = gemmlowp::FixedPoint; + const FixedPoint4 input_val_f4 = FixedPoint4::FromRaw(input_val_rescaled); + const FixedPoint0 output_val_f0 = gemmlowp::logistic(input_val_f4); + // Convert from Q0.31 to Q23.8. + using gemmlowp::RoundingDivideByPOT; + int32 output_val_s32 = RoundingDivideByPOT(output_val_f0.raw(), 23); + if (output_val_s32 == 256) { + output_val_s32 = 255; } + // Reinterpret as U0.8. + TFLITE_DCHECK_GE(output_val_s32, 0); + TFLITE_DCHECK_LE(output_val_s32, 255); + output_val = static_cast(output_val_s32); } + output_data[i] = output_val; } } inline void Logistic(const int16* input_data, const Dims<4>& input_dims, int16* output_data, const Dims<4>& output_dims) { - const int flat_size = RequiredBufferSizeForDims(output_dims); - TFLITE_DCHECK_EQ(RequiredBufferSizeForDims(input_dims), flat_size); + const int flat_size = MatchingFlatSize(output_dims, input_dims); for (int i = 0; i < flat_size; i++) { // F0 uses 0 integer bits, range [-1, 1]. @@ -2692,20 +2516,12 @@ inline void Logistic(const int16* input_data, const Dims<4>& input_dims, inline void Tanh(const float* input_data, const Dims<4>& input_dims, float* output_data, const Dims<4>& output_dims) { - const int batches = MatchingArraySize(input_dims, 3, output_dims, 3); - const int height = MatchingArraySize(input_dims, 2, output_dims, 2); - const int width = MatchingArraySize(input_dims, 1, output_dims, 1); - const int depth = MatchingArraySize(input_dims, 0, output_dims, 0); - for (int b = 0; b < batches; ++b) { - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - for (int c = 0; c < depth; ++c) { - float val = input_data[Offset(input_dims, c, x, y, b)]; - float result = std::tanh(val); - output_data[Offset(output_dims, c, x, y, b)] = result; - } - } - } + const int flat_size = MatchingFlatSize(output_dims, input_dims); + + for (int i = 0; i < flat_size; i++) { + float val = input_data[i]; + float result = std::tanh(val); + output_data[i] = result; } } @@ -2714,47 +2530,38 @@ inline void Tanh(const uint8* input_data, const Dims<4>& input_dims, int32 input_multiplier, int input_left_shift, uint8* output_data, const Dims<4>& output_dims) { const int32 output_zero_point = 128; - const int batches = MatchingArraySize(input_dims, 3, output_dims, 3); - const int height = MatchingArraySize(input_dims, 2, output_dims, 2); - const int width = MatchingArraySize(input_dims, 1, output_dims, 1); - const int depth = MatchingArraySize(input_dims, 0, output_dims, 0); - for (int b = 0; b < batches; ++b) { - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - for (int c = 0; c < depth; ++c) { - const uint8 input_val_u8 = input_data[Offset(input_dims, c, x, y, b)]; - const int32 input_val_centered = - static_cast(input_val_u8) - input_zero_point; - uint8 output_val; - if (input_val_centered <= -input_range_radius) { - output_val = 0; - } else if (input_val_centered >= input_range_radius) { - output_val = 255; - } else { - const int32 input_val_rescaled = - MultiplyByQuantizedMultiplierGreaterThanOne( - input_val_centered, input_multiplier, input_left_shift); - using FixedPoint4 = gemmlowp::FixedPoint; - using FixedPoint0 = gemmlowp::FixedPoint; - const FixedPoint4 input_val_f4 = - FixedPoint4::FromRaw(input_val_rescaled); - const FixedPoint0 output_val_f0 = gemmlowp::tanh(input_val_f4); - // Convert from Q0.31 to Q24.7. - using gemmlowp::RoundingDivideByPOT; - int32 output_val_s32 = RoundingDivideByPOT(output_val_f0.raw(), 24); - output_val_s32 += output_zero_point; - if (output_val_s32 == 256) { - output_val_s32 = 255; - } - // Reinterpret as Q0.7, encoded in uint8. - TFLITE_DCHECK_GE(output_val_s32, 0); - TFLITE_DCHECK_LE(output_val_s32, 255); - output_val = static_cast(output_val_s32); - } - output_data[Offset(output_dims, c, x, y, b)] = output_val; - } + const int flat_size = MatchingFlatSize(output_dims, input_dims); + + for (int i = 0; i < flat_size; i++) { + const uint8 input_val_u8 = input_data[i]; + const int32 input_val_centered = + static_cast(input_val_u8) - input_zero_point; + uint8 output_val; + if (input_val_centered <= -input_range_radius) { + output_val = 0; + } else if (input_val_centered >= input_range_radius) { + output_val = 255; + } else { + const int32 input_val_rescaled = + MultiplyByQuantizedMultiplierGreaterThanOne( + input_val_centered, input_multiplier, input_left_shift); + using FixedPoint4 = gemmlowp::FixedPoint; + using FixedPoint0 = gemmlowp::FixedPoint; + const FixedPoint4 input_val_f4 = FixedPoint4::FromRaw(input_val_rescaled); + const FixedPoint0 output_val_f0 = gemmlowp::tanh(input_val_f4); + // Convert from Q0.31 to Q24.7. + using gemmlowp::RoundingDivideByPOT; + int32 output_val_s32 = RoundingDivideByPOT(output_val_f0.raw(), 24); + output_val_s32 += output_zero_point; + if (output_val_s32 == 256) { + output_val_s32 = 255; } + // Reinterpret as Q0.7, encoded in uint8. + TFLITE_DCHECK_GE(output_val_s32, 0); + TFLITE_DCHECK_LE(output_val_s32, 255); + output_val = static_cast(output_val_s32); } + output_data[i] = output_val; } } @@ -2766,8 +2573,7 @@ inline void Tanh(const int16* input_data, const Dims<4>& input_dims, TFLITE_DCHECK_GE(input_left_shift, 0); TFLITE_DCHECK_LE(input_left_shift, 1); - const int flat_size = RequiredBufferSizeForDims(output_dims); - TFLITE_DCHECK_EQ(RequiredBufferSizeForDims(input_dims), flat_size); + const int flat_size = MatchingFlatSize(output_dims, input_dims); // F0 uses 0 integer bits, range [-1, 1]. // This is the return type of math functions such as tanh, logistic, @@ -2795,20 +2601,12 @@ inline void Tanh(const int16* input_data, const Dims<4>& input_dims, inline void Dequantize(const uint8* input_data, const Dims<4>& input_dims, int32 zero_point, double scale, float* output_data, const Dims<4>& output_dims) { - const int batches = MatchingArraySize(input_dims, 3, output_dims, 3); - const int height = MatchingArraySize(input_dims, 2, output_dims, 2); - const int width = MatchingArraySize(input_dims, 1, output_dims, 1); - const int depth = MatchingArraySize(input_dims, 0, output_dims, 0); - for (int b = 0; b < batches; ++b) { - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - for (int c = 0; c < depth; ++c) { - int32 val = input_data[Offset(input_dims, c, x, y, b)]; - float result = static_cast(scale * (val - zero_point)); - output_data[Offset(output_dims, c, x, y, b)] = result; - } - } - } + const int flat_size = MatchingFlatSize(output_dims, input_dims); + + for (int i = 0; i < flat_size; i++) { + int32 val = input_data[i]; + float result = static_cast(scale * (val - zero_point)); + output_data[i] = result; } } @@ -2872,61 +2670,37 @@ inline void FakeQuant(const float* input_data, const Dims<4>& input_dims, TFLITE_DCHECK_LE(zero_point, qmax); } - const int batches = MatchingArraySize(input_dims, 3, output_dims, 3); - const int height = MatchingArraySize(input_dims, 2, output_dims, 2); - const int width = MatchingArraySize(input_dims, 1, output_dims, 1); - const int depth = MatchingArraySize(input_dims, 0, output_dims, 0); - for (int b = 0; b < batches; ++b) { - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - for (int c = 0; c < depth; ++c) { - const float src_val = input_data[Offset(input_dims, c, x, y, b)]; - const float unclamped_quantized_val = - TfLiteRound(zero_point + src_val / scale); - const float quantized_val = std::min( - qmax_float, std::max(qmin_float, unclamped_quantized_val)); - const float dst_val = scale * (quantized_val - zero_point); - output_data[Offset(output_dims, c, x, y, b)] = dst_val; - } - } - } + const int flat_size = MatchingFlatSize(output_dims, input_dims); + + for (int i = 0; i < flat_size; i++) { + const float src_val = input_data[i]; + const float unclamped_quantized_val = + TfLiteRound(zero_point + src_val / scale); + const float quantized_val = + std::min(qmax_float, std::max(qmin_float, unclamped_quantized_val)); + const float dst_val = scale * (quantized_val - zero_point); + output_data[i] = dst_val; } } template inline void Cast(const SrcT* input_data, const Dims<4>& input_dims, DstT* output_data, const Dims<4>& output_dims) { - const int batches = MatchingArraySize(input_dims, 3, output_dims, 3); - const int height = MatchingArraySize(input_dims, 2, output_dims, 2); - const int width = MatchingArraySize(input_dims, 1, output_dims, 1); - const int depth = MatchingArraySize(input_dims, 0, output_dims, 0); - for (int b = 0; b < batches; ++b) { - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - for (int c = 0; c < depth; ++c) { - int offset = Offset(input_dims, c, x, y, b); - output_data[offset] = static_cast(input_data[offset]); - } - } - } + const int flat_size = MatchingFlatSize(output_dims, input_dims); + + for (int i = 0; i < flat_size; i++) { + int offset = i; + output_data[offset] = static_cast(input_data[offset]); } } inline void Floor(const float* input_data, const Dims<4>& input_dims, float* output_data, const Dims<4>& output_dims) { - const int batches = MatchingArraySize(input_dims, 3, output_dims, 3); - const int height = MatchingArraySize(input_dims, 2, output_dims, 2); - const int width = MatchingArraySize(input_dims, 1, output_dims, 1); - const int depth = MatchingArraySize(input_dims, 0, output_dims, 0); - for (int b = 0; b < batches; ++b) { - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - for (int c = 0; c < depth; ++c) { - int offset = Offset(input_dims, c, x, y, b); - output_data[offset] = std::floor(input_data[offset]); - } - } - } + const int flat_size = MatchingFlatSize(output_dims, input_dims); + + for (int i = 0; i < flat_size; i++) { + int offset = i; + output_data[offset] = std::floor(input_data[offset]); } } @@ -3375,23 +3149,11 @@ template void TensorFlowMinimum(const T* input1_data, const Dims<4>& input1_dims, const T* input2_data, T* output_data, const Dims<4>& output_dims) { - int batches = MatchingArraySize(input1_dims, 3, output_dims, 3); - int input_height = MatchingArraySize(input1_dims, 2, output_dims, 2); - int input_width = MatchingArraySize(input1_dims, 1, output_dims, 1); - int depth = MatchingArraySize(input1_dims, 0, output_dims, 0); + const int flat_size = MatchingFlatSize(output_dims, input1_dims); auto min_value = input2_data[0]; - - for (int b = 0; b < batches; b++) { - for (int y = 0; y < input_height; y++) { - for (int x = 0; x < input_width; x++) { - for (int c = 0; c < depth; c++) { - int offset = Offset(input1_dims, c, x, y, b); - output_data[offset] = - input1_data[offset] > min_value ? min_value : input1_data[offset]; - } - } - } + for (int i = 0; i < flat_size; i++) { + output_data[i] = input1_data[i] > min_value ? min_value : input1_data[i]; } } @@ -3399,23 +3161,11 @@ template void TensorFlowMaximum(const T* input1_data, const Dims<4>& input1_dims, const T* input2_data, T* output_data, const Dims<4>& output_dims) { - int batches = MatchingArraySize(input1_dims, 3, output_dims, 3); - int input_height = MatchingArraySize(input1_dims, 2, output_dims, 2); - int input_width = MatchingArraySize(input1_dims, 1, output_dims, 1); - int depth = MatchingArraySize(input1_dims, 0, output_dims, 0); + const int flat_size = MatchingFlatSize(output_dims, input1_dims); auto max_value = input2_data[0]; - - for (int b = 0; b < batches; b++) { - for (int y = 0; y < input_height; y++) { - for (int x = 0; x < input_width; x++) { - for (int c = 0; c < depth; c++) { - int offset = Offset(input1_dims, c, x, y, b); - output_data[offset] = - input1_data[offset] < max_value ? max_value : input1_data[offset]; - } - } - } + for (int i = 0; i < flat_size; i++) { + output_data[i] = input1_data[i] < max_value ? max_value : input1_data[i]; } } @@ -3456,25 +3206,20 @@ void ArgMax(const T3* axis, const T1* input_data, const Dims<4>& input_dims, // input dimensions here. We enforce the constraint that the last dimension // must always be 1. TFLITE_DCHECK_EQ(ArraySize(output_dims, 0), 1); - const int batches = MatchingArraySize(input_dims, 3, output_dims, 3); - const int height = MatchingArraySize(input_dims, 2, output_dims, 2); - const int width = MatchingArraySize(input_dims, 1, output_dims, 1); + const int outer_size = MatchingFlatSizeSkipDim(input_dims, 0, output_dims); const int depth = ArraySize(input_dims, 0); - for (int b = 0; b < batches; ++b) { - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - auto max_value = input_data[Offset(input_dims, 0, x, y, b)]; - int max_index = 0; - for (int d = 1; d < depth; ++d) { - const auto& curr_value = input_data[Offset(input_dims, d, x, y, b)]; - if (curr_value > max_value) { - max_value = curr_value; - max_index = d; - } - } - output_data[Offset(output_dims, 0, x, y, b)] = max_index; + + for (int i = 0; i < outer_size; ++i) { + auto max_value = input_data[i * depth]; + int max_index = 0; + for (int d = 1; d < depth; ++d) { + const auto& curr_value = input_data[i * depth + d]; + if (curr_value > max_value) { + max_value = curr_value; + max_index = d; } } + output_data[i] = max_index; } } @@ -3524,11 +3269,11 @@ inline void TransposeConv(const float* input_data, const Dims<4>& input_dims, // Although transpose convolution simplifies to convolution with transposed // weights for strides of 1, non-unitary striding complicates matters. To - // keep this reference implementation as clear as possible, we use a "scatter" - // access pattern, where we loop through all the input elements, computing - // their influence on the output, rather than looping through the output - // elements in the typical "gather" access pattern of a conv. We therefore - // must initialize the output array to zero. + // keep this reference implementation as clear as possible, we use a + // "scatter" access pattern, where we loop through all the input elements, + // computing their influence on the output, rather than looping through the + // output elements in the typical "gather" access pattern of a conv. We + // therefore must initialize the output array to zero. for (int i = 0; i < RequiredBufferSizeForDims(output_dims); i++) { output_data[i] = 0.0f; } diff --git a/tensorflow/contrib/lite/kernels/internal/types.h b/tensorflow/contrib/lite/kernels/internal/types.h index 293538fcbb..3290c364c1 100644 --- a/tensorflow/contrib/lite/kernels/internal/types.h +++ b/tensorflow/contrib/lite/kernels/internal/types.h @@ -130,14 +130,125 @@ int MatchingArraySize(const ArrayType1& array1, int index1, return MatchingArraySize(array1, index1, args...); } -inline int RequiredBufferSizeForDims(const Dims<4>& dims) { +template +inline int FlatSize(const Dims& dims) { int max_offset = 0; - for (int i = 0; i < 4; i++) { + for (int i = 0; i < N; i++) { max_offset += (dims.sizes[i] - 1) * dims.strides[i]; } return max_offset + 1; } +// Deprecated. Prefer FlatSize. +inline int RequiredBufferSizeForDims(const Dims<4>& dims) { + return FlatSize(dims); +} + +// Flat size calculation, checking that dimensions match with one or more other +// arrays. +template +inline int MatchingFlatSize(const Dims& dims, const Dims& check_dims_0) { + for (int i = 0; i < N; i++) { + TFLITE_DCHECK_EQ(ArraySize(dims, i), ArraySize(check_dims_0, i)); + } + return FlatSize(dims); +} + +template +inline int MatchingFlatSize(const Dims& dims, const Dims& check_dims_0, + const Dims& check_dims_1) { + for (int i = 0; i < N; i++) { + TFLITE_DCHECK_EQ(ArraySize(dims, i), ArraySize(check_dims_0, i)); + } + return MatchingFlatSize(dims, check_dims_1); +} + +template +inline int MatchingFlatSize(const Dims& dims, const Dims& check_dims_0, + const Dims& check_dims_1, + const Dims& check_dims_2) { + for (int i = 0; i < N; i++) { + TFLITE_DCHECK_EQ(ArraySize(dims, i), ArraySize(check_dims_0, i)); + } + return FlatSize(dims, check_dims_1, check_dims_2); +} + +template +inline int MatchingFlatSize(const Dims& dims, const Dims& check_dims_0, + const Dims& check_dims_1, + const Dims& check_dims_2, + const Dims& check_dims_3) { + for (int i = 0; i < N; i++) { + TFLITE_DCHECK_EQ(ArraySize(dims, i), ArraySize(check_dims_0, i)); + } + return FlatSize(dims, check_dims_1, check_dims_2, check_dims_3); +} + +// Data is required to be contiguous, and so many operators can use either the +// full array flat size or the flat size with one dimension skipped (commonly +// the depth). +template +inline int FlatSizeSkipDim(const Dims& dims, int skip_dim) { + TFLITE_DCHECK(skip_dim >= 0 && skip_dim < N); + int flat_size = 1; + for (int i = 0; i < N; i++) { + flat_size *= (i == skip_dim) ? 1 : dims.sizes[i]; + } + return flat_size; +} + +// A combination of MatchingFlatSize() and FlatSizeSkipDim(). +template +inline int MatchingFlatSizeSkipDim(const Dims& dims, int skip_dim, + const Dims& check_dims_0) { + for (int i = 0; i < N; i++) { + if (i != skip_dim) { + TFLITE_DCHECK_EQ(ArraySize(dims, i), ArraySize(check_dims_0, i)); + } + } + return FlatSizeSkipDim(dims, skip_dim); +} + +template +inline int MatchingFlatSizeSkipDim(const Dims& dims, int skip_dim, + const Dims& check_dims_0, + const Dims& check_dims_1) { + for (int i = 0; i < N; i++) { + if (i != skip_dim) { + TFLITE_DCHECK_EQ(ArraySize(dims, i), ArraySize(check_dims_0, i)); + } + } + return MatchingFlatSizeSkipDim(dims, skip_dim, check_dims_1); +} + +template +inline int MatchingFlatSizeSkipDim(const Dims& dims, int skip_dim, + const Dims& check_dims_0, + const Dims& check_dims_1, + const Dims& check_dims_2) { + for (int i = 0; i < N; i++) { + if (i != skip_dim) { + TFLITE_DCHECK_EQ(ArraySize(dims, i), ArraySize(check_dims_0, i)); + } + } + return MatchingFlatSizeSkipDim(dims, skip_dim, check_dims_1, check_dims_2); +} + +template +inline int MatchingFlatSizeSkipDim(const Dims& dims, int skip_dim, + const Dims& check_dims_0, + const Dims& check_dims_1, + const Dims& check_dims_2, + const Dims& check_dims_3) { + for (int i = 0; i < N; i++) { + if (i != skip_dim) { + TFLITE_DCHECK_EQ(ArraySize(dims, i), ArraySize(check_dims_0, i)); + } + } + return MatchingFlatSizeSkipDim(dims, skip_dim, check_dims_1, check_dims_2, + check_dims_3); +} + template bool IsPackedWithoutStrides(const Dims& dims) { int expected_stride = 1; -- GitLab From fe3f9dddb39171dd7cd9fbb9e044a40e08072c50 Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Tue, 10 Apr 2018 12:41:29 -0700 Subject: [PATCH 101/791] Make custom_graph_optimizer_registry header-only (#18387) Adds it as a dependency to libtensorflow_framework.so so its symbols are available to shared objects which want to register optimizers. No other rules include it, so shared objects won't accidentally get their own version of the registry. --- tensorflow/BUILD | 5 +++-- tensorflow/core/grappler/optimizers/BUILD | 23 ++++++++++++++++++++++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/tensorflow/BUILD b/tensorflow/BUILD index cfafffdd13..f2ad16fa04 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -450,11 +450,12 @@ tf_cc_shared_object( linkstatic = 1, visibility = ["//visibility:public"], deps = [ + "//tensorflow/core:core_cpu_impl", "//tensorflow/core:framework_internal_impl", + "//tensorflow/core:gpu_runtime_impl", + "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry_impl", "//tensorflow/core:lib_internal_impl", - "//tensorflow/core:core_cpu_impl", "//tensorflow/stream_executor:stream_executor_impl", - "//tensorflow/core:gpu_runtime_impl", ] + tf_additional_binary_deps(), ) diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD index e4bc030885..696cbd6d79 100644 --- a/tensorflow/core/grappler/optimizers/BUILD +++ b/tensorflow/core/grappler/optimizers/BUILD @@ -11,6 +11,10 @@ load( "//tensorflow/core:platform/default/build_config.bzl", "tf_protos_grappler", ) +load( + "//tensorflow/core:platform/default/build_config_root.bzl", + "if_static", +) cc_library( name = "static_schedule", @@ -532,11 +536,28 @@ tf_cuda_cc_test( ], ) +# This rule is header-only unless the build is static (--config=monolithic). Its +# implementation is included directly in the framework shared object. cc_library( name = "custom_graph_optimizer_registry", - srcs = ["custom_graph_optimizer_registry.cc"], hdrs = ["custom_graph_optimizer_registry.h"], visibility = ["//visibility:public"], + deps = [ + ":custom_graph_optimizer", + "//tensorflow/core:lib", + ] + if_static( + [":custom_graph_optimizer_registry_impl"], + ), +) + +# This rule contains static variables for the optimizer registry. Do not depend +# on it directly; use :custom_graph_optimizer_registry, and link against +# libtensorflow_framework.so for the registry symbols. +cc_library( + name = "custom_graph_optimizer_registry_impl", + srcs = ["custom_graph_optimizer_registry.cc"], + hdrs = ["custom_graph_optimizer_registry.h"], + visibility = ["//tensorflow:__subpackages__"], deps = [ ":custom_graph_optimizer", "//tensorflow/core:lib", -- GitLab From 4bf8270ed534c4cd37160e757d7b8a3dc765d1f0 Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Tue, 10 Apr 2018 12:54:03 -0700 Subject: [PATCH 102/791] Checkpointable: wrap restore ops in init_scope This should make restore() work with defun-wrapped code, when variables are created inside the function. Just lifts the restore code into the outer context. Adds a test for it. PiperOrigin-RevId: 192331065 --- .../eager/python/checkpointable_utils_test.py | 45 ++++++++++++++ .../optimizer_v2/checkpointable_utils_test.py | 45 ++++++++++++++ tensorflow/python/training/checkpointable.py | 58 ++++++++++--------- 3 files changed, 120 insertions(+), 28 deletions(-) diff --git a/tensorflow/contrib/eager/python/checkpointable_utils_test.py b/tensorflow/contrib/eager/python/checkpointable_utils_test.py index 3ec5c3de39..688befa772 100644 --- a/tensorflow/contrib/eager/python/checkpointable_utils_test.py +++ b/tensorflow/contrib/eager/python/checkpointable_utils_test.py @@ -25,6 +25,7 @@ from tensorflow.contrib.eager.python import checkpointable_utils from tensorflow.python.client import session as session_lib from tensorflow.python.eager import backprop from tensorflow.python.eager import context +from tensorflow.python.eager import function from tensorflow.python.eager import test from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes @@ -580,6 +581,50 @@ class CheckpointingTests(test.TestCase): self.assertEqual(training_continuation + 1, self.evaluate(root.save_counter)) + # pylint: disable=cell-var-from-loop + @test_util.run_in_graph_and_eager_modes() + def testWithDefun(self): + num_training_steps = 2 + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + for training_continuation in range(3): + with ops.Graph().as_default(), self.test_session( + graph=ops.get_default_graph()), test_util.device(use_gpu=True): + model = MyModel() + # Don't actually train so we can test variable values + optimizer = adam.AdamOptimizer(0.) + root = checkpointable_utils.Checkpoint( + optimizer=optimizer, model=model, + global_step=training_util.get_or_create_global_step()) + checkpoint_path = core_saver.latest_checkpoint(checkpoint_directory) + status = root.restore(save_path=checkpoint_path) + def train_fn(): + @function.defun + def _call_model(x): + return model(x) + with backprop.GradientTape() as tape: + loss = _call_model(constant_op.constant([[3.]])) + gradients = tape.gradient(loss, model.variables) + return optimizer.apply_gradients(zip(gradients, model.variables), + global_step=root.global_step) + if not context.executing_eagerly(): + train_fn = functools.partial( + self.evaluate, train_fn()) + status.initialize_or_restore() + for _ in range(num_training_steps): + train_fn() + if training_continuation > 0: + status.assert_consumed() + self.assertAllClose([[42.]], self.evaluate(model.variables[0])) + else: + self.evaluate(model.variables[0].assign([[42.]])) + root.save(file_prefix=checkpoint_prefix) + self.assertEqual((training_continuation + 1) * num_training_steps, + self.evaluate(root.global_step)) + self.assertEqual(training_continuation + 1, + self.evaluate(root.save_counter)) + # pylint: enable=cell-var-from-loop + def _get_checkpoint_name(self, name): root = checkpointable.Checkpointable() checkpointable_utils.add_variable( diff --git a/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py b/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py index 08f9699e85..abcffeb618 100644 --- a/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py +++ b/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py @@ -29,6 +29,7 @@ from tensorflow.contrib.optimizer_v2 import adam from tensorflow.python.client import session as session_lib from tensorflow.python.eager import backprop from tensorflow.python.eager import context +from tensorflow.python.eager import function from tensorflow.python.eager import test from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes @@ -372,6 +373,50 @@ class CheckpointingTests(test.TestCase): self.assertEqual(training_continuation + 1, self.evaluate(root.save_counter)) + # pylint: disable=cell-var-from-loop + @test_util.run_in_graph_and_eager_modes() + def testWithDefun(self): + num_training_steps = 2 + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + for training_continuation in range(3): + with ops.Graph().as_default(), self.test_session( + graph=ops.get_default_graph()), test_util.device(use_gpu=True): + model = MyModel() + # Don't actually train so we can test variable values + optimizer = adam.AdamOptimizer(0.) + root = checkpointable_utils.Checkpoint( + optimizer=optimizer, model=model, + global_step=training_util.get_or_create_global_step()) + checkpoint_path = core_saver.latest_checkpoint(checkpoint_directory) + status = root.restore(save_path=checkpoint_path) + def train_fn(): + @function.defun + def _call_model(x): + return model(x) + with backprop.GradientTape() as tape: + loss = _call_model(constant_op.constant([[3.]])) + gradients = tape.gradient(loss, model.variables) + return optimizer.apply_gradients(zip(gradients, model.variables), + global_step=root.global_step) + if not context.executing_eagerly(): + train_fn = functools.partial( + self.evaluate, train_fn()) + status.initialize_or_restore() + for _ in range(num_training_steps): + train_fn() + if training_continuation > 0: + status.assert_consumed() + self.assertAllClose([[42.]], self.evaluate(model.variables[0])) + else: + self.evaluate(model.variables[0].assign([[42.]])) + root.save(file_prefix=checkpoint_prefix) + self.assertEqual((training_continuation + 1) * num_training_steps, + self.evaluate(root.global_step)) + self.assertEqual(training_continuation + 1, + self.evaluate(root.save_counter)) + # pylint: enable=cell-var-from-loop + def _get_checkpoint_name(self, name): root = checkpointable.Checkpointable() checkpointable_utils.add_variable( diff --git a/tensorflow/python/training/checkpointable.py b/tensorflow/python/training/checkpointable.py index bbbe1e8ac5..9bf48df22e 100644 --- a/tensorflow/python/training/checkpointable.py +++ b/tensorflow/python/training/checkpointable.py @@ -94,12 +94,13 @@ class _CheckpointPosition(object): def restore(self, checkpointable): """Restore this value into `checkpointable`.""" - if self.bind_object(checkpointable): - # This object's correspondence with a checkpointed object is new, so - # process deferred restorations for it and its dependencies. - restore_ops = checkpointable._restore_from_checkpoint_position(self) # pylint: disable=protected-access - if restore_ops: - self._checkpoint.restore_ops.extend(restore_ops) + with ops.init_scope(): + if self.bind_object(checkpointable): + # This object's correspondence with a checkpointed object is new, so + # process deferred restorations for it and its dependencies. + restore_ops = checkpointable._restore_from_checkpoint_position(self) # pylint: disable=protected-access + if restore_ops: + self._checkpoint.restore_ops.extend(restore_ops) def bind_object(self, checkpointable): """Set a checkpoint<->object correspondence and process slot variables. @@ -409,28 +410,29 @@ class CheckpointableBase(object): "Checkpointable._add_variable called to create another with " "that name. Variable names must be unique within a Checkpointable " "object.") % (name,)) - if context.executing_eagerly(): - # If this is a variable with a single Tensor stored in the checkpoint, we - # can set that value as an initializer rather than initializing and then - # assigning (when executing eagerly). This call returns None if there is - # nothing to restore. - checkpoint_initializer = self._preload_simple_restoration( - name=name, shape=shape) - else: - checkpoint_initializer = None - if (checkpoint_initializer is not None - and not ( - isinstance(initializer, CheckpointInitialValue) - and initializer.restore_uid > checkpoint_initializer.restore_uid)): - # If multiple Checkpointable objects are "creating" the same variable via - # the magic of custom getters, the one with the highest restore UID (the - # one called last) has to make the final initializer. If another custom - # getter interrupts this process by overwriting the initializer, then - # we'll catch that when we call _track_checkpointable. So this is "best - # effort" to set the initializer with the highest restore UID. - initializer = checkpoint_initializer - shape = None - + with ops.init_scope(): + if context.executing_eagerly(): + # If this is a variable with a single Tensor stored in the checkpoint, + # we can set that value as an initializer rather than initializing and + # then assigning (when executing eagerly). This call returns None if + # there is nothing to restore. + checkpoint_initializer = self._preload_simple_restoration( + name=name, shape=shape) + else: + checkpoint_initializer = None + if (checkpoint_initializer is not None + and not ( + isinstance(initializer, CheckpointInitialValue) + and (initializer.restore_uid + > checkpoint_initializer.restore_uid))): + # If multiple Checkpointable objects are "creating" the same variable + # via the magic of custom getters, the one with the highest restore UID + # (the one called last) has to make the final initializer. If another + # custom getter interrupts this process by overwriting the initializer, + # then we'll catch that when we call _track_checkpointable. So this is + # "best effort" to set the initializer with the highest restore UID. + initializer = checkpoint_initializer + shape = None new_variable = getter( name=name, shape=shape, dtype=dtype, initializer=initializer, **kwargs_for_getter) -- GitLab From 6b593d329005ffb1a10b1c9cd1374d2cdb620b21 Mon Sep 17 00:00:00 2001 From: Smit Hinsu Date: Tue, 10 Apr 2018 13:32:38 -0700 Subject: [PATCH 103/791] Update declaration order in staging ops Buffer class according to C++ style guide PiperOrigin-RevId: 192336966 --- tensorflow/core/kernels/stage_op.cc | 83 +++++++++++++---------------- 1 file changed, 38 insertions(+), 45 deletions(-) diff --git a/tensorflow/core/kernels/stage_op.cc b/tensorflow/core/kernels/stage_op.cc index 03fc4467a1..73a02a34cf 100644 --- a/tensorflow/core/kernels/stage_op.cc +++ b/tensorflow/core/kernels/stage_op.cc @@ -32,53 +32,8 @@ namespace { class Buffer : public ResourceBase { public: - // public types using Tuple = std::vector; - private: - // private variables - std::size_t capacity_; - std::size_t memory_limit_; - std::size_t current_bytes_; - std::mutex mu_; - std::condition_variable non_empty_cond_var_; - std::condition_variable full_cond_var_; - std::deque buf_; - - private: - // private methods - - // If the buffer is configured for bounded capacity, notify - // waiting inserters that space is now available - void notify_inserters_if_bounded(std::unique_lock* lock) { - if (IsBounded()) { - lock->unlock(); - // Notify all inserters. The removal of an element - // may make memory available for many inserters - // to insert new elements - full_cond_var_.notify_all(); - } - } - - // Are there a limit number of elements or a memory limit - // configued on this buffer? - bool IsBounded() const { return capacity_ > 0 || memory_limit_ > 0; } - - bool IsCapacityFull() const { return buf_.size() >= capacity_; } - - bool WouldExceedMemoryLimit(std::size_t bytes) const { - return bytes + current_bytes_ > memory_limit_; - } - - std::size_t GetTupleBytes(const Tuple& tuple) { - return std::accumulate(tuple.begin(), tuple.end(), 0, - [](const std::size_t& lhs, const Tensor& rhs) { - return lhs + rhs.TotalBytes(); - }); - } - - public: - // public methods explicit Buffer(std::size_t capacity, std::size_t memory_limit) : capacity_(capacity), memory_limit_(memory_limit), current_bytes_(0) {} @@ -181,6 +136,44 @@ class Buffer : public ResourceBase { std::unique_lock lock(mu_); return strings::StrCat("Staging size: ", buf_.size()); } + + private: + // If the buffer is configured for bounded capacity, notify + // waiting inserters that space is now available + void notify_inserters_if_bounded(std::unique_lock* lock) { + if (IsBounded()) { + lock->unlock(); + // Notify all inserters. The removal of an element + // may make memory available for many inserters + // to insert new elements + full_cond_var_.notify_all(); + } + } + + // Are there a limit number of elements or a memory limit + // configued on this buffer? + bool IsBounded() const { return capacity_ > 0 || memory_limit_ > 0; } + + bool IsCapacityFull() const { return buf_.size() >= capacity_; } + + bool WouldExceedMemoryLimit(std::size_t bytes) const { + return bytes + current_bytes_ > memory_limit_; + } + + std::size_t GetTupleBytes(const Tuple& tuple) { + return std::accumulate(tuple.begin(), tuple.end(), 0, + [](const std::size_t& lhs, const Tensor& rhs) { + return lhs + rhs.TotalBytes(); + }); + } + + std::size_t capacity_; + std::size_t memory_limit_; + std::size_t current_bytes_; + std::mutex mu_; + std::condition_variable non_empty_cond_var_; + std::condition_variable full_cond_var_; + std::deque buf_; }; Status GetBuffer(OpKernelContext* ctx, const NodeDef& ndef, Buffer** buf) { -- GitLab From 693b339ab2f062ec5bbb29f976c5d1fd94fbffa5 Mon Sep 17 00:00:00 2001 From: Francois Chollet Date: Tue, 10 Apr 2018 13:49:37 -0700 Subject: [PATCH 104/791] Refactor layers: - tf.layers layers now subclasses tf.keras.layers layers. - tf.keras.layers is now agnostic to variable scopes and global collections (future-proof). It also uses ResourceVariable everywhere by default. - As a result tf.keras.layers is in general lower-complexity, with fewer hacks and workarounds. However some of current code is temporary (variable creation should be moved to Checkpointable, arguably, and there are some dependency issues that will require later refactors). - The legacy tf.layers layers behavior is kept, with references to variable scopes and global collections injected in the subclassed tf.layers.base.Layer class (the content of tf.layers.base.Layer is the complexity differential between the old implementation and the new one). Note: this refactor does slightly change the behavior of tf.layers.base.Layer, by disabling extreme edge-case behavior that either has long been invalid, or is dangerous and should most definitely be disabled. This will not affect any users since such behaviors only existed in the base Layer unit tests. The behaviors disabled are: - Option to create reusable variables in `call` (already invalid for some time). - Option to use a variable scope to create layer variables outside of the layer while not having the layer track such variables locally. PiperOrigin-RevId: 192339798 --- .../cudnn_rnn/python/ops/cudnn_rnn_ops.py | 18 +- .../eager/python/checkpointable_utils_test.py | 2 +- tensorflow/contrib/eager/python/network.py | 5 +- .../contrib/eager/python/network_test.py | 32 - .../optimizer_v2/checkpointable_utils_test.py | 2 +- tensorflow/python/BUILD | 9 +- tensorflow/python/__init__.py | 10 + .../python/feature_column/feature_column.py | 35 +- tensorflow/python/keras/BUILD | 138 +- .../python/keras/_impl/keras/activations.py | 8 - .../python/keras/_impl/keras/backend.py | 24 +- .../keras/_impl/keras/engine/base_layer.py | 1736 +++++++++++++++-- .../keras/_impl/keras/engine/input_layer.py | 5 +- .../keras/_impl/keras/engine/network.py | 46 +- .../keras/_impl/keras/engine/saving_test.py | 2 +- .../keras/_impl/keras/engine/sequential.py | 2 +- .../_impl/keras/engine/sequential_test.py | 1 + .../keras/_impl/keras/engine/topology_test.py | 61 +- .../keras/_impl/keras/engine/training.py | 19 +- .../python/keras/_impl/keras/initializers.py | 2 + .../keras/_impl/keras/integration_test.py | 139 +- .../keras/_impl/keras/layers/convolutional.py | 937 +++++++-- .../python/keras/_impl/keras/layers/core.py | 166 +- .../keras/_impl/keras/layers/core_test.py | 1 - .../keras/_impl/keras/layers/embeddings.py | 6 +- .../keras/_impl/keras/layers/normalization.py | 653 ++++++- .../_impl/keras/layers/normalization_test.py | 20 + .../keras/_impl/keras/layers/pooling.py | 411 +++- .../_impl/keras/layers/recurrent_test.py | 4 +- .../keras/_impl/keras/layers/wrappers.py | 4 +- .../_impl/keras/model_subclassing_test.py | 35 +- .../keras/_impl/keras/utils/conv_utils.py | 143 +- .../keras/_impl/keras/utils/tf_utils.py | 74 + tensorflow/python/layers/base.py | 1443 ++------------ tensorflow/python/layers/base_test.py | 94 +- tensorflow/python/layers/convolutional.py | 702 +------ tensorflow/python/layers/core.py | 142 +- tensorflow/python/layers/normalization.py | 516 +---- tensorflow/python/layers/pooling.py | 258 +-- tensorflow/python/layers/utils_test.py | 29 - tensorflow/python/ops/nn.py | 2 - .../api/golden/tensorflow.keras.-model.pbtxt | 11 +- .../golden/tensorflow.keras.-sequential.pbtxt | 11 +- .../tensorflow.keras.layers.-activation.pbtxt | 13 +- ...eras.layers.-activity-regularization.pbtxt | 13 +- .../golden/tensorflow.keras.layers.-add.pbtxt | 13 +- ...nsorflow.keras.layers.-alpha-dropout.pbtxt | 13 +- ...low.keras.layers.-average-pooling1-d.pbtxt | 18 +- ...low.keras.layers.-average-pooling2-d.pbtxt | 16 +- ...low.keras.layers.-average-pooling3-d.pbtxt | 16 +- .../tensorflow.keras.layers.-average.pbtxt | 13 +- ...tensorflow.keras.layers.-avg-pool1-d.pbtxt | 18 +- ...tensorflow.keras.layers.-avg-pool2-d.pbtxt | 16 +- ...tensorflow.keras.layers.-avg-pool3-d.pbtxt | 16 +- ...ow.keras.layers.-batch-normalization.pbtxt | 16 +- ...nsorflow.keras.layers.-bidirectional.pbtxt | 13 +- ...tensorflow.keras.layers.-concatenate.pbtxt | 13 +- ...orflow.keras.layers.-conv-l-s-t-m2-d.pbtxt | 13 +- .../tensorflow.keras.layers.-conv1-d.pbtxt | 18 +- ...flow.keras.layers.-conv2-d-transpose.pbtxt | 18 +- .../tensorflow.keras.layers.-conv2-d.pbtxt | 16 +- ...flow.keras.layers.-conv3-d-transpose.pbtxt | 18 +- .../tensorflow.keras.layers.-conv3-d.pbtxt | 16 +- ...sorflow.keras.layers.-convolution1-d.pbtxt | 18 +- ...ras.layers.-convolution2-d-transpose.pbtxt | 18 +- ...sorflow.keras.layers.-convolution2-d.pbtxt | 16 +- ...ras.layers.-convolution3-d-transpose.pbtxt | 18 +- ...sorflow.keras.layers.-convolution3-d.pbtxt | 16 +- ...tensorflow.keras.layers.-cropping1-d.pbtxt | 13 +- ...tensorflow.keras.layers.-cropping2-d.pbtxt | 13 +- ...tensorflow.keras.layers.-cropping3-d.pbtxt | 13 +- .../tensorflow.keras.layers.-dense.pbtxt | 14 +- ...flow.keras.layers.-depthwise-conv2-d.pbtxt | 16 +- .../golden/tensorflow.keras.layers.-dot.pbtxt | 13 +- .../tensorflow.keras.layers.-dropout.pbtxt | 14 +- .../tensorflow.keras.layers.-e-l-u.pbtxt | 13 +- .../tensorflow.keras.layers.-embedding.pbtxt | 13 +- .../tensorflow.keras.layers.-flatten.pbtxt | 14 +- .../tensorflow.keras.layers.-g-r-u-cell.pbtxt | 13 +- .../tensorflow.keras.layers.-g-r-u.pbtxt | 13 +- ...rflow.keras.layers.-gaussian-dropout.pbtxt | 13 +- ...sorflow.keras.layers.-gaussian-noise.pbtxt | 13 +- ...as.layers.-global-average-pooling1-d.pbtxt | 15 +- ...as.layers.-global-average-pooling2-d.pbtxt | 15 +- ...as.layers.-global-average-pooling3-d.pbtxt | 15 +- ...low.keras.layers.-global-avg-pool1-d.pbtxt | 15 +- ...low.keras.layers.-global-avg-pool2-d.pbtxt | 15 +- ...low.keras.layers.-global-avg-pool3-d.pbtxt | 15 +- ...low.keras.layers.-global-max-pool1-d.pbtxt | 15 +- ...low.keras.layers.-global-max-pool2-d.pbtxt | 15 +- ...low.keras.layers.-global-max-pool3-d.pbtxt | 15 +- ....keras.layers.-global-max-pooling1-d.pbtxt | 15 +- ....keras.layers.-global-max-pooling2-d.pbtxt | 15 +- ....keras.layers.-global-max-pooling3-d.pbtxt | 15 +- ...tensorflow.keras.layers.-input-layer.pbtxt | 13 +- .../tensorflow.keras.layers.-input-spec.pbtxt | 2 +- ...ensorflow.keras.layers.-l-s-t-m-cell.pbtxt | 13 +- .../tensorflow.keras.layers.-l-s-t-m.pbtxt | 13 +- .../tensorflow.keras.layers.-lambda.pbtxt | 13 +- .../tensorflow.keras.layers.-layer.pbtxt | 15 +- ...ensorflow.keras.layers.-leaky-re-l-u.pbtxt | 13 +- ...w.keras.layers.-locally-connected1-d.pbtxt | 13 +- ...w.keras.layers.-locally-connected2-d.pbtxt | 13 +- .../tensorflow.keras.layers.-masking.pbtxt | 13 +- ...tensorflow.keras.layers.-max-pool1-d.pbtxt | 18 +- ...tensorflow.keras.layers.-max-pool2-d.pbtxt | 16 +- ...tensorflow.keras.layers.-max-pool3-d.pbtxt | 16 +- ...sorflow.keras.layers.-max-pooling1-d.pbtxt | 18 +- ...sorflow.keras.layers.-max-pooling2-d.pbtxt | 16 +- ...sorflow.keras.layers.-max-pooling3-d.pbtxt | 16 +- .../tensorflow.keras.layers.-maximum.pbtxt | 13 +- .../tensorflow.keras.layers.-multiply.pbtxt | 13 +- .../tensorflow.keras.layers.-p-re-l-u.pbtxt | 13 +- .../tensorflow.keras.layers.-permute.pbtxt | 13 +- .../tensorflow.keras.layers.-r-n-n.pbtxt | 13 +- ...nsorflow.keras.layers.-repeat-vector.pbtxt | 13 +- .../tensorflow.keras.layers.-reshape.pbtxt | 13 +- ...flow.keras.layers.-separable-conv1-d.pbtxt | 18 +- ...flow.keras.layers.-separable-conv2-d.pbtxt | 18 +- ...ras.layers.-separable-convolution1-d.pbtxt | 18 +- ...ras.layers.-separable-convolution2-d.pbtxt | 18 +- ...flow.keras.layers.-simple-r-n-n-cell.pbtxt | 13 +- ...ensorflow.keras.layers.-simple-r-n-n.pbtxt | 13 +- .../tensorflow.keras.layers.-softmax.pbtxt | 13 +- ...low.keras.layers.-spatial-dropout1-d.pbtxt | 14 +- ...low.keras.layers.-spatial-dropout2-d.pbtxt | 14 +- ...low.keras.layers.-spatial-dropout3-d.pbtxt | 14 +- ...ow.keras.layers.-stacked-r-n-n-cells.pbtxt | 13 +- ...low.keras.layers.-thresholded-re-l-u.pbtxt | 13 +- ...rflow.keras.layers.-time-distributed.pbtxt | 13 +- ...sorflow.keras.layers.-up-sampling1-d.pbtxt | 13 +- ...sorflow.keras.layers.-up-sampling2-d.pbtxt | 13 +- ...sorflow.keras.layers.-up-sampling3-d.pbtxt | 13 +- .../tensorflow.keras.layers.-wrapper.pbtxt | 13 +- ...orflow.keras.layers.-zero-padding1-d.pbtxt | 13 +- ...orflow.keras.layers.-zero-padding2-d.pbtxt | 13 +- ...orflow.keras.layers.-zero-padding3-d.pbtxt | 13 +- .../tensorflow.keras.models.-model.pbtxt | 11 +- .../tensorflow.keras.models.-sequential.pbtxt | 11 +- ...ensorflow.layers.-average-pooling1-d.pbtxt | 46 +- ...ensorflow.layers.-average-pooling2-d.pbtxt | 46 +- ...ensorflow.layers.-average-pooling3-d.pbtxt | 46 +- ...nsorflow.layers.-batch-normalization.pbtxt | 44 +- .../golden/tensorflow.layers.-conv1-d.pbtxt | 46 +- ...tensorflow.layers.-conv2-d-transpose.pbtxt | 48 +- .../golden/tensorflow.layers.-conv2-d.pbtxt | 46 +- ...tensorflow.layers.-conv3-d-transpose.pbtxt | 48 +- .../golden/tensorflow.layers.-conv3-d.pbtxt | 46 +- .../api/golden/tensorflow.layers.-dense.pbtxt | 44 +- .../golden/tensorflow.layers.-dropout.pbtxt | 44 +- .../golden/tensorflow.layers.-flatten.pbtxt | 44 +- .../tensorflow.layers.-input-spec.pbtxt | 2 +- .../api/golden/tensorflow.layers.-layer.pbtxt | 45 +- .../tensorflow.layers.-max-pooling1-d.pbtxt | 46 +- .../tensorflow.layers.-max-pooling2-d.pbtxt | 46 +- .../tensorflow.layers.-max-pooling3-d.pbtxt | 46 +- ...tensorflow.layers.-separable-conv1-d.pbtxt | 48 +- ...tensorflow.layers.-separable-conv2-d.pbtxt | 48 +- ...flow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt | 43 +- ...orflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt | 43 +- ...nsorflow.nn.rnn_cell.-device-wrapper.pbtxt | 43 +- ...sorflow.nn.rnn_cell.-dropout-wrapper.pbtxt | 43 +- .../tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt | 43 +- ...tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt | 43 +- ...orflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt | 43 +- .../tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt | 45 +- ...orflow.nn.rnn_cell.-residual-wrapper.pbtxt | 43 +- 167 files changed, 5493 insertions(+), 5060 deletions(-) create mode 100644 tensorflow/python/keras/_impl/keras/utils/tf_utils.py diff --git a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py index 588a5e705d..1dd490b386 100644 --- a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py +++ b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py @@ -23,7 +23,7 @@ from tensorflow.python.framework import common_shapes from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import random_seed -from tensorflow.python.layers import base as base_layer +from tensorflow.python.keras._impl.keras.engine import base_layer from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_cudnn_rnn_ops from tensorflow.python.ops import init_ops @@ -520,10 +520,7 @@ class CudnnLSTMSaveable(CudnnOpaqueParamsSaveable): _rnn_mode = CUDNN_LSTM _num_params_per_layer = CUDNN_LSTM_PARAMS_PER_LAYER - # pylint:disable=protected-access - _rnn_cell_name = base_layer._to_snake_case(CudnnCompatibleLSTMCell.__name__) - - # pylint:enable=protected-access + _rnn_cell_name = base_layer.to_snake_case(CudnnCompatibleLSTMCell.__name__) def _cudnn_to_tf_gate_params(self, *cu_gate_order): i_g, f_g, c_g, o_g = cu_gate_order @@ -644,10 +641,7 @@ class CudnnGRUSaveable(CudnnOpaqueParamsSaveable): _rnn_mode = CUDNN_GRU _num_params_per_layer = CUDNN_GRU_PARAMS_PER_LAYER - # pylint:disable=protected-access - _rnn_cell_name = base_layer._to_snake_case(CudnnCompatibleGRUCell.__name__) - - # pylint:enable=protected-access + _rnn_cell_name = base_layer.to_snake_case(CudnnCompatibleGRUCell.__name__) def _cudnn_to_tf_weights(self, *cu_weights): r"""Stitching cudnn canonical weights to generate tf canonical weights.""" @@ -726,11 +720,7 @@ class CudnnGRUSaveable(CudnnOpaqueParamsSaveable): class CudnnRNNSimpleSaveable(CudnnLSTMSaveable): """SaveableObject implementation handling Cudnn RNN Tanh opaque params.""" - # pylint:disable=protected-access - _rnn_cell_name = base_layer._to_snake_case( - rnn_cell_impl.BasicRNNCell.__name__) - - # pylint:enable=protected-access + _rnn_cell_name = base_layer.to_snake_case(rnn_cell_impl.BasicRNNCell.__name__) def _cudnn_to_tf_weights(self, *cu_weights): r"""Stitching cudnn canonical weights to generate tf canonical weights.""" diff --git a/tensorflow/contrib/eager/python/checkpointable_utils_test.py b/tensorflow/contrib/eager/python/checkpointable_utils_test.py index 688befa772..36670aa210 100644 --- a/tensorflow/contrib/eager/python/checkpointable_utils_test.py +++ b/tensorflow/contrib/eager/python/checkpointable_utils_test.py @@ -33,7 +33,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.keras._impl.keras.engine import sequential from tensorflow.python.keras._impl.keras.engine import training -from tensorflow.python.layers import core +from tensorflow.python.keras._impl.keras.layers import core from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import init_ops diff --git a/tensorflow/contrib/eager/python/network.py b/tensorflow/contrib/eager/python/network.py index e55a9276ab..2f8721324f 100644 --- a/tensorflow/contrib/eager/python/network.py +++ b/tensorflow/contrib/eager/python/network.py @@ -25,6 +25,7 @@ import weakref from tensorflow.python.eager import context from tensorflow.python.estimator import util as estimator_util from tensorflow.python.framework import ops +from tensorflow.python.keras._impl.keras.engine import base_layer as keras_base_layer from tensorflow.python.layers import base from tensorflow.python.ops import variable_scope from tensorflow.python.training import checkpoint_utils @@ -176,7 +177,7 @@ class Network(base.Layer): avoid_names = parent_network._owned_layers name_uid_map = parent_network._sub_layer_name_uids else: - name_uid_map = base._get_default_graph_uid_map() + name_uid_map = keras_base_layer.get_default_graph_uid_map() # Figure out which names we have to avoid based on which variable scope # we're nested in. strip_name = self._default_parent_variable_scope.name @@ -326,6 +327,8 @@ class Network(base.Layer): raise TypeError( "Network.track_layer() passed type %s, not a tf.layers.Layer" % (type(layer),)) + # Always use `ResourceVariable` with legacy layers. + layer._use_resource_variables = True if isinstance(layer, Network): layer._finalize_name(parent_network=self) else: diff --git a/tensorflow/contrib/eager/python/network_test.py b/tensorflow/contrib/eager/python/network_test.py index 3329fc6c51..f43376d5d7 100644 --- a/tensorflow/contrib/eager/python/network_test.py +++ b/tensorflow/contrib/eager/python/network_test.py @@ -20,12 +20,10 @@ import gc from tensorflow.contrib.eager.python import network from tensorflow.contrib.layers.python.layers import regularizers -from tensorflow.python.eager import context from tensorflow.python.eager import function from tensorflow.python.eager import test from tensorflow.python.framework import constant_op from tensorflow.python.framework import errors_impl -from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.layers import core from tensorflow.python.ops import math_ops @@ -469,36 +467,6 @@ class NetworkTest(test.TestCase): self.assertIsInstance(net.trainable_weights[0], resource_variable_ops.ResourceVariable) - def testGraphOpNames(self): - """Network operation names should match variable naming.""" - - def _check_op_prefixes(expected_prefix, checked_ops): - for operation in ops.get_default_graph().get_operations(): - if operation.name == "ignore": - continue - if operation.name in checked_ops: - continue - checked_ops.add(operation.name) - self.assertStartsWith(expected_start=expected_prefix, - actual=operation.name) - self.assertNotIn("my_network", operation.name[len(expected_prefix):]) - self.assertNotIn("dense", operation.name[len(expected_prefix):]) - - with context.graph_mode(): - net = MyNetwork() - zero = constant_op.constant([[0.]], name="ignore") - net(zero) - checked_ops = set() - _check_op_prefixes(expected_prefix="my_network/dense/", - checked_ops=checked_ops) - net.net2 = net.track_layer(MyNetwork()) - net.net2(zero) - _check_op_prefixes(expected_prefix="my_network/my_network/dense/", - checked_ops=checked_ops) - MyNetwork()(zero) - _check_op_prefixes(expected_prefix="my_network_1/dense/", - checked_ops=checked_ops) - @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) def testVariableRegularizers(self): net = RegularizedNetwork() diff --git a/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py b/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py index abcffeb618..54bc23cdef 100644 --- a/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py +++ b/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py @@ -36,7 +36,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.keras._impl.keras.engine import training -from tensorflow.python.layers import core +from tensorflow.python.keras._impl.keras.layers import core from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import state_ops diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index a22b9f40b1..7b548d2c70 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -2273,7 +2273,6 @@ py_library( ":clip_ops", ":framework_for_generated_wrappers", ":init_ops", - ":layers_base", ":math_ops", ":nn_ops", ":partitioned_variables", @@ -2949,11 +2948,13 @@ py_library( ":util", ":variable_scope", ":variables", + "//third_party/py/numpy", + "@six_archive//:six", "//tensorflow/python/eager:backprop", "//tensorflow/python/eager:context", "//tensorflow/python/ops/losses", - "//third_party/py/numpy", - "@six_archive//:six", + # `layers` dependency only exists due to the use of a small utility. + "//tensorflow/python/keras:layers", ], ) @@ -4310,6 +4311,7 @@ py_library( ":variables", "//tensorflow/python/eager:context", "//tensorflow/python/estimator:util", + "//tensorflow/python/keras:engine", "//third_party/py/numpy", ], ) @@ -4346,6 +4348,7 @@ py_library( ":variables", "//tensorflow/python/eager:context", "//tensorflow/python/estimator:util", + "//tensorflow/python/keras:layers", "//third_party/py/numpy", "@six_archive//:six", ], diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py index ab1d01a835..da836aca6f 100644 --- a/tensorflow/python/__init__.py +++ b/tensorflow/python/__init__.py @@ -149,6 +149,16 @@ from tensorflow.python.ops import tensor_array_ops from tensorflow.python.eager.context import executing_eagerly from tensorflow.python.framework.ops import enable_eager_execution +# Necessary for the symbols in this module to be taken into account by +# the namespace management system (API decorators). +from tensorflow.python.ops import rnn +from tensorflow.python.ops import rnn_cell + +# Required due to `rnn` and `rnn_cell` not being imported in `nn` directly +# (due to a circular dependency issue: rnn depends on layers). +nn.dynamic_rnn = rnn.dynamic_rnn +nn.rnn_cell = rnn_cell + # Symbols whitelisted for export without documentation. # TODO(cwhipkey): review these and move to contrib, expose through # documentation, or remove. diff --git a/tensorflow/python/feature_column/feature_column.py b/tensorflow/python/feature_column/feature_column.py index 3a315e5c2e..7a104fa4ac 100644 --- a/tensorflow/python/feature_column/feature_column.py +++ b/tensorflow/python/feature_column/feature_column.py @@ -581,24 +581,25 @@ class _LinearModel(training.Model): **kwargs) def call(self, features): - for column in self._feature_columns: - if not isinstance(column, (_DenseColumn, _CategoricalColumn)): - raise ValueError( - 'Items of feature_columns must be either a ' - '_DenseColumn or _CategoricalColumn. Given: {}'.format(column)) - weighted_sums = [] - ordered_columns = [] - builder = _LazyBuilder(features) - for layer in sorted(self._column_layers.values(), key=lambda x: x.name): - ordered_columns.append(layer._feature_column) # pylint: disable=protected-access - weighted_sum = layer(builder) - weighted_sums.append(weighted_sum) + with variable_scope.variable_scope(self.name): + for column in self._feature_columns: + if not isinstance(column, (_DenseColumn, _CategoricalColumn)): + raise ValueError( + 'Items of feature_columns must be either a ' + '_DenseColumn or _CategoricalColumn. Given: {}'.format(column)) + weighted_sums = [] + ordered_columns = [] + builder = _LazyBuilder(features) + for layer in sorted(self._column_layers.values(), key=lambda x: x.name): + ordered_columns.append(layer._feature_column) # pylint: disable=protected-access + weighted_sum = layer(builder) + weighted_sums.append(weighted_sum) - _verify_static_batch_size_equality(weighted_sums, ordered_columns) - predictions_no_bias = math_ops.add_n( - weighted_sums, name='weighted_sum_no_bias') - predictions = nn_ops.bias_add( - predictions_no_bias, self._bias_layer(builder), name='weighted_sum') # pylint: disable=not-callable + _verify_static_batch_size_equality(weighted_sums, ordered_columns) + predictions_no_bias = math_ops.add_n( + weighted_sums, name='weighted_sum_no_bias') + predictions = nn_ops.bias_add( + predictions_no_bias, self._bias_layer(builder), name='weighted_sum') # pylint: disable=not-callable return predictions def _add_layers(self, layers): diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD index f6e1d0eec3..da5bc3e6f1 100755 --- a/tensorflow/python/keras/BUILD +++ b/tensorflow/python/keras/BUILD @@ -20,7 +20,6 @@ py_library( srcs = [ "__init__.py", "_impl/keras/__init__.py", - "_impl/keras/activations.py", "_impl/keras/applications/__init__.py", "_impl/keras/applications/densenet.py", "_impl/keras/applications/imagenet_utils.py", @@ -32,9 +31,6 @@ py_library( "_impl/keras/applications/vgg16.py", "_impl/keras/applications/vgg19.py", "_impl/keras/applications/xception.py", - "_impl/keras/backend.py", - "_impl/keras/callbacks.py", - "_impl/keras/constraints.py", "_impl/keras/datasets/__init__.py", "_impl/keras/datasets/boston_housing.py", "_impl/keras/datasets/cifar.py", @@ -44,49 +40,13 @@ py_library( "_impl/keras/datasets/imdb.py", "_impl/keras/datasets/mnist.py", "_impl/keras/datasets/reuters.py", - "_impl/keras/engine/__init__.py", - "_impl/keras/engine/base_layer.py", - "_impl/keras/engine/input_layer.py", - "_impl/keras/engine/network.py", - "_impl/keras/engine/saving.py", - "_impl/keras/engine/sequential.py", - "_impl/keras/engine/training.py", - "_impl/keras/engine/training_arrays.py", - "_impl/keras/engine/training_eager.py", - "_impl/keras/engine/training_generator.py", - "_impl/keras/engine/training_utils.py", "_impl/keras/estimator.py", - "_impl/keras/initializers.py", - "_impl/keras/layers/__init__.py", - "_impl/keras/layers/advanced_activations.py", - "_impl/keras/layers/convolutional.py", - "_impl/keras/layers/convolutional_recurrent.py", - "_impl/keras/layers/core.py", - "_impl/keras/layers/embeddings.py", - "_impl/keras/layers/local.py", - "_impl/keras/layers/merge.py", - "_impl/keras/layers/noise.py", - "_impl/keras/layers/normalization.py", - "_impl/keras/layers/pooling.py", - "_impl/keras/layers/recurrent.py", - "_impl/keras/layers/serialization.py", - "_impl/keras/layers/wrappers.py", - "_impl/keras/losses.py", - "_impl/keras/metrics.py", - "_impl/keras/models.py", - "_impl/keras/optimizers.py", "_impl/keras/preprocessing/__init__.py", "_impl/keras/preprocessing/image.py", "_impl/keras/preprocessing/sequence.py", "_impl/keras/preprocessing/text.py", - "_impl/keras/regularizers.py", "_impl/keras/testing_utils.py", "_impl/keras/utils/__init__.py", - "_impl/keras/utils/conv_utils.py", - "_impl/keras/utils/data_utils.py", - "_impl/keras/utils/generic_utils.py", - "_impl/keras/utils/io_utils.py", - "_impl/keras/utils/layer_utils.py", "_impl/keras/utils/multi_gpu_utils.py", "_impl/keras/utils/np_utils.py", "_impl/keras/utils/vis_utils.py", @@ -136,7 +96,21 @@ py_library( ":empty_condition": [], "//conditions:default": [], }) + [ - "@six_archive//:six", + ":backend", + ":engine", + ":layers", + "//tensorflow/python/estimator", + "//tensorflow/python/estimator:model_fn", + "//tensorflow/python/saved_model", + "//tensorflow/python:training", + ], +) + +py_library( + name = "backend", + srcs = ["_impl/keras/backend.py"], + srcs_version = "PY2AND3", + deps = [ "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", "//tensorflow/python:check_ops", @@ -152,8 +126,6 @@ py_library( "//tensorflow/python:gradients", "//tensorflow/python:image_ops", "//tensorflow/python:init_ops", - "//tensorflow/python:layers", - "//tensorflow/python:layers_base", "//tensorflow/python:logging_ops", "//tensorflow/python:math_ops", "//tensorflow/python:metrics", @@ -168,13 +140,83 @@ py_library( "//tensorflow/python:tensor_array_grad", "//tensorflow/python:tensor_array_ops", "//tensorflow/python:tensor_shape", - "//tensorflow/python:training", "//tensorflow/python:util", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", - "//tensorflow/python/estimator", - "//tensorflow/python/estimator:model_fn", - "//tensorflow/python/saved_model", + ], +) + +py_library( + name = "engine", + srcs = [ + "_impl/keras/activations.py", + "_impl/keras/callbacks.py", + "_impl/keras/constraints.py", + "_impl/keras/engine/__init__.py", + "_impl/keras/engine/base_layer.py", + "_impl/keras/engine/input_layer.py", + "_impl/keras/engine/network.py", + "_impl/keras/engine/saving.py", + "_impl/keras/engine/sequential.py", + "_impl/keras/engine/training.py", + "_impl/keras/engine/training_arrays.py", + "_impl/keras/engine/training_eager.py", + "_impl/keras/engine/training_generator.py", + "_impl/keras/engine/training_utils.py", + "_impl/keras/initializers.py", + "_impl/keras/losses.py", + "_impl/keras/metrics.py", + "_impl/keras/models.py", + "_impl/keras/optimizers.py", + "_impl/keras/regularizers.py", + "_impl/keras/utils/data_utils.py", + "_impl/keras/utils/io_utils.py", + ], + srcs_version = "PY2AND3", + deps = [ + ":backend", + "@six_archive//:six", + ], +) + +py_library( + name = "layers", + srcs = [ + "_impl/keras/layers/__init__.py", + "_impl/keras/layers/advanced_activations.py", + "_impl/keras/layers/convolutional.py", + "_impl/keras/layers/convolutional_recurrent.py", + "_impl/keras/layers/core.py", + "_impl/keras/layers/embeddings.py", + "_impl/keras/layers/local.py", + "_impl/keras/layers/merge.py", + "_impl/keras/layers/noise.py", + "_impl/keras/layers/normalization.py", + "_impl/keras/layers/pooling.py", + "_impl/keras/layers/recurrent.py", + "_impl/keras/layers/serialization.py", + "_impl/keras/layers/wrappers.py", + "_impl/keras/utils/conv_utils.py", + "_impl/keras/utils/generic_utils.py", + "_impl/keras/utils/layer_utils.py", + "_impl/keras/utils/tf_utils.py", + ], + srcs_version = "PY2AND3", + deps = [ + ":engine", + "//tensorflow/python:array_ops", + "//tensorflow/python:dtypes", + "//tensorflow/python:framework_ops", + "//tensorflow/python:logging_ops", + "//tensorflow/python:math_ops", + "//tensorflow/python:nn", + "//tensorflow/python:random_ops", + "//tensorflow/python:sparse_ops", + "//tensorflow/python:sparse_tensor", + "//tensorflow/python:standard_ops", + "//tensorflow/python:tensor_array_ops", + "//tensorflow/python:tensor_shape", + "//tensorflow/python:util", ], ) @@ -605,7 +647,7 @@ py_test( py_test( name = "data_utils_test", - size = "medium", + size = "large", srcs = ["_impl/keras/utils/data_utils_test.py"], srcs_version = "PY2AND3", tags = [ diff --git a/tensorflow/python/keras/_impl/keras/activations.py b/tensorflow/python/keras/_impl/keras/activations.py index b518898ad8..8def7ec493 100644 --- a/tensorflow/python/keras/_impl/keras/activations.py +++ b/tensorflow/python/keras/_impl/keras/activations.py @@ -22,10 +22,8 @@ import six from tensorflow.python.keras._impl.keras import backend as K from tensorflow.python.keras._impl.keras.utils.generic_utils import deserialize_keras_object -from tensorflow.python.layers.base import Layer from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn -from tensorflow.python.platform import tf_logging as logging from tensorflow.python.util.tf_export import tf_export @@ -136,12 +134,6 @@ def get(identifier): identifier = str(identifier) return deserialize(identifier) elif callable(identifier): - if isinstance(identifier, Layer): - logging.warning( - 'Do not pass a layer instance (such as {identifier}) as the ' - 'activation argument of another layer. Instead, advanced ' - 'activation layers should be used just like any other ' - 'layer in a model.'.format(identifier=identifier.__class__.__name__)) return identifier else: raise ValueError('Could not interpret ' diff --git a/tensorflow/python/keras/_impl/keras/backend.py b/tensorflow/python/keras/_impl/keras/backend.py index 3aac6a9065..096db8db32 100644 --- a/tensorflow/python/keras/_impl/keras/backend.py +++ b/tensorflow/python/keras/_impl/keras/backend.py @@ -24,6 +24,7 @@ from __future__ import print_function import collections import json import os +import weakref import numpy as np @@ -35,7 +36,6 @@ from tensorflow.python.framework import dtypes as dtypes_module from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor from tensorflow.python.framework import tensor_util -from tensorflow.python.layers import base as tf_base_layers from tensorflow.python.ops import array_ops from tensorflow.python.ops import clip_ops from tensorflow.python.ops import control_flow_ops @@ -55,7 +55,7 @@ from tensorflow.python.ops import state_ops from tensorflow.python.ops import tensor_array_grad # pylint: disable=unused-import from tensorflow.python.ops import tensor_array_ops from tensorflow.python.ops import variables as variables_module -from tensorflow.python.training import moving_averages + from tensorflow.python.util import tf_contextlib from tensorflow.python.util import tf_inspect from tensorflow.python.util.tf_export import tf_export @@ -263,6 +263,12 @@ def set_image_data_format(data_format): _IMAGE_DATA_FORMAT = str(data_format) +# A global dictionary mapping graph objects to an index of counters used +# for various layer names in each graph. +# Allows to give unique autogenerated names to layers, in a graph-specific way. +PER_GRAPH_LAYER_NAME_UIDS = weakref.WeakKeyDictionary() + + @tf_export('keras.backend.get_uid') def get_uid(prefix=''): """Associates a string prefix with an integer counter in a TensorFlow graph. @@ -283,17 +289,16 @@ def get_uid(prefix=''): ``` """ graph = ops.get_default_graph() - if graph not in tf_base_layers.PER_GRAPH_LAYER_NAME_UIDS: - tf_base_layers.PER_GRAPH_LAYER_NAME_UIDS[graph] = collections.defaultdict( - int) - layer_name_uids = tf_base_layers.PER_GRAPH_LAYER_NAME_UIDS[graph] + if graph not in PER_GRAPH_LAYER_NAME_UIDS: + PER_GRAPH_LAYER_NAME_UIDS[graph] = collections.defaultdict(int) + layer_name_uids = PER_GRAPH_LAYER_NAME_UIDS[graph] layer_name_uids[prefix] += 1 return layer_name_uids[prefix] @tf_export('keras.backend.reset_uids') def reset_uids(): - per_graph_layer_name_uids = tf_base_layers.PER_GRAPH_LAYER_NAME_UIDS + per_graph_layer_name_uids = PER_GRAPH_LAYER_NAME_UIDS keys = list(per_graph_layer_name_uids.keys()) for key in keys: del per_graph_layer_name_uids[key] @@ -1276,6 +1281,11 @@ def moving_average_update(x, value, momentum): Returns: An Operation to update the variable. """ + # `training` is higher-up than the Keras backend in the abstraction hierarchy. + # In particular, `training` depends on layers, and thus on Keras. + # moving_averages, being low-level ops, should not be part of the training + # module. + from tensorflow.python.training import moving_averages # pylint: disable=g-import-not-at-top return moving_averages.assign_moving_average( x, value, momentum, zero_debias=True) diff --git a/tensorflow/python/keras/_impl/keras/engine/base_layer.py b/tensorflow/python/keras/_impl/keras/engine/base_layer.py index 755607aafb..3b3af7d092 100644 --- a/tensorflow/python/keras/_impl/keras/engine/base_layer.py +++ b/tensorflow/python/keras/_impl/keras/engine/base_layer.py @@ -13,143 +13,145 @@ # limitations under the License. # ============================================================================== # pylint: disable=protected-access -"""Base layer code (`Layer`). -""" +"""Contains the base Layer class, from which all layers inherit.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function +import collections import inspect # Necessary supplement to tf_inspect to deal with variadic args. +import re +import numpy as np from six.moves import zip # pylint: disable=redefined-builtin from tensorflow.python.eager import context +from tensorflow.python.estimator import util as estimator_util +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape -from tensorflow.python.keras._impl.keras import backend as K +from tensorflow.python.keras._impl.keras import backend from tensorflow.python.keras._impl.keras import constraints from tensorflow.python.keras._impl.keras import initializers from tensorflow.python.keras._impl.keras import regularizers from tensorflow.python.keras._impl.keras.utils import generic_utils -from tensorflow.python.layers import base as tf_base_layers -from tensorflow.python.platform import tf_logging as logging +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import variable_scope as vs +from tensorflow.python.ops import variables as tf_variables +from tensorflow.python.training import checkpointable +from tensorflow.python.util import nest from tensorflow.python.util import tf_decorator from tensorflow.python.util import tf_inspect from tensorflow.python.util.tf_export import tf_export -# pylint: disable=invalid-name -InputSpec = tf_base_layers.InputSpec -Node = tf_base_layers.Node -TFBaseLayer = tf_base_layers.Layer -# pylint: enable=invalid-name +@tf_export('keras.layers.Layer') +class Layer(checkpointable.CheckpointableBase): + """Base layer class. + This is the class from which all layers inherit. -@tf_export('keras.layers.Layer') -class Layer(tf_base_layers.Layer): - """Abstract base layer class. - - # Properties - name: String, must be unique within a model. - input_spec: List of InputSpec class instances - each entry describes one required input: - - ndim - - dtype - A layer with `n` input tensors must have - an `input_spec` of length `n`. - trainable: Boolean, whether the layer weights - will be updated during training. - uses_learning_phase: Whether any operation - of the layer uses `K.in_training_phase()` - or `K.in_test_phase()`. - input_shape: Shape tuple. Provided for convenience, - but note that there may be cases in which this - attribute is ill-defined (e.g. a shared layer - with multiple input shapes), in which case - requesting `input_shape` will raise an Exception. - Prefer using `layer.get_input_shape_for(input_shape)`, - or `layer.get_input_shape_at(node_index)`. - output_shape: Shape tuple. See above. - inbound_nodes: List of nodes. - outbound_nodes: List of nodes. - input, output: Input/output tensor(s). Note that if the layer is used - more than once (shared layer), this is ill-defined - and will raise an exception. In such cases, use - `layer.get_input_at(node_index)`. - input_mask, output_mask: Same as above, for masks. - trainable_weights: List of variables. - non_trainable_weights: List of variables. - weights: The concatenation of the lists trainable_weights and - non_trainable_weights (in this order). - - # Methods - call(x, mask=None): Where the layer's logic lives. - __call__(x, mask=None): Wrapper around the layer logic (`call`). - If x is a Keras tensor: - - Connect current layer with last layer from tensor: - `self._add_inbound_node(last_layer)` - - Add layer to tensor history - If layer is not built: - - Build from inputs shape - get_weights() - set_weights(weights) - get_config() - count_params() - compute_output_shape(input_shape) - compute_mask(x, mask) - get_input_at(node_index) - get_output_at(node_index) - get_input_shape_at(node_index) - get_output_shape_at(node_index) - get_input_mask_at(node_index) - get_output_mask_at(node_index) - - # Class Methods - from_config(config) - - # Internal methods: - build(input_shape) - _add_inbound_node(layer, index=0) + A layer is a class implementing common neural networks operations, such + as convolution, batch norm, etc. These operations require managing weights, + losses, updates, and inter-layer connectivity. + + Users will just instantiate a layer and then treat it as a callable. + + We recommend that descendants of `Layer` implement the following methods: + * `__init__()`: Save configuration in member variables + * `build()`: Called once from `__call__`, when we know the shapes of inputs + and `dtype`. Should have the calls to `add_weight()`, and then + call the super's `build()` (which sets `self.built = True`, which is + nice in case the user wants to call `build()` manually before the + first `__call__`). + * `call()`: Called in `__call__` after making sure `build()` has been called + once. Should actually perform the logic of applying the layer to the + input tensors (which should be passed in as the first argument). + + Arguments: + trainable: Boolean, whether the layer's variables should be trainable. + name: String name of the layer. + dtype: Default dtype of the layer's weights (default of `None` means use the + type of the first input). + + Read-only properties: + name: The name of the layer (string). + dtype: Default dtype of the layer's weights (default of `None` means use the + type of the first input). + trainable_variables: List of trainable variables. + non_trainable_variables: List of non-trainable variables. + variables: List of all variables of this layer, trainable and + non-trainable. + updates: List of update ops of this layer. + losses: List of losses added by this layer. + trainable_weights: List of variables to be included in backprop. + non_trainable_weights: List of variables that should not be + included in backprop. + weights: The concatenation of the lists trainable_weights and + non_trainable_weights (in this order). + + Mutable properties: + trainable: Whether the layer should be trained (boolean). + input_spec: Optional (list of) `InputSpec` object(s) specifying the + constraints on inputs that can be accepted by the layer. """ - def __init__(self, **kwargs): + def __init__(self, trainable=True, name=None, dtype=None, **kwargs): # These properties should be set by the user via keyword arguments. # note that 'dtype', 'input_shape' and 'batch_input_shape' # are only applicable to input layers: do not pass these keywords # to non-input layers. allowed_kwargs = { - 'activity_regularizer', 'input_shape', 'batch_input_shape', 'batch_size', - 'dtype', - 'name', - 'trainable', 'weights', + 'activity_regularizer', } # Validate optional keyword arguments. for kwarg in kwargs: if kwarg not in allowed_kwargs: raise TypeError('Keyword argument not understood:', kwarg) - # Get layer name. - name = kwargs.get('name') - - # Get `trainable` status. - trainable = kwargs.get('trainable', True) - - # Get `dtype`. - dtype = kwargs.get('dtype') - if dtype is None: - dtype = K.floatx() - - # Call super, which will set all properties common to Keras layers - # and core TF layers. - super(Layer, self).__init__( - name=name, dtype=dtype, trainable=trainable, - activity_regularizer=kwargs.get('activity_regularizer')) + # Mutable properties + # Indicates whether the layer's weights are updated during training + # and whether the layer's updates are run during training + self.trainable = trainable + # A stateful layer is a layer whose updates are run during inference too, + # for instance stateful RNNs. + self.stateful = False + # Indicates whether `build` needs to be called upon layer call, to create + # the layer's weights. + self.built = False + # Provides information about which inputs are compatible with the layer. + self.input_spec = None + + self._init_set_name(name) + + activity_regularizer = kwargs.pop('activity_regularizer', None) + if activity_regularizer and context.executing_eagerly(): + raise ValueError( + ('Activity regularization is not supported when executing eagerly. ' + 'Got activity_regularizer=%s') % (activity_regularizer,)) + self._activity_regularizer = activity_regularizer + self._trainable_weights = [] + self._non_trainable_weights = [] + self._updates = [] + # When executing eagerly, _losses is a list of zero-argument lambdas which + # return tensors. When using graph execution, _losses is a list of ops. + self._losses = [] + self._dtype = None if dtype is None else dtypes.as_dtype(dtype).name + self._call_fn_args = estimator_util.fn_args(self.call) + self._compute_previous_mask = ('mask' in self._call_fn_args or + hasattr(self, 'compute_mask')) self._uses_inputs_arg = True - # Add properties that are Keras-only for now. + # These lists will be filled via successive calls + # to self._add_inbound_node(). + self._inbound_nodes = [] + self._outbound_nodes = [] + self.supports_masking = False # Manage input shape information if passed. @@ -172,39 +174,404 @@ class Layer(tf_base_layers.Layer): else: self._initial_weights = None - def add_weight(self, - name, - shape, + def _init_set_name(self, name, zero_based=True): + if not name: + self._name = unique_layer_name( + to_snake_case(self.__class__.__name__), zero_based=zero_based) + else: + self._name = name + + @property + def dtype(self): + return self._dtype + + @property + def name(self): + return self._name + + @property + def activity_regularizer(self): + """Optional regularizer function for the output of this layer.""" + return self._activity_regularizer + + @activity_regularizer.setter + def activity_regularizer(self, regularizer): + """Optional regularizer function for the output of this layer.""" + self._activity_regularizer = regularizer + + @property + def trainable_weights(self): + return self._trainable_weights if self.trainable else [] + + @property + def non_trainable_weights(self): + if self.trainable: + return self._non_trainable_weights + else: + return self._trainable_weights + self._non_trainable_weights + + @property + def trainable_variables(self): + return self.trainable_weights + + @property + def non_trainable_variables(self): + return self.non_trainable_weights + + @property + def weights(self): + """Returns the list of all layer variables/weights. + + Returns: + A list of variables. + """ + return self.trainable_weights + self.non_trainable_weights + + @property + def variables(self): + """Returns the list of all layer variables/weights. + + Returns: + A list of variables. + """ + return self.weights + + @property + def updates(self): + if context.executing_eagerly(): + raise RuntimeError('Layer.updates not supported in Eager mode.') + if not self.trainable and not self.stateful: + return [] + return self._updates + + def add_update(self, updates, inputs=None): + """Add update op(s), potentially dependent on layer inputs. + + Weight updates (for instance, the updates of the moving mean and variance + in a BatchNormalization layer) may be dependent on the inputs passed + when calling a layer. Hence, when reusing the same layer on + different inputs `a` and `b`, some entries in `layer.updates` may be + dependent on `a` and some on `b`. This method automatically keeps track + of dependencies. + + The `get_updates_for` method allows to retrieve the updates relevant to a + specific set of inputs. + + This call is ignored when eager execution is enabled (in that case, variable + updates are run on the fly and thus do not need to be tracked for later + execution). + + Arguments: + updates: Update op, or list/tuple of update ops. + inputs: If anything other than None is passed, it signals the updates + are conditional on some of the layer's inputs, + and thus they should only be run where these inputs are available. + This is the case for BatchNormalization updates, for instance. + If None, the updates will be taken into account unconditionally, + and you are responsible for making sure that any dependency they might + have is available at runtime. + A step counter might fall into this category. + """ + if context.executing_eagerly(): + return # Updates already applied when in eager mode. + + def process_update(x): + if isinstance(x, ops.Operation): + return x + elif hasattr(x, 'op'): + return x.op + else: + return ops.convert_to_tensor(x) + + updates = generic_utils.to_list(updates) + updates = [process_update(x) for x in updates] + self._updates += updates + if inputs is None: + for u in updates: + u._unconditional_update = True # pylint: disable=protected-access + else: + for u in updates: + u._unconditional_update = False # pylint: disable=protected-access + + def get_updates_for(self, inputs): + """Retrieves updates relevant to a specific set of inputs. + + Arguments: + inputs: Input tensor or list/tuple of input tensors. + + Returns: + List of update ops of the layer that depend on `inputs`. + + Raises: + RuntimeError: If called in Eager mode. + """ + if context.executing_eagerly(): + raise RuntimeError('`get_updates_for()` not supported in Eager mode.') + + # Updates disabled if layer is not trainable and not explicitly stateful. + if not self.trainable and not self.stateful: + return [] + + if inputs is None: + # Requesting unconditional updates. + return [x for x in self.updates if x._unconditional_update] # pylint: disable=protected-access + + # Requesting input-conditional updates. + inputs = nest.flatten(inputs) + reachable = get_reachable_from_inputs(inputs, self.updates) + updates = [] + for update in self.updates: + if update in reachable: + updates.append(update) + return updates + + @property + def losses(self): + """Losses which are associated with this `Layer`. + + Note that when executing eagerly, getting this property evaluates + regularizers. When using graph execution, variable regularization ops have + already been created and are simply returned here. + + Returns: + A list of tensors. + """ + if context.executing_eagerly(): + # _losses may only contain variable regularization losses when executing + # eagerly, and they have been saved as lambdas to be executed when + # requested. + return [regularizer() for regularizer in self._losses] + else: + return self._losses + + def add_loss(self, losses, inputs=None): + """Add loss tensor(s), potentially dependent on layer inputs. + + Some losses (for instance, activity regularization losses) may be dependent + on the inputs passed when calling a layer. Hence, when reusing the same + layer on different inputs `a` and `b`, some entries in `layer.losses` may + be dependent on `a` and some on `b`. This method automatically keeps track + of dependencies. + + The `get_losses_for` method allows to retrieve the losses relevant to a + specific set of inputs. + + Note that `add_loss` is not supported when executing eagerly. Instead, + variable regularizers may be added through `add_variable`. Activity + regularization is not supported directly (but such losses may be returned + from `Layer.call()`). + + Arguments: + losses: Loss tensor, or list/tuple of tensors. + inputs: If anything other than None is passed, it signals the losses + are conditional on some of the layer's inputs, + and thus they should only be run where these inputs are available. + This is the case for activity regularization losses, for instance. + If `None` is passed, the losses are assumed + to be unconditional, and will apply across all dataflows of the layer + (e.g. weight regularization losses). + + Raises: + RuntimeError: If called in Eager mode. + """ + if context.executing_eagerly(): + # TODO(fchollet): it should be possible (and highly desirable) to support + # `add_loss` in eager mode. This allows great convenience and flexibility + # in defining custom losses on the fly (e.g. in VAEs). + # Simply appending the loss value to `self._losses` + # is the correct behavior. + # The only caveat is that we need to force the user to only call + # `add_loss` from inside a model or Layer's `call` method + # (otherwise the loss computation cannot be backproped through). + raise RuntimeError('Layer.add_loss not supported in Eager mode.') + + losses = generic_utils.to_list(losses) + self._losses += losses + if inputs is None: + for loss in losses: + loss._unconditional_loss = True # pylint: disable=protected-access + else: + for loss in losses: + loss._unconditional_loss = False # pylint: disable=protected-access + + def get_losses_for(self, inputs): + """Retrieves losses relevant to a specific set of inputs. + + Arguments: + inputs: Input tensor or list/tuple of input tensors. + + Returns: + List of loss tensors of the layer that depend on `inputs`. + + Raises: + RuntimeError: If called in Eager mode. + """ + if context.executing_eagerly(): + raise RuntimeError('Layer.get_losses_for not supported in Eager mode.') + + if inputs is None: + # Requesting unconditional losses. + return [x for x in self.losses if x._unconditional_loss] # pylint: disable=protected-access + + # Requesting input-conditional losses. + inputs = nest.flatten(inputs) + # Retrieve the set of tensors in the TF graph that depend on `inputs`. + # The losses we want to return will be part of this set. + # To avoid unnecessary work, we stop the search in case all of + # `self.losses` have been retrieved. + reachable = get_reachable_from_inputs(inputs, self.losses) + losses = [] + for loss in self.losses: + if loss in reachable: + losses.append(loss) + return losses + + def _name_scope(self): + return self.name + + def build(self, _): + """Creates the variables of the layer.""" + self.built = True + + def add_variable(self, *args, **kwargs): + """Alias for `add_weight`.""" + return self.add_weight(*args, **kwargs) + + def add_weight(self, name, shape, dtype=None, initializer=None, regularizer=None, trainable=True, - constraint=None): - """Adds a weight variable to the layer. + constraint=None, + partitioner=None, + use_resource=None, + getter=None): + """Adds a new variable to the layer, or gets an existing one; returns it. Arguments: - name: String, the name for the weight variable. - shape: The shape tuple of the weight. - dtype: The dtype of the weight. - initializer: An Initializer instance (callable). - regularizer: An optional Regularizer instance. - trainable: A boolean, whether the weight should - be trained via backprop or not (assuming - that the layer itself is also trainable). - constraint: An optional Constraint instance. + name: variable name. + shape: variable shape. + dtype: The type of the variable. Defaults to `self.dtype` or `float32`. + initializer: initializer instance (callable). + regularizer: regularizer instance (callable). + trainable: whether the variable should be part of the layer's + "trainable_variables" (e.g. variables, biases) + or "non_trainable_variables" (e.g. BatchNorm mean, stddev). + Note, if the current variable scope is marked as non-trainable + then this parameter is ignored and any added variables are also + marked as non-trainable. + constraint: constraint instance (callable). + partitioner: Partitioner to be passed to the `Checkpointable` API. + use_resource: Whether to use `ResourceVariable`. + getter: Variable getter argument to be passed to the `Checkpointable` API. Returns: - The created weight variable. + The created variable. Usually either a `Variable` or `ResourceVariable` + instance. If `partitioner` is not `None`, a `PartitionedVariable` + instance is returned. + + Raises: + RuntimeError: If called with partioned variable regularization and + eager execution is enabled. """ if dtype is None: - dtype = K.floatx() - weight = self.add_variable(name, shape, - dtype=dtype, - initializer=initializers.get(initializer), - regularizer=regularizers.get(regularizer), - constraint=constraints.get(constraint), - trainable=trainable) - return weight + dtype = self.dtype or backend.floatx() + initializer = initializers.get(initializer) + if initializer is None: + # Default TensorFlow initializer. + initializer = initializers.glorot_uniform() + regularizer = regularizers.get(regularizer) + constraint = constraints.get(constraint) + + variable = self._add_variable_with_custom_getter( + name=name, + shape=shape, + # TODO(allenl): a `make_variable` equivalent should be added as a + # `Checkpointable` method. + getter=getter or make_variable, + # Manage errors in Layer rather than Checkpointable. + overwrite=True, + initializer=initializer, + dtype=dtypes.as_dtype(dtype), + constraint=constraint, + trainable=trainable and self.trainable, + partitioner=partitioner, + use_resource=use_resource) + + if regularizer is not None: + # TODO(fchollet): in the future, this should be handled at the + # level of variable creation, and weight regularization losses + # should be variable attributes. + self._handle_weight_regularization(name, variable, regularizer) + + if trainable: + self._trainable_weights.append(variable) + else: + self._non_trainable_weights.append(variable) + return variable + + def _handle_weight_regularization(self, name, variable, regularizer): + # `init_graph` should point to the graph in which variable initialization + # will occur; it should be None if and only if initialization will take + # place in the eager context. + init_graph = None + if not context.executing_eagerly(): + default_graph = ops.get_default_graph() + if default_graph.building_function: + with ops.init_scope(): + # Retrieve the variables from the graph into which variables + # will be lifted; if initialization ops will be lifted into + # the eager context, then there is nothing to retrieve, since variable + # collections are not supported when eager execution is enabled. + if not context.executing_eagerly(): + init_graph = ops.get_default_graph() + else: + # Initialization ops will not be lifted out of the default graph. + init_graph = default_graph + + if init_graph is not None: # pylint: disable=protected-access + # The variable was created and initialized in a graph. + if regularizer: + if isinstance(variable, tf_variables.PartitionedVariable): + for v in variable: + with ops.colocate_with(v.op): + with ops.name_scope(name + '/Regularizer'): + regularization = regularizer(v) + if regularization is not None: + self.add_loss(regularization) + else: + with ops.colocate_with(variable.op): + with ops.name_scope(name + '/Regularizer'): + regularization = regularizer(variable) + if regularization is not None: + self.add_loss(regularization) + elif regularizer: # initialization took place in an eager context + if isinstance(variable, tf_variables.PartitionedVariable): + raise RuntimeError( + 'Partitioned variable regularization is not yet ' + 'supported when executing eagerly. File a feature request' + 'if this is important to you.') + # Save a zero-argument lambda which runs the regularizer on the + # variable, to be executed when `Layer.losses` is requested. + # This makes losses responsive to variable updates when executing + # eagerly. + # + # TODO(akshayka): Do the same for graphs as well, so that losses + # collected in a while_loop can be run outside its control flow + # context and so that losses won't be swallowed up by graph functions + # (i.e., `.losses()` should always create regularizers). + self._losses.append(lambda: regularizer(variable)) + + def _handle_activity_regularization(self, inputs, outputs): + # Apply activity regularization. + # Note that it should be applied every time the layer creates a new + # output, since it is output-specific. + if self._activity_regularizer: + output_list = nest.flatten(outputs) + for output in output_list: + with ops.name_scope('ActivityRegularizer'): + activity_regularization = self._activity_regularizer(output) + self.add_loss(activity_regularization, inputs=inputs) def call(self, inputs, **kwargs): # pylint: disable=unused-argument """This is where the layer's logic lives. @@ -218,6 +585,215 @@ class Layer(tf_base_layers.Layer): """ return inputs + def __call__(self, inputs, *args, **kwargs): + """Wraps `call`, applying pre- and post-processing steps. + + Arguments: + inputs: input tensor(s). + *args: additional positional arguments to be passed to `self.call`. + **kwargs: additional keyword arguments to be passed to `self.call`. + + Returns: + Output tensor(s). + + Note: + - The following optional keyword arguments are reserved for specific uses: + * `training`: Boolean scalar tensor of Python boolean indicating + whether the `call` is meant for training or inference. + * `mask`: Boolean input mask. + - If the layer's `call` method takes a `mask` argument (as some Keras + layers do), its default value will be set to the mask generated + for `inputs` by the previous layer (if `input` did come from + a layer that generated a corresponding mask, i.e. if it came from + a Keras layer with masking support. + + Raises: + ValueError: if the layer's `call` method returns None (an invalid value). + """ + input_list = nest.flatten(inputs) + + build_graph = not context.executing_eagerly() + # TODO(fchollet, allenl): Make deferred mode work with subclassed Models + # which don't use an "inputs" argument. + in_deferred_mode = isinstance(input_list[0], DeferredTensor) + + # Handle Keras mask propagation from previous layer to current layer. + previous_mask = None + if (not hasattr(self, '_compute_previous_mask') or + self._compute_previous_mask): + previous_mask = collect_previous_mask(inputs) + if not hasattr(self, '_call_fn_args'): + self._call_fn_args = estimator_util.fn_args(self.call) + if ('mask' in self._call_fn_args and 'mask' not in kwargs and + not is_all_none(previous_mask)): + # The previous layer generated a mask, and mask was not explicitly pass + # to __call__, hence we set previous_mask as the default value. + kwargs['mask'] = previous_mask + + input_shapes = None + + with ops.name_scope(self._name_scope()): + if not self.built: + if not build_graph: + # Activity regularization is currently unsupported in Eager mode. + if self._activity_regularizer: + raise ValueError( + 'activity_regularizer currently unsupported with ' + 'eager execution enabled. Found an activity_regularizer in ' + '%s(%s).' % (self.__class__.__name__, self)) + if not build_graph and not in_deferred_mode: + for x in input_list: + if hasattr(x, '_keras_history'): + raise ValueError('_keras_history currently unsupported in ' + 'Eager mode. Found _keras_history in %s while ' + 'executing __call__ for %s(%s)' % + (x, self.__class_.__name__, self)) + + # Check input assumptions set before layer building, e.g. input rank. + self._assert_input_compatibility(inputs) + if input_list and self._dtype is None: + try: + self._dtype = input_list[0].dtype.base_dtype.name + except AttributeError: + pass + if all(hasattr(x, 'get_shape') for x in input_list): + input_shapes = nest.map_structure(lambda x: x.get_shape(), inputs) + self.build(input_shapes) + + # Check input assumptions set after layer building, e.g. input shape. + if build_graph or in_deferred_mode: + self._assert_input_compatibility(inputs) + + if not in_deferred_mode: + outputs = self.call(inputs, *args, **kwargs) + if outputs is None: + raise ValueError('A layer\'s `call` method should return a Tensor ' + 'or a list of Tensors, not None (layer: ' + + self.name + ').') + else: + # Deferred mode behavior: use `compute_output_shape` to + # infer the number of outputs of the layer and their shapes. + if input_shapes is None: + input_shapes = nest.map_structure(lambda x: x.get_shape(), inputs) + + output_shapes = self.compute_output_shape(input_shapes) + output_shapes = nest.flatten(output_shapes) + outputs = [ + # TODO(fchollet): name the deferred tensors? + DeferredTensor(shape=shape, dtype=self._dtype) + for shape in output_shapes + ] + if len(outputs) == 1: + outputs = outputs[0] + + if build_graph: + self._handle_activity_regularization(inputs, outputs) + # TODO(fchollet): consider enabling masking for Eager mode. + self._set_mask_metadata(inputs, outputs, previous_mask) + + if in_deferred_mode or build_graph and have_all_keras_metadata(inputs): + inputs, outputs = self._set_connectivity_metadata_( + inputs, outputs, args, kwargs) + + self.built = True + if context.executing_eagerly(): + return outputs + + if hasattr(self, '_symbolic_set_inputs') and not self.inputs: + # Subclassed network: explicitly set metadata normally set by a call to + # self._set_inputs(). This is not relevant in eager execution. + self._symbolic_set_inputs(inputs, outputs) + + if in_deferred_mode or build_graph: + self._set_learning_phase_metadata(inputs, outputs) + + # Optionally load weight values that were specified at layer instantiation. + # TODO(fchollet): consider enabling this with eager execution too. + if hasattr(self, '_initial_weights') and self._initial_weights is not None: + self.set_weights(self._initial_weights) + del self._initial_weights + return outputs + + def apply(self, inputs, *args, **kwargs): + """Apply the layer on a input. + + This simply wraps `self.__call__`. + + Arguments: + inputs: Input tensor(s). + *args: additional positional arguments to be passed to `self.call`. + **kwargs: additional keyword arguments to be passed to `self.call`. + + Returns: + Output tensor(s). + """ + return self.__call__(inputs, *args, **kwargs) + + def _set_learning_phase_metadata(self, inputs, outputs): + # Update learning phase info. To work with subclassed models, + # this should be done even if Keras metadata is absent. + output_tensors = generic_utils.to_list(outputs) + uses_lp = any( + [getattr(x, '_uses_learning_phase', False) + for x in generic_utils.to_list(inputs)]) + uses_lp = getattr(self, 'uses_learning_phase', False) or uses_lp + for i in range(len(output_tensors)): + try: + output_tensors[i]._uses_learning_phase = getattr( + output_tensors[i], '_uses_learning_phase', False) or uses_lp + except AttributeError: + # An output element happens to be a C type (such as tuple or dict). + # We don't track learning phase info in such edge cases. + pass + + def _set_mask_metadata(self, inputs, outputs, previous_mask): + if hasattr(self, 'compute_mask'): + output_mask = self.compute_mask(inputs, previous_mask) + if isinstance(outputs, (list, tuple)): + if output_mask is None: + output_mask = [None for _ in range(len(outputs))] + for x, m in zip(outputs, output_mask): + try: + x._keras_mask = m # pylint: disable=protected-access + except AttributeError: + pass # C type such as dict. Masking not supported in this case. + else: + try: + outputs._keras_mask = output_mask # pylint: disable=protected-access + except AttributeError: + pass # C type such as dict. Masking not supported in this case. + + def _set_connectivity_metadata_(self, inputs, outputs, args, kwargs): + if args and getattr(self, '_uses_inputs_arg', True): + raise TypeError( + 'This Layer takes an `inputs` argument to call(), and only the ' + '`inputs` argument may be specified as a positional argument. ' + 'Pass everything else as a keyword argument (those arguments will' + ' not be tracked as inputs to the Layer).') + + # If the layer returns tensors from its inputs, unmodified, + # we copy them to avoid loss of tensor metadata. + output_ls = nest.flatten(outputs) + output_ls_copy = [] + for x in output_ls: + if x in nest.flatten(inputs): + with ops.name_scope(self.name): + x = array_ops.identity(x) + output_ls_copy.append(x) + if len(output_ls_copy) == 1: + outputs = output_ls_copy[0] + else: + outputs = output_ls_copy + + inputs, kwargs = self._inputs_from_call_args( + call_args=(inputs,) + args, call_kwargs=kwargs) + # Add an inbound node to the layer, so it can keep track of this call. + # This updates the layer history of the output tensor(s). + kwargs.pop('mask', None) # `mask` should not be serialized. + self._add_inbound_node( + input_tensors=inputs, output_tensors=outputs, arguments=kwargs) + return inputs, outputs + def _inputs_from_call_args(self, call_args, call_kwargs): """Get Layer inputs from __call__ *args and **kwargs. @@ -282,71 +858,6 @@ class Layer(tf_base_layers.Layer): input_arg_values.extend(bound_args[call_arg_spec.varargs]) return input_arg_values, non_input_arg_values - def __call__(self, inputs, *args, **kwargs): - """Wrapper around self.call(), for handling internal references. - - If a Keras tensor is passed: - - We call self._add_inbound_node(). - - If necessary, we `build` the layer to match - the shape of the input(s). - - We update the _keras_history of the output tensor(s) - with the current layer. - This is done as part of _add_inbound_node(). - - Arguments: - inputs: Can be a tensor or list/tuple of tensors. - *args: Additional positional arguments to be passed to `call()`. Only - allowed in subclassed Models with custom call() signatures. In other - cases, `Layer` inputs must be passed using the `inputs` argument and - non-inputs must be keyword arguments. - **kwargs: Additional keyword arguments to be passed to `call()`. - - Returns: - Output of the layer's `call` method. - - Raises: - ValueError: in case the layer is missing shape information - for its `build` call. - TypeError: If positional arguments are passed and this `Layer` is not a - subclassed `Model`. - """ - # Actually call the layer (optionally building it). - output = super(Layer, self).__call__(inputs, *args, **kwargs) - - if args and getattr(self, '_uses_inputs_arg', True): - raise TypeError( - 'This Layer takes an `inputs` argument to call(), and only the ' - '`inputs` argument may be specified as a positional argument. Pass ' - 'everything else as a keyword argument (those arguments will not be ' - 'tracked as inputs to the Layer).') - - if context.executing_eagerly(): - return output - - inputs, kwargs = self._inputs_from_call_args( - call_args=(inputs,) + args, call_kwargs=kwargs) - - if hasattr(self, '_symbolic_set_inputs') and not self.inputs: - # Subclassed network: explicitly set metadata normally set by a call to - # self._set_inputs(). - self._symbolic_set_inputs(inputs, output) - - # Update learning phase info. - output_tensors = generic_utils.to_list(output) - uses_lp = any( - [getattr(x, '_uses_learning_phase', False) - for x in generic_utils.to_list(inputs)]) - uses_lp = getattr(self, 'uses_learning_phase', False) or uses_lp - for i in range(len(output_tensors)): - output_tensors[i]._uses_learning_phase = getattr( - output_tensors[i], '_uses_learning_phase', False) or uses_lp - - # Optionally load weight values that were specified at layer instantiation. - if hasattr(self, '_initial_weights') and self._initial_weights is not None: - self.set_weights(self._initial_weights) - del self._initial_weights - return output - def compute_output_shape(self, input_shape): """Computes the output shape of the layer. @@ -362,13 +873,7 @@ class Layer(tf_base_layers.Layer): Returns: An input shape tuple. """ - logging.warning( - 'All custom layers should implement the ' - '`compute_output_shape` method. This layer (' + self.name + ') ' - 'is relying on the base `Layer.compute_output_shape` implementation, ' - 'which will start raising a `NotImplementedError` ' - 'as of July 1st, 2018.') - return input_shape + raise NotImplementedError def compute_mask(self, inputs, mask=None): # pylint: disable=unused-argument """Computes an output mask tensor. @@ -396,6 +901,87 @@ class Layer(tf_base_layers.Layer): # carry over the input mask return mask + def _add_inbound_node(self, + input_tensors, + output_tensors, + arguments=None): + """Internal method to create an inbound node for the layer. + + Arguments: + input_tensors: list of input tensors. + output_tensors: list of output tensors. + arguments: dictionary of keyword arguments that were passed to the + `call` method of the layer at the call that created the node. + """ + input_tensors = nest.flatten(input_tensors) + output_tensors = nest.flatten(output_tensors) + + # Collect input tensor(s) coordinates. + inbound_layers = [] + node_indices = [] + tensor_indices = [] + for x in input_tensors: + assert hasattr(x, '_keras_history') + inbound_layer, node_index, tensor_index = x._keras_history # pylint: disable=protected-access + inbound_layers.append(inbound_layer) + node_indices.append(node_index) + tensor_indices.append(tensor_index) + + # Create node, add it to inbound nodes. + Node( + self, + inbound_layers=inbound_layers, + node_indices=node_indices, + tensor_indices=tensor_indices, + input_tensors=input_tensors, + output_tensors=output_tensors, + arguments=arguments) + + # Update tensor history metadata. + for i in range(len(output_tensors)): + # The metadata attribute consists of 1) a layer instance + # 2) a node index for the layer, 3) a tensor index for the node. + # The allows layer reuse (multiple nodes per layer) and multi-output + # or multi-input layers (e.g. a layer can return multiple tensors, + # and each can be sent to a different layer). + output_tensors[i]._keras_history = (self, len(self._inbound_nodes) - 1, i) # pylint: disable=protected-access + + def _get_node_attribute_at_index(self, node_index, attr, attr_name): + """Private utility to retrieves an attribute (e.g. inputs) from a node. + + This is used to implement the methods: + - get_input_shape_at + - get_output_shape_at + - get_input_at + etc... + + Arguments: + node_index: Integer index of the node from which + to retrieve the attribute. + attr: Exact node attribute name. + attr_name: Human-readable attribute name, for error messages. + + Returns: + The layer's attribute `attr` at the node of index `node_index`. + + Raises: + RuntimeError: If the layer has no inbound nodes, or if called in Eager + mode. + ValueError: If the index provided does not match any node. + """ + if not self._inbound_nodes: + raise RuntimeError('The layer has never been called ' + 'and thus has no defined ' + attr_name + '.') + if not len(self._inbound_nodes) > node_index: + raise ValueError('Asked to get ' + attr_name + ' at node ' + + str(node_index) + ', but the layer has only ' + + str(len(self._inbound_nodes)) + ' inbound nodes.') + values = getattr(self._inbound_nodes[node_index], attr) + if len(values) == 1: + return values[0] + else: + return values + def get_input_mask_at(self, node_index): """Retrieves the input mask tensor(s) of a layer at a given node. @@ -476,6 +1062,325 @@ class Layer(tf_base_layers.Layer): else: return getattr(output, '_keras_mask', None) + def get_input_shape_at(self, node_index): + """Retrieves the input shape(s) of a layer at a given node. + + Arguments: + node_index: Integer, index of the node + from which to retrieve the attribute. + E.g. `node_index=0` will correspond to the + first time the layer was called. + + Returns: + A shape tuple + (or list of shape tuples if the layer has multiple inputs). + + Raises: + RuntimeError: If called in Eager mode. + """ + return self._get_node_attribute_at_index(node_index, 'input_shapes', + 'input shape') + + def get_output_shape_at(self, node_index): + """Retrieves the output shape(s) of a layer at a given node. + + Arguments: + node_index: Integer, index of the node + from which to retrieve the attribute. + E.g. `node_index=0` will correspond to the + first time the layer was called. + + Returns: + A shape tuple + (or list of shape tuples if the layer has multiple outputs). + + Raises: + RuntimeError: If called in Eager mode. + """ + return self._get_node_attribute_at_index(node_index, 'output_shapes', + 'output shape') + + def get_input_at(self, node_index): + """Retrieves the input tensor(s) of a layer at a given node. + + Arguments: + node_index: Integer, index of the node + from which to retrieve the attribute. + E.g. `node_index=0` will correspond to the + first time the layer was called. + + Returns: + A tensor (or list of tensors if the layer has multiple inputs). + + Raises: + RuntimeError: If called in Eager mode. + """ + return self._get_node_attribute_at_index(node_index, 'input_tensors', + 'input') + + def get_output_at(self, node_index): + """Retrieves the output tensor(s) of a layer at a given node. + + Arguments: + node_index: Integer, index of the node + from which to retrieve the attribute. + E.g. `node_index=0` will correspond to the + first time the layer was called. + + Returns: + A tensor (or list of tensors if the layer has multiple outputs). + + Raises: + RuntimeError: If called in Eager mode. + """ + return self._get_node_attribute_at_index(node_index, 'output_tensors', + 'output') + + @property + def input(self): + """Retrieves the input tensor(s) of a layer. + + Only applicable if the layer has exactly one input, + i.e. if it is connected to one incoming layer. + + Returns: + Input tensor or list of input tensors. + + Raises: + AttributeError: if the layer is connected to + more than one incoming layers. + + Raises: + RuntimeError: If called in Eager mode. + AttributeError: If no inbound nodes are found. + """ + if not self._inbound_nodes: + raise AttributeError('Layer ' + self.name + + ' is not connected, no input to return.') + return self._get_node_attribute_at_index(0, 'input_tensors', 'input') + + @property + def output(self): + """Retrieves the output tensor(s) of a layer. + + Only applicable if the layer has exactly one output, + i.e. if it is connected to one incoming layer. + + Returns: + Output tensor or list of output tensors. + + Raises: + AttributeError: if the layer is connected to more than one incoming + layers. + RuntimeError: if called in Eager mode. + """ + if not self._inbound_nodes: + raise AttributeError('Layer ' + self.name + ' has no inbound nodes.') + return self._get_node_attribute_at_index(0, 'output_tensors', 'output') + + @property + def input_shape(self): + """Retrieves the input shape(s) of a layer. + + Only applicable if the layer has exactly one input, + i.e. if it is connected to one incoming layer, or if all inputs + have the same shape. + + Returns: + Input shape, as an integer shape tuple + (or list of shape tuples, one tuple per input tensor). + + Raises: + AttributeError: if the layer has no defined input_shape. + RuntimeError: if called in Eager mode. + """ + if not self._inbound_nodes: + raise AttributeError('The layer has never been called ' + 'and thus has no defined input shape.') + all_input_shapes = set( + [str(node.input_shapes) for node in self._inbound_nodes]) + if len(all_input_shapes) == 1: + input_shapes = self._inbound_nodes[0].input_shapes + if len(input_shapes) == 1: + return tuple(tensor_shape.TensorShape(input_shapes[0]).as_list()) + else: + return [ + tuple(tensor_shape.TensorShape(shape).as_list()) + for shape in input_shapes + ] + else: + raise AttributeError('The layer "' + str(self.name) + + ' has multiple inbound nodes, ' + 'with different input shapes. Hence ' + 'the notion of "input shape" is ' + 'ill-defined for the layer. ' + 'Use `get_input_shape_at(node_index)` ' + 'instead.') + + def count_params(self): + """Count the total number of scalars composing the weights. + + Returns: + An integer count. + + Raises: + ValueError: if the layer isn't yet built + (in which case its weights aren't yet defined). + """ + if not self.built: + if self.__class__.__name__ == 'Sequential': + self.build() # pylint: disable=no-value-for-parameter + else: + raise ValueError('You tried to call `count_params` on ' + self.name + + ', but the layer isn\'t built. ' + 'You can build it manually via: `' + self.name + + '.build(batch_input_shape)`.') + weight_shapes = [w.get_shape().as_list() for w in self.weights] + return int(sum([np.prod(w) for w in weight_shapes])) + + @property + def output_shape(self): + """Retrieves the output shape(s) of a layer. + + Only applicable if the layer has one output, + or if all outputs have the same shape. + + Returns: + Output shape, as an integer shape tuple + (or list of shape tuples, one tuple per output tensor). + + Raises: + AttributeError: if the layer has no defined output shape. + RuntimeError: if called in Eager mode. + """ + if not self._inbound_nodes: + raise AttributeError('The layer has never been called ' + 'and thus has no defined output shape.') + all_output_shapes = set( + [str(node.output_shapes) for node in self._inbound_nodes]) + if len(all_output_shapes) == 1: + output_shapes = self._inbound_nodes[0].output_shapes + if len(output_shapes) == 1: + return tuple(tensor_shape.TensorShape(output_shapes[0]).as_list()) + else: + return [ + tuple(tensor_shape.TensorShape(shape).as_list()) + for shape in output_shapes + ] + else: + raise AttributeError('The layer "%s"' + ' has multiple inbound nodes, ' + 'with different output shapes. Hence ' + 'the notion of "output shape" is ' + 'ill-defined for the layer. ' + 'Use `get_output_shape_at(node_index)` ' + 'instead.' % self.name) + + @property + def inbound_nodes(self): + """Deprecated, do NOT use! Only for compatibility with external Keras.""" + return self._inbound_nodes + + @property + def outbound_nodes(self): + """Deprecated, do NOT use! Only for compatibility with external Keras.""" + return self._outbound_nodes + + def _assert_input_compatibility(self, inputs): + """Checks compatibility between the layer and provided inputs. + + This checks that the tensor(s) `inputs` verify the input assumptions + of the layer (if any). If not, a clear and actional exception gets raised. + + Arguments: + inputs: input tensor or list of input tensors. + + Raises: + ValueError: in case of mismatch between + the provided inputs and the expectations of the layer. + """ + if not self.input_spec: + return + if not isinstance(self.input_spec, (list, tuple)): + input_spec = nest.flatten(self.input_spec) + else: + input_spec = self.input_spec + inputs = nest.flatten(inputs) + if len(inputs) != len(input_spec): + raise ValueError('Layer ' + self.name + ' expects ' + + str(len(input_spec)) + ' inputs, ' + 'but it received ' + str(len(inputs)) + + ' input tensors. Inputs received: ' + str(inputs)) + for input_index, (x, spec) in enumerate(zip(inputs, input_spec)): + if spec is None: + continue + + if (spec.ndim is not None or + spec.min_ndim is not None or + spec.max_ndim is not None): + if x.get_shape().ndims is None: + raise ValueError('Input ' + str(input_index) + ' of layer ' + + self.name + ' is incompatible with the layer: ' + 'its rank is undefined, but the layer requires a ' + 'defined rank.') + + # Check ndim. + if spec.ndim is not None: + ndim = x.get_shape().ndims + if ndim != spec.ndim: + raise ValueError('Input ' + str(input_index) + ' of layer ' + + self.name + ' is incompatible with the layer: ' + 'expected ndim=' + str(spec.ndim) + ', found ndim=' + + str(ndim) + '. Full shape received: ' + + str(x.get_shape().as_list())) + if spec.max_ndim is not None: + ndim = x.get_shape().ndims + if ndim is not None and ndim > spec.max_ndim: + raise ValueError('Input ' + str(input_index) + ' of layer ' + + self.name + ' is incompatible with the layer: ' + 'expected max_ndim=' + str(spec.max_ndim) + + ', found ndim=' + str(ndim)) + if spec.min_ndim is not None: + ndim = x.get_shape().ndims + if ndim is not None and ndim < spec.min_ndim: + raise ValueError('Input ' + str(input_index) + ' of layer ' + + self.name + ' is incompatible with the layer: ' + ': expected min_ndim=' + str(spec.min_ndim) + + ', found ndim=' + str(ndim) + + '. Full shape received: ' + + str(x.get_shape().as_list())) + # Check dtype. + if spec.dtype is not None: + if x.dtype != spec.dtype: + raise ValueError('Input ' + str(input_index) + ' of layer ' + + self.name + ' is incompatible with the layer: ' + 'expected dtype=' + str(spec.dtype) + + ', found dtype=' + str(x.dtype)) + # Check specific shape axes. + if spec.axes: + shape = x.get_shape().as_list() + if shape is not None: + for axis, value in spec.axes.items(): + if hasattr(value, 'value'): + value = value.value + if value is not None and shape[int(axis)] not in {value, None}: + raise ValueError( + 'Input ' + str(input_index) + ' of layer ' + self.name + ' is' + ' incompatible with the layer: expected axis ' + str(axis) + + ' of input shape to have value ' + str(value) + + ' but received input with shape ' + str(shape)) + # Check shape. + if spec.shape is not None: + shape = x.get_shape().as_list() + if shape is not None: + for spec_dim, dim in zip(spec.shape, shape): + if spec_dim is not None and dim is not None: + if spec_dim != dim: + raise ValueError('Input ' + str(input_index) + + ' is incompatible with layer ' + self.name + + ': expected shape=' + str(spec.shape) + + ', found shape=' + str(shape)) + def set_weights(self, weights): """Sets the weights of the layer, from Numpy arrays. @@ -500,14 +1405,14 @@ class Layer(tf_base_layers.Layer): if not params: return weight_value_tuples = [] - param_values = K.batch_get_value(params) + param_values = backend.batch_get_value(params) for pv, p, w in zip(param_values, params, weights): if pv.shape != w.shape: raise ValueError('Layer weight shape ' + str(pv.shape) + ' not compatible with ' 'provided weight shape ' + str(w.shape)) weight_value_tuples.append((p, w)) - K.batch_set_value(weight_value_tuples) + backend.batch_set_value(weight_value_tuples) def get_weights(self): """Returns the current weights of the layer. @@ -516,7 +1421,7 @@ class Layer(tf_base_layers.Layer): Weights values as a list of numpy arrays. """ params = self.weights - return K.batch_get_value(params) + return backend.batch_get_value(params) def get_config(self): """Returns the config of the layer. @@ -558,9 +1463,196 @@ class Layer(tf_base_layers.Layer): """ return cls(**config) - @tf_base_layers.Layer.activity_regularizer.setter - def activity_regularizer(self, activity_regularizer): - self._activity_regularizer = activity_regularizer + +@tf_export('keras.layers.InputSpec', 'layers.InputSpec') +class InputSpec(object): + """Specifies the ndim, dtype and shape of every input to a layer. + + Every layer should expose (if appropriate) an `input_spec` attribute: + a list of instances of InputSpec (one per input tensor). + + A None entry in a shape is compatible with any dimension, + a None shape is compatible with any shape. + + Arguments: + dtype: Expected DataType of the input. + shape: Shape tuple, expected shape of the input + (may include None for unchecked axes). + ndim: Integer, expected rank of the input. + max_ndim: Integer, maximum rank of the input. + min_ndim: Integer, minimum rank of the input. + axes: Dictionary mapping integer axes to + a specific dimension value. + """ + + def __init__(self, + dtype=None, + shape=None, + ndim=None, + max_ndim=None, + min_ndim=None, + axes=None): + self.dtype = dtype + self.shape = shape + if shape is not None: + self.ndim = len(shape) + else: + self.ndim = ndim + self.max_ndim = max_ndim + self.min_ndim = min_ndim + self.axes = axes or {} + + def __repr__(self): + spec = [('dtype=' + str(self.dtype)) if self.dtype else '', + ('shape=' + str(self.shape)) if self.shape else '', + ('ndim=' + str(self.ndim)) if self.ndim else '', + ('max_ndim=' + str(self.max_ndim)) if self.max_ndim else '', + ('min_ndim=' + str(self.min_ndim)) if self.min_ndim else '', + ('axes=' + str(self.axes)) if self.axes else ''] + return 'InputSpec(%s)' % ', '.join(x for x in spec if x) + + +class Node(object): + """A `Node` describes the connectivity between two layers. + + Each time a layer is connected to some new input, + a node is added to `layer._inbound_nodes`. + Each time the output of a layer is used by another layer, + a node is added to `layer._outbound_nodes`. + + Arguments: + outbound_layer: the layer that takes + `input_tensors` and turns them into `output_tensors` + (the node gets created when the `call` + method of the layer was called). + inbound_layers: a list of layers, the same length as `input_tensors`, + the layers from where `input_tensors` originate. + node_indices: a list of integers, the same length as `inbound_layers`. + `node_indices[i]` is the origin node of `input_tensors[i]` + (necessary since each inbound layer might have several nodes, + e.g. if the layer is being shared with a different data stream). + tensor_indices: a list of integers, + the same length as `inbound_layers`. + `tensor_indices[i]` is the index of `input_tensors[i]` within the + output of the inbound layer + (necessary since each inbound layer might + have multiple tensor outputs, with each one being + independently manipulable). + input_tensors: list of input tensors. + output_tensors: list of output tensors. + arguments: dictionary of keyword arguments that were passed to the + `call` method of the layer at the call that created the node. + + `node_indices` and `tensor_indices` are basically fine-grained coordinates + describing the origin of the `input_tensors`. + + A node from layer A to layer B is added to: + - A._outbound_nodes + - B._inbound_nodes + """ + + def __init__(self, + outbound_layer, + inbound_layers, + node_indices, + tensor_indices, + input_tensors, + output_tensors, + arguments=None): + # Layer instance (NOT a list). + if isinstance(outbound_layer, list): + raise ValueError( + '`outbound_layer` should be a layer instance, not a list.') + # this is the layer that takes a list of input tensors + # and turns them into a list of output tensors. + # the current node will be added to + # the inbound_nodes of outbound_layer. + self.outbound_layer = outbound_layer + + # The following 3 properties describe where + # the input tensors come from: which layers, + # and for each layer, which node and which + # tensor output of each node. + + # List of layer instances. + self.inbound_layers = inbound_layers + # List of integers, 1:1 mapping with inbound_layers. + self.node_indices = node_indices + # List of integers, 1:1 mapping with inbound_layers. + self.tensor_indices = tensor_indices + + # Following 2 properties: + # tensor inputs and outputs of outbound_layer. + + # List of tensors. 1:1 mapping with inbound_layers. + self.input_tensors = input_tensors + # List of tensors, created by outbound_layer.call(). + self.output_tensors = output_tensors + + # Following 2 properties: input and output shapes. + + # List of shape tuples, shapes of input_tensors. + self.input_shapes = [static_shape(x) for x in input_tensors] + # List of shape tuples, shapes of output_tensors. + self.output_shapes = [static_shape(x) for x in output_tensors] + + # Optional keyword arguments to layer's `call`. + self.arguments = arguments + + # Add nodes to all layers involved. + for layer in inbound_layers: + if layer is not None: + # For compatibility with external Keras, we use the deprecated + # accessor here. + layer.outbound_nodes.append(self) + # For compatibility with external Keras, we use the deprecated + # accessor here. + outbound_layer.inbound_nodes.append(self) + + def get_config(self): + inbound_names = [] + for layer in self.inbound_layers: + if layer: + inbound_names.append(layer.name) + else: + inbound_names.append(None) + return { + 'outbound_layer': self.outbound_layer.name, + 'inbound_layers': inbound_names, + 'node_indices': self.node_indices, + 'tensor_indices': self.tensor_indices + } + + +class DeferredTensor(object): + """Tensor-like object used to build graphs of layers in Eager mode. + + When calling a layer on a DeferredTensor, the layer will not perform any + computation and will simply perfom shape inference to return new + DeferredTensors with appropriate shape information. Thus DeferredTensor + behaves like a graph-mode Tensor when manipulated by layers. + """ + + def __init__(self, shape, dtype, name=None): + self.shape = tensor_shape.TensorShape(shape) + if dtype is None: + self.dtype = dtypes.as_dtype(np.float32) + else: + self.dtype = dtypes.as_dtype(dtype) + self.name = name + + def get_shape(self): + return self.shape + + def __str__(self): + return "DeferredTensor('%s', shape=%s, dtype=%s)" % (self.name, + self.get_shape(), + self.dtype.name) + + def __repr__(self): + return "" % (self.name, + self.get_shape(), + self.dtype.name) def shape_type_conversion(fn): @@ -589,3 +1681,251 @@ def shape_type_conversion(fn): return tensor_shape.TensorShape(output_shape) return wrapper + + +def object_list_uid(object_list): + """Creates a single string from object ids.""" + object_list = nest.flatten(object_list) + return ', '.join([str(abs(id(x))) for x in object_list]) + + +def static_shape(x): + """Get the static shape of a Tensor, or None if it is unavailable.""" + if x is None: + return None + try: + return tuple(x.get_shape().as_list()) + except ValueError: + return None + + +def get_reachable_from_inputs(inputs, targets=None): + """Returns the set of tensors/ops reachable from `inputs`. + + Stops if all targets have been found (target is optional). + + Only valid in Symbolic mode, not Eager mode. + + Args: + inputs: List of tensors. + targets: List of tensors. + + Returns: + A set of tensors reachable from the inputs (includes the inputs themselves). + """ + reachable = set(inputs) + if targets: + targets = set(targets) + queue = inputs[:] + + while queue: + x = queue.pop() + if isinstance(x, ops.Operation): + outputs = x.outputs[:] or [] + outputs += x._control_outputs + elif isinstance(x, ops.Tensor): + outputs = x.consumers() + elif isinstance(x, tf_variables.Variable): + outputs = [x.op] + else: + raise TypeError('Expected Operation, Variable, or Tensor, got ' + str(x)) + + for y in outputs: + if y not in reachable: + reachable.add(y) + queue.insert(0, y) + + if targets and targets.issubset(reachable): + return reachable + return reachable + + +def unique_layer_name(name, name_uid_map=None, avoid_names=None, namespace='', + zero_based=False): + """Makes a layer name (or arbitrary string) unique within a TensorFlow graph. + + Arguments: + name: String name to make unique. + name_uid_map: An optional defaultdict(int) to use when creating unique + names. If None (default), uses a per-Graph dictionary. + avoid_names: An optional set or dict with names which should not be used. If + None (default) does not avoid any names. + namespace: Gets a name which is unique within the (graph, namespace). Layers + which are not Networks use a blank namespace and so get graph-global + names. + zero_based: If True, name sequences start with no suffix (e.g. "dense", + "dense_1"). If False, naming is one-based ("dense_1", "dense_2"). + + Returns: + Unique string name. + + Example: + + ```python + _unique_layer_name('dense') # dense_1 + _unique_layer_name('dense') # dense_2 + ``` + """ + if name_uid_map is None: + name_uid_map = get_default_graph_uid_map() + if avoid_names is None: + avoid_names = set() + proposed_name = None + while proposed_name is None or proposed_name in avoid_names: + name_key = (namespace, name) + if zero_based: + number = name_uid_map[name_key] + if number: + proposed_name = name + '_' + str(number) + else: + proposed_name = name + name_uid_map[name_key] += 1 + else: + name_uid_map[name_key] += 1 + proposed_name = name + '_' + str(name_uid_map[name_key]) + return proposed_name + + +def to_snake_case(name): + intermediate = re.sub('(.)([A-Z][a-z0-9]+)', r'\1_\2', name) + insecure = re.sub('([a-z])([A-Z])', r'\1_\2', intermediate).lower() + # If the class is private the name starts with "_" which is not secure + # for creating scopes. We prefix the name with "private" in this case. + if insecure[0] != '_': + return insecure + return 'private' + insecure + + +def is_all_none(iterable_or_element): + if not isinstance(iterable_or_element, (list, tuple)): + iterable = [iterable_or_element] + else: + iterable = iterable_or_element + # We cannot use Python's `any` because the iterable may return Tensors. + for element in iterable: + if element is not None: + return False + return True + + +def have_all_keras_metadata(iterable_or_element): + if not isinstance(iterable_or_element, (list, tuple)): + iterable = [iterable_or_element] + else: + iterable = iterable_or_element + return all([hasattr(x, '_keras_history') for x in iterable]) + + +def collect_previous_mask(input_tensors): + """Retrieves the output mask(s) of the previous node. + + Arguments: + input_tensors: A tensor or list of tensors. + + Returns: + A mask tensor or list of mask tensors. + """ + input_tensors = nest.flatten(input_tensors) + masks = [] + for x in input_tensors: + if hasattr(x, '_keras_mask'): + mask = x._keras_mask # pylint: disable=protected-access + masks.append(mask) + else: + masks.append(None) + if len(masks) == 1: + return masks[0] + return masks + + +def is_tensor_or_tensor_list(v): + v = nest.flatten(v) + if v and isinstance(v[0], ops.Tensor): + return True + else: + return False + + +def get_default_graph_uid_map(): + # TODO(fchollet): refactor this into backend. + graph = ops.get_default_graph() + name_uid_map = backend.PER_GRAPH_LAYER_NAME_UIDS.get(graph, None) + if name_uid_map is None: + name_uid_map = collections.defaultdict(int) + backend.PER_GRAPH_LAYER_NAME_UIDS[graph] = name_uid_map + return name_uid_map + + +def make_variable(name, + shape=None, + dtype=dtypes.float32, + initializer=None, + partition_info=None, + trainable=True, + caching_device=None, + validate_shape=True, + constraint=None, + use_resource=None, + partitioner=None): # pylint: disable=unused-argument + """Temporary util to create a variable (relies on `variable_scope.variable`). + + Some reuse-related technicalities prevent us from using + `variable_scope.get_variable()` directly, so we use a subcomponent + that has fewer constraints (`variable_scope.variable()`). + + In the longer term, it seems like a similar "default variable creator" method + should exist in `CheckpointableBase` instead. When this happens, we can get + rid of this temporary solution. + + TODO(fchollet): remove this method when no longer needed. + TODO(fchollet): handle `partitioner` argument. + + Arguments: + name: Variable name. + shape: Variable shape. + dtype: The type of the variable. Defaults to `self.dtype` or `float32`. + initializer: Initializer instance (callable). + partition_info: Not handled at this time. + trainable: Whether the variable should be part of the layer's + "trainable_variables" (e.g. variables, biases) + or "non_trainable_variables" (e.g. BatchNorm mean, stddev). + Note, if the current variable scope is marked as non-trainable + then this parameter is ignored and any added variables are also + marked as non-trainable. + caching_device: Passed to `vs.variable`. + validate_shape: Passed to `vs.variable`. + constraint: Constraint instance (callable). + use_resource: Whether to use a `ResourceVariable`. + partitioner: Not handled at this time. + + Returns: + Variable instance. + """ + initializing_from_value = False + if initializer is not None and not callable(initializer): + initializing_from_value = True + + with ops.init_scope(): + if initializing_from_value: + init_val = initializer + variable_dtype = None + else: + # Instantiate initializer if provided initializer is a type object. + if isinstance(initializer, type(init_ops.Initializer)): + initializer = initializer(dtype=dtype) + init_val = lambda: initializer( # pylint: disable=g-long-lambda + shape, dtype=dtype, partition_info=partition_info) + variable_dtype = dtype.base_dtype + if use_resource is None: + use_resource = True + + v = vs.variable( + initial_value=init_val, + name=name, + trainable=trainable, + caching_device=caching_device, + dtype=variable_dtype, + validate_shape=validate_shape, + constraint=constraint, + use_resource=use_resource) + return v diff --git a/tensorflow/python/keras/_impl/keras/engine/input_layer.py b/tensorflow/python/keras/_impl/keras/engine/input_layer.py index b51dd8a218..bd9dcbe3c5 100644 --- a/tensorflow/python/keras/_impl/keras/engine/input_layer.py +++ b/tensorflow/python/keras/_impl/keras/engine/input_layer.py @@ -23,7 +23,6 @@ from tensorflow.python.eager import context from tensorflow.python.framework import tensor_shape from tensorflow.python.keras._impl.keras import backend as K from tensorflow.python.keras._impl.keras.engine import base_layer -from tensorflow.python.layers import base as tf_base_layers from tensorflow.python.ops import array_ops from tensorflow.python.util.tf_export import tf_export @@ -95,7 +94,7 @@ class InputLayer(base_layer.Layer): if context.executing_eagerly(): # In eager mode, create a temporary placeholder to call the layer on. - input_tensor = tf_base_layers._DeferredTensor( # pylint: disable=protected-access + input_tensor = base_layer.DeferredTensor( # pylint: disable=protected-access shape=batch_input_shape, dtype=dtype, name=self.name) @@ -123,7 +122,7 @@ class InputLayer(base_layer.Layer): # Create an input node to add to self.outbound_node # and set output_tensors' _keras_history. input_tensor._keras_history = (self, 0, 0) # pylint: disable=protected-access - tf_base_layers.Node( + base_layer.Node( self, inbound_layers=[], node_indices=[], diff --git a/tensorflow/python/keras/_impl/keras/engine/network.py b/tensorflow/python/keras/_impl/keras/engine/network.py index 9f1c7de115..cc177c14a8 100644 --- a/tensorflow/python/keras/_impl/keras/engine/network.py +++ b/tensorflow/python/keras/_impl/keras/engine/network.py @@ -35,8 +35,6 @@ from tensorflow.python.keras._impl.keras.engine import saving from tensorflow.python.keras._impl.keras.utils import generic_utils from tensorflow.python.keras._impl.keras.utils.io_utils import ask_to_proceed_with_overwrite from tensorflow.python.keras._impl.keras.utils.layer_utils import print_summary as print_layer_summary -from tensorflow.python.layers import base as tf_base_layers -from tensorflow.python.layers import utils as tf_layers_util from tensorflow.python.platform import tf_logging as logging from tensorflow.python.training import checkpointable from tensorflow.python.util import nest @@ -82,7 +80,7 @@ class Network(base_layer.Layer): # self.losses # self.updates - self._init_set_name(name) + self._init_set_name(name, zero_based=True) self._activity_regularizer = None # This acts just like the `trainable` attribute of any layer instance. # It does not affect users of the underlying layers, only users of the @@ -132,14 +130,14 @@ class Network(base_layer.Layer): if context.executing_eagerly(): # Check that all inputs/outputs are DeferredTensors. for tensor in self.inputs: - if not isinstance(tensor, tf_base_layers._DeferredTensor): # pylint: disable=protected-access + if not isinstance(tensor, base_layer.DeferredTensor): # pylint: disable=protected-access raise TypeError('When eager execution is enabled, ' 'inputs must come from a call to ' '`tf.keras.Input` (called after ' 'tfe.enable_eager_execution()). ' 'Received invalid input: ' + str(tensor)) for tensor in self.outputs: - if not isinstance(tensor, tf_base_layers._DeferredTensor): # pylint: disable=protected-access + if not isinstance(tensor, base_layer.DeferredTensor): # pylint: disable=protected-access raise TypeError('When eager execution is enabled, ' 'outputs must come from a call to ' 'a layer (called after ' @@ -230,7 +228,7 @@ class Network(base_layer.Layer): self._layers_by_depth = layers_by_depth # Create the node linking internal inputs to internal outputs. - tf_base_layers.Node( + base_layer.Node( outbound_layer=self, inbound_layers=[], node_indices=[], @@ -243,8 +241,8 @@ class Network(base_layer.Layer): for x in self.inputs: mask = x._keras_mask if hasattr(x, '_keras_mask') else None # pylint: disable=protected-access masks.append(mask) - mask_cache_key = (tf_layers_util.object_list_uid(self.inputs) + '_' + - tf_layers_util.object_list_uid(masks)) + mask_cache_key = (base_layer.object_list_uid(self.inputs) + '_' + + base_layer.object_list_uid(masks)) masks = [] for x in self.outputs: mask = x._keras_mask if hasattr(x, '_keras_mask') else None # pylint: disable=protected-access @@ -289,7 +287,7 @@ class Network(base_layer.Layer): self.built = False def __setattr__(self, name, value): - if isinstance(value, (tf_base_layers.Layer, Network)): + if isinstance(value, (base_layer.Layer, Network)): try: is_graph_network = self._is_graph_network except AttributeError: @@ -299,6 +297,10 @@ class Network(base_layer.Layer): if not is_graph_network: if value not in self._layers: self._layers.append(value) + if hasattr(value, '_use_resource_variables'): + # In subclassed models, legacy layers (tf.layers) must always use + # resource variables. + value._use_resource_variables = True if isinstance(value, checkpointable.CheckpointableBase): # Layer (and therefore Network/Model) inherit from CheckpointableBase # rather than Checkpointable, which means there is no Checkpointable @@ -387,8 +389,8 @@ class Network(base_layer.Layer): masks = [None for _ in range(len(inputs))] else: masks = generic_utils.to_list(mask) - cache_key = (tf_layers_util.object_list_uid(inputs) - + '_' + tf_layers_util.object_list_uid(masks)) + cache_key = (base_layer.object_list_uid(inputs) + + '_' + base_layer.object_list_uid(masks)) if cache_key in self._output_mask_cache: return self._output_mask_cache[cache_key] else: @@ -502,8 +504,7 @@ class Network(base_layer.Layer): relevant_inputs += inputs else: relevant_inputs.append(inputs) - reachable = tf_layers_util.get_reachable_from_inputs(relevant_inputs, - updates) + reachable = base_layer.get_reachable_from_inputs(relevant_inputs, updates) relevant_conditional_updates = [x for x in updates if x in reachable] unconditional_updates = [ x for x in updates if x._unconditional_update] # pylint: disable=protected-access @@ -540,8 +541,7 @@ class Network(base_layer.Layer): relevant_inputs += inputs else: relevant_inputs.append(inputs) - reachable = tf_layers_util.get_reachable_from_inputs(relevant_inputs, - losses) + reachable = base_layer.get_reachable_from_inputs(relevant_inputs, losses) relevant_conditional_losses = [x for x in losses if x in reachable] unconditional_losses = [ x for x in losses if x._unconditional_loss] # pylint: disable=protected-access @@ -623,8 +623,8 @@ class Network(base_layer.Layer): if not context.executing_eagerly(): # Try to retrieve cached outputs if the layer has already been called # on these exact inputs. - cache_key = (tf_layers_util.object_list_uid(inputs) - + '_' + tf_layers_util.object_list_uid(masks)) + cache_key = (base_layer.object_list_uid(inputs) + + '_' + base_layer.object_list_uid(masks)) if cache_key in self._output_tensor_cache: # Cache hit. return self._output_tensor_cache[cache_key] @@ -656,7 +656,7 @@ class Network(base_layer.Layer): ': model has ' + str(len(self._input_layers)) + ' tensor inputs.') - cache_key = tf_layers_util.object_list_uid(input_shapes) + cache_key = base_layer.object_list_uid(input_shapes) if cache_key not in self._output_shape_cache: # Cache miss. We have to run the network graph manually (recursive calls # to `compute_output_shape`). @@ -845,7 +845,7 @@ class Network(base_layer.Layer): for x in self.outputs: assert str(id(x)) in tensor_map, 'Could not compute output ' + str(x) tensor, mask = tensor_map[str(id(x))] - output_shapes.append(tf_layers_util.static_shape(x)) + output_shapes.append(base_layer.static_shape(x)) output_tensors.append(tensor) output_masks.append(mask) @@ -859,14 +859,14 @@ class Network(base_layer.Layer): if not context.executing_eagerly(): # Update cache; # keys are based on ids on input tensors and inputs masks. - cache_key = (tf_layers_util.object_list_uid(inputs) - + '_' + tf_layers_util.object_list_uid(masks)) + cache_key = (base_layer.object_list_uid(inputs) + + '_' + base_layer.object_list_uid(masks)) self._output_tensor_cache[cache_key] = output_tensors self._output_mask_cache[cache_key] = output_masks if output_shapes is not None: - input_shapes = [tf_layers_util.static_shape(x) for x in inputs] - cache_key = tf_layers_util.object_list_uid(input_shapes) + input_shapes = [base_layer.static_shape(x) for x in inputs] + cache_key = base_layer.object_list_uid(input_shapes) self._output_shape_cache[cache_key] = output_shapes return output_tensors, output_masks diff --git a/tensorflow/python/keras/_impl/keras/engine/saving_test.py b/tensorflow/python/keras/_impl/keras/engine/saving_test.py index dde0901204..3b1578cddf 100644 --- a/tensorflow/python/keras/_impl/keras/engine/saving_test.py +++ b/tensorflow/python/keras/_impl/keras/engine/saving_test.py @@ -422,7 +422,7 @@ class TestWholeModelSaving(test.TestCase): f = keras.layers.Dense(2, name='nested_model_dense_%d' % (i,))(f) # This layer name will make the `weights_name` # HDF5 attribute blow out of proportion. - f = keras.layers.Dense(2, name='nested_model_output' + ('x' * (2**15)))(f) + f = keras.layers.Dense(2, name='nested_model_output' + ('x' * (2**14)))(f) nested_model = keras.Model(inputs=[x], outputs=[f], name='nested_model') x = keras.Input(shape=(2,), name='outer_model_input') diff --git a/tensorflow/python/keras/_impl/keras/engine/sequential.py b/tensorflow/python/keras/_impl/keras/engine/sequential.py index 2ef99d5ab3..bd13ca6713 100644 --- a/tensorflow/python/keras/_impl/keras/engine/sequential.py +++ b/tensorflow/python/keras/_impl/keras/engine/sequential.py @@ -123,7 +123,7 @@ class Sequential(Model): multiple output tensors, or is already connected somewhere else (forbidden in `Sequential` models). """ - if not isinstance(layer, (base_layer.Layer, base_layer.TFBaseLayer)): + if not isinstance(layer, base_layer.Layer): raise TypeError('The added layer must be ' 'an instance of class Layer. ' 'Found: ' + str(layer)) diff --git a/tensorflow/python/keras/_impl/keras/engine/sequential_test.py b/tensorflow/python/keras/_impl/keras/engine/sequential_test.py index c9a47581df..8aba16aef3 100644 --- a/tensorflow/python/keras/_impl/keras/engine/sequential_test.py +++ b/tensorflow/python/keras/_impl/keras/engine/sequential_test.py @@ -151,6 +151,7 @@ class TestSequential(test.TestCase): with self.test_session(): model = keras.models.Sequential() model.add(keras.layers.BatchNormalization(input_shape=(4,))) + assert model.updates model.trainable = False assert not model.updates diff --git a/tensorflow/python/keras/_impl/keras/engine/topology_test.py b/tensorflow/python/keras/_impl/keras/engine/topology_test.py index 9ab4b6fdcf..49cc1cd3b3 100644 --- a/tensorflow/python/keras/_impl/keras/engine/topology_test.py +++ b/tensorflow/python/keras/_impl/keras/engine/topology_test.py @@ -25,7 +25,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import test_util from tensorflow.python.keras._impl import keras -from tensorflow.python.layers import base as tf_base_layers +from tensorflow.python.keras._impl.keras.engine import base_layer from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import state_ops @@ -52,11 +52,13 @@ class TopologyConstructionTest(test.TestCase): (1, 1), 'float32', trainable=False) - self.add_update(state_ops.assign_add(self.a, [[1.]])) + self.add_update(state_ops.assign_add(self.a, [[1.]], + name='unconditional_update')) self.built = True def call(self, inputs): - self.add_update(state_ops.assign_add(self.a, inputs), + self.add_update(state_ops.assign_add(self.b, inputs, + name='conditional_update'), inputs=True) return inputs + 1 @@ -97,10 +99,20 @@ class TopologyConstructionTest(test.TestCase): self.assertEqual(len(network.updates), 4) self.assertEqual(len(network.get_updates_for(None)), 2) - network.add_update(state_ops.assign_add(layer.a, x4), inputs=True) + network.add_update(state_ops.assign_add(layer.b, x4), inputs=True) self.assertEqual(len(network.updates), 5) self.assertEqual(len(network.get_updates_for(x4)), 2) + def test_get_updates_bn(self): + x1 = keras.Input(shape=(1,)) + layer = keras.layers.BatchNormalization() + _ = layer.apply(x1) + + print('BN updates', layer._updates) + self.assertEqual(len(layer.updates), 2) + self.assertEqual(len(layer.get_updates_for(x1)), 2) + self.assertEqual(len(layer.get_updates_for(None)), 0) + def test_get_losses(self): class MyLayer(keras.layers.Layer): @@ -875,25 +887,25 @@ class TopologyConstructionTest(test.TestCase): class DeferredModeTest(test.TestCase): def testDeferredTensorAttributes(self): - x = tf_base_layers._DeferredTensor(shape=(None, 2), - dtype='float32', - name='x') + x = base_layer.DeferredTensor(shape=(None, 2), + dtype='float32', + name='x') self.assertEqual(str(x), 'DeferredTensor(\'x\', shape=(?, 2), dtype=float32)') self.assertEqual(repr(x), - '<_DeferredTensor \'x\' shape=(?, 2) dtype=float32>') + '') @test_util.run_in_graph_and_eager_modes() def testSimpleNetworkBuilding(self): inputs = keras.engine.Input(shape=(32,)) if context.executing_eagerly(): - self.assertIsInstance(inputs, tf_base_layers._DeferredTensor) + self.assertIsInstance(inputs, base_layer.DeferredTensor) self.assertEqual(inputs.dtype.name, 'float32') self.assertEqual(inputs.shape.as_list(), [None, 32]) x = keras.layers.Dense(2)(inputs) if context.executing_eagerly(): - self.assertIsInstance(x, tf_base_layers._DeferredTensor) + self.assertIsInstance(x, base_layer.DeferredTensor) self.assertEqual(x.dtype.name, 'float32') self.assertEqual(x.shape.as_list(), [None, 2]) @@ -936,5 +948,34 @@ class DeferredModeTest(test.TestCase): self.assertEqual(outputs[0].shape.as_list(), [10, 16]) self.assertEqual(outputs[1].shape.as_list(), [10, 2]) + +class GraphUtilsTest(test.TestCase): + + def testGetReachableFromInputs(self): + + with self.test_session(): + pl_1 = array_ops.placeholder(shape=None, dtype='float32') + pl_2 = array_ops.placeholder(shape=None, dtype='float32') + pl_3 = array_ops.placeholder(shape=None, dtype='float32') + x_1 = pl_1 + pl_2 + x_2 = pl_2 * 2 + x_3 = pl_3 + 1 + x_4 = x_1 + x_2 + x_5 = x_3 * pl_1 + + self.assertEqual( + keras.engine.base_layer.get_reachable_from_inputs([pl_1]), + {pl_1, x_1, x_4, x_5, x_1.op, x_4.op, x_5.op}) + self.assertEqual( + keras.engine.base_layer.get_reachable_from_inputs([pl_1, pl_2]), + {pl_1, pl_2, x_1, x_2, x_4, x_5, x_1.op, x_2.op, x_4.op, x_5.op}) + self.assertEqual( + keras.engine.base_layer.get_reachable_from_inputs([pl_3]), + {pl_3, x_3, x_5, x_3.op, x_5.op}) + self.assertEqual( + keras.engine.base_layer.get_reachable_from_inputs([x_3]), + {x_3, x_5, x_5.op}) + + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/keras/_impl/keras/engine/training.py b/tensorflow/python/keras/_impl/keras/engine/training.py index 71de657da8..7c46743814 100644 --- a/tensorflow/python/keras/_impl/keras/engine/training.py +++ b/tensorflow/python/keras/_impl/keras/engine/training.py @@ -31,10 +31,10 @@ from tensorflow.python.keras._impl.keras.engine import training_arrays from tensorflow.python.keras._impl.keras.engine import training_eager from tensorflow.python.keras._impl.keras.engine import training_generator from tensorflow.python.keras._impl.keras.engine import training_utils +from tensorflow.python.keras._impl.keras.engine.base_layer import DeferredTensor from tensorflow.python.keras._impl.keras.engine.base_layer import Layer from tensorflow.python.keras._impl.keras.engine.network import Network from tensorflow.python.keras._impl.keras.utils.generic_utils import slice_arrays -from tensorflow.python.layers.base import _DeferredTensor from tensorflow.python.ops import array_ops from tensorflow.python.platform import tf_logging as logging from tensorflow.python.training import optimizer as tf_optimizer_module @@ -891,15 +891,6 @@ class Model(Network): else: self._symbolic_set_inputs(inputs, training=training) - def _set_scope(self, scope=None): - """Modify the Layer scope creation logic to create ResourceVariables.""" - super(Model, self)._set_scope(scope=scope) - # Subclassed Models create ResourceVariables by default. This makes it - # easier to use Models in an eager/graph agnostic way (since eager execution - # always uses ResourceVariables). - if not self._is_graph_network: - self._scope.set_use_resource(True) - def _eager_set_inputs(self, inputs): """Set model's input and output specs based on the input data received. @@ -933,11 +924,11 @@ class Model(Network): else: dummy_output_values = [dummy_output_values] self.outputs = [ - _DeferredTensor(shape=(None for _ in v.shape), - dtype=v.dtype) for v in dummy_output_values] + DeferredTensor(shape=(None for _ in v.shape), + dtype=v.dtype) for v in dummy_output_values] self.inputs = [ - _DeferredTensor(shape=(None for _ in v.shape), - dtype=v.dtype) for v in dummy_input_values] + DeferredTensor(shape=(None for _ in v.shape), + dtype=v.dtype) for v in dummy_input_values] self.input_names = [ 'input_%d' % (i + 1) for i in range(len(dummy_input_values))] self.output_names = [ diff --git a/tensorflow/python/keras/_impl/keras/initializers.py b/tensorflow/python/keras/_impl/keras/initializers.py index 300bed5e14..ecb71d00e2 100644 --- a/tensorflow/python/keras/_impl/keras/initializers.py +++ b/tensorflow/python/keras/_impl/keras/initializers.py @@ -201,6 +201,8 @@ def deserialize(config, custom_objects=None): @tf_export('keras.initializers.get') def get(identifier): + if identifier is None: + return None if isinstance(identifier, dict): return deserialize(identifier) elif isinstance(identifier, six.string_types): diff --git a/tensorflow/python/keras/_impl/keras/integration_test.py b/tensorflow/python/keras/_impl/keras/integration_test.py index 280f7ed1b1..c44808421f 100644 --- a/tensorflow/python/keras/_impl/keras/integration_test.py +++ b/tensorflow/python/keras/_impl/keras/integration_test.py @@ -29,16 +29,15 @@ from tensorflow.python.platform import test class KerasIntegrationTest(test.TestCase): - def test_vector_classification_declarative(self): + def test_vector_classification_sequential(self): with self.test_session(): np.random.seed(1337) - (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data( - train_samples=200, - test_samples=100, + (x_train, y_train), _ = testing_utils.get_test_data( + train_samples=100, + test_samples=0, input_shape=(10,), num_classes=2) y_train = keras.utils.to_categorical(y_train) - y_test = keras.utils.to_categorical(y_test) model = keras.models.Sequential([ keras.layers.Dense(16, @@ -48,23 +47,22 @@ class KerasIntegrationTest(test.TestCase): keras.layers.Dense(y_train.shape[-1], activation='softmax') ]) model.compile(loss='categorical_crossentropy', - optimizer='rmsprop', + optimizer=keras.optimizers.Adam(lr=0.1), metrics=['accuracy']) history = model.fit(x_train, y_train, epochs=10, batch_size=16, - validation_data=(x_test, y_test), + validation_data=(x_train, y_train), verbose=2) - self.assertGreater(history.history['val_acc'][-1], 0.85) + self.assertGreater(history.history['val_acc'][-1], 0.7) def test_vector_classification_functional(self): with self.test_session(): np.random.seed(1337) - (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data( - train_samples=200, - test_samples=100, - input_shape=(10,), + (x_train, y_train), _ = testing_utils.get_test_data( + train_samples=100, + test_samples=0, + input_shape=(20,), num_classes=2) y_train = keras.utils.to_categorical(y_train) - y_test = keras.utils.to_categorical(y_test) inputs = keras.layers.Input(shape=x_train.shape[1:]) x = keras.layers.Dense(16, activation='relu')(inputs) @@ -73,77 +71,78 @@ class KerasIntegrationTest(test.TestCase): model = keras.models.Model(inputs, outputs) model.compile(loss='categorical_crossentropy', - optimizer='rmsprop', + optimizer=keras.optimizers.Adam(lr=0.1), metrics=['accuracy']) history = model.fit(x_train, y_train, epochs=10, batch_size=16, - validation_data=(x_test, y_test), + validation_data=(x_train, y_train), verbose=2) - self.assertGreater(history.history['val_acc'][-1], 0.85) + self.assertGreater(history.history['val_acc'][-1], 0.7) - def test_temporal_classification_declarative(self): + def test_temporal_classification_sequential(self): with self.test_session(): - np.random.seed(1336) - (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data( - train_samples=200, - test_samples=100, - input_shape=(4, 8), + np.random.seed(1337) + (x_train, y_train), _ = testing_utils.get_test_data( + train_samples=100, + test_samples=0, + input_shape=(4, 10), num_classes=2) y_train = keras.utils.to_categorical(y_train) - y_test = keras.utils.to_categorical(y_test) model = keras.models.Sequential() model.add(keras.layers.LSTM(5, return_sequences=True, input_shape=x_train.shape[1:])) model.add(keras.layers.GRU(y_train.shape[-1], activation='softmax')) model.compile(loss='categorical_crossentropy', - optimizer='adam', + optimizer=keras.optimizers.Adam(lr=0.1), metrics=['accuracy']) history = model.fit(x_train, y_train, epochs=10, batch_size=16, - validation_data=(x_test, y_test), + validation_data=(x_train, y_train), verbose=2) - self.assertGreater(history.history['val_acc'][-1], 0.85) + self.assertGreater(history.history['val_acc'][-1], 0.7) - def test_image_classification_declarative(self): + def test_image_classification_sequential(self): with self.test_session(): np.random.seed(1337) - (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data( - train_samples=200, - test_samples=100, - input_shape=(8, 8, 3), + (x_train, y_train), _ = testing_utils.get_test_data( + train_samples=100, + test_samples=0, + input_shape=(12, 12, 3), num_classes=2) y_train = keras.utils.to_categorical(y_train) - y_test = keras.utils.to_categorical(y_test) model = keras.models.Sequential() model.add(keras.layers.Conv2D( - 8, 3, + 4, 3, + padding='same', activation='relu', input_shape=x_train.shape[1:])) - model.add(keras.layers.BatchNormalization()) model.add(keras.layers.Conv2D( 8, 3, padding='same', activation='relu')) - model.add(keras.layers.GlobalMaxPooling2D()) + model.add(keras.layers.Conv2D( + 16, 3, + padding='same', + activation='relu')) + model.add(keras.layers.Flatten()) model.add(keras.layers.Dense(y_train.shape[-1], activation='softmax')) model.compile(loss='categorical_crossentropy', - optimizer='adam', + optimizer=keras.optimizers.SGD(lr=0.01, momentum=0.8), metrics=['accuracy']) history = model.fit(x_train, y_train, epochs=10, batch_size=16, - validation_data=(x_test, y_test), + validation_data=(x_train, y_train), verbose=2) - self.assertGreater(history.history['val_acc'][-1], 0.85) + self.assertGreater(history.history['val_acc'][-1], 0.7) def test_video_classification_functional(self): with self.test_session(): np.random.seed(1337) - (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data( - train_samples=200, - test_samples=100, + (x_train, y_train), _ = testing_utils.get_test_data( + train_samples=100, + test_samples=0, input_shape=(4, 8, 8, 3), num_classes=3) y_train = keras.utils.to_categorical(y_train) - y_test = keras.utils.to_categorical(y_test) inputs = keras.layers.Input(shape=x_train.shape[1:]) x = keras.layers.TimeDistributed( @@ -159,22 +158,21 @@ class KerasIntegrationTest(test.TestCase): optimizer=keras.optimizers.SGD(lr=0.01, momentum=0.8), metrics=['accuracy']) history = model.fit(x_train, y_train, epochs=10, batch_size=16, - validation_data=(x_test, y_test), + validation_data=(x_train, y_train), verbose=2) - self.assertGreater(history.history['val_acc'][-1], 0.70) + self.assertGreater(history.history['val_acc'][-1], 0.7) def test_vector_classification_shared_sequential(self): # Test that Sequential models that feature internal updates # and internal losses can be shared. with self.test_session(): np.random.seed(1337) - (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data( - train_samples=200, - test_samples=100, + (x_train, y_train), _ = testing_utils.get_test_data( + train_samples=100, + test_samples=0, input_shape=(10,), num_classes=2) y_train = keras.utils.to_categorical(y_train) - y_test = keras.utils.to_categorical(y_test) base_model = keras.models.Sequential([ keras.layers.Dense(16, @@ -189,27 +187,26 @@ class KerasIntegrationTest(test.TestCase): y = keras.layers.Dense(y_train.shape[-1], activation='softmax')(y) model = keras.models.Model(x, y) model.compile(loss='categorical_crossentropy', - optimizer='rmsprop', + optimizer=keras.optimizers.Adam(lr=0.1), metrics=['accuracy']) self.assertEqual(len(model.losses), 2) self.assertEqual(len(model.updates), 2) history = model.fit(x_train, y_train, epochs=10, batch_size=16, - validation_data=(x_test, y_test), + validation_data=(x_train, y_train), verbose=2) - self.assertGreater(history.history['val_acc'][-1], 0.84) + self.assertGreater(history.history['val_acc'][-1], 0.7) def test_vector_classification_shared_model(self): # Test that functional models that feature internal updates # and internal losses can be shared. with self.test_session(): np.random.seed(1337) - (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data( - train_samples=200, - test_samples=100, + (x_train, y_train), _ = testing_utils.get_test_data( + train_samples=100, + test_samples=0, input_shape=(10,), num_classes=2) y_train = keras.utils.to_categorical(y_train) - y_test = keras.utils.to_categorical(y_test) inputs = keras.layers.Input(x_train.shape[1:]) x = keras.layers.Dense(16, @@ -225,12 +222,12 @@ class KerasIntegrationTest(test.TestCase): y = keras.layers.Dense(y_train.shape[-1], activation='softmax')(y) model = keras.models.Model(x, y) model.compile(loss='categorical_crossentropy', - optimizer='rmsprop', + optimizer=keras.optimizers.Adam(lr=0.1), metrics=['accuracy']) history = model.fit(x_train, y_train, epochs=10, batch_size=16, - validation_data=(x_test, y_test), + validation_data=(x_train, y_train), verbose=2) - self.assertGreater(history.history['val_acc'][-1], 0.85) + self.assertGreater(history.history['val_acc'][-1], 0.7) def test_embedding_with_clipnorm(self): with self.test_session(): @@ -242,9 +239,9 @@ class KerasIntegrationTest(test.TestCase): def test_using_tf_layers_in_keras_sequential_model(self): with self.test_session(): np.random.seed(1337) - (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data( - train_samples=200, - test_samples=100, + (x_train, y_train), _ = testing_utils.get_test_data( + train_samples=100, + test_samples=0, input_shape=(10,), num_classes=2) @@ -254,25 +251,23 @@ class KerasIntegrationTest(test.TestCase): model.summary() y_train = keras.utils.to_categorical(y_train) - y_test = keras.utils.to_categorical(y_test) model.compile(loss='categorical_crossentropy', - optimizer='adam', + optimizer=keras.optimizers.Adam(lr=0.1), metrics=['accuracy']) history = model.fit(x_train, y_train, epochs=10, batch_size=16, - validation_data=(x_test, y_test), + validation_data=(x_train, y_train), verbose=0) - self.assertGreater(history.history['val_acc'][-1], 0.85) + self.assertGreater(history.history['val_acc'][-1], 0.7) def test_using_tf_layers_in_keras_functional_model(self): with self.test_session(): np.random.seed(1337) - (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data( - train_samples=200, - test_samples=100, + (x_train, y_train), _ = testing_utils.get_test_data( + train_samples=100, + test_samples=0, input_shape=(10,), num_classes=2) y_train = keras.utils.to_categorical(y_train) - y_test = keras.utils.to_categorical(y_test) inputs = keras.Input(shape=(10,)) x = tf_core_layers.Dense(32, activation=nn.relu)(inputs) @@ -281,12 +276,12 @@ class KerasIntegrationTest(test.TestCase): model.summary() model.compile(loss='categorical_crossentropy', - optimizer='adam', + optimizer=keras.optimizers.Adam(lr=0.1), metrics=['accuracy']) history = model.fit(x_train, y_train, epochs=10, batch_size=16, - validation_data=(x_test, y_test), + validation_data=(x_train, y_train), verbose=0) - self.assertGreater(history.history['val_acc'][-1], 0.85) + self.assertGreater(history.history['val_acc'][-1], 0.7) if __name__ == '__main__': diff --git a/tensorflow/python/keras/_impl/keras/layers/convolutional.py b/tensorflow/python/keras/_impl/keras/layers/convolutional.py index 7cdebc6aa4..d202b6551d 100644 --- a/tensorflow/python/keras/_impl/keras/layers/convolutional.py +++ b/tensorflow/python/keras/_impl/keras/layers/convolutional.py @@ -19,9 +19,10 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.eager import context from tensorflow.python.framework import tensor_shape from tensorflow.python.keras._impl.keras import activations -from tensorflow.python.keras._impl.keras import backend as K +from tensorflow.python.keras._impl.keras import backend from tensorflow.python.keras._impl.keras import constraints from tensorflow.python.keras._impl.keras import initializers from tensorflow.python.keras._impl.keras import regularizers @@ -38,12 +39,232 @@ from tensorflow.python.keras._impl.keras.layers.pooling import MaxPooling2D from tensorflow.python.keras._impl.keras.layers.pooling import MaxPooling3D # pylint: enable=unused-import from tensorflow.python.keras._impl.keras.utils import conv_utils -from tensorflow.python.layers import convolutional as tf_convolutional_layers +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import nn_ops from tensorflow.python.util.tf_export import tf_export +class Conv(Layer): + """Abstract nD convolution layer (private, used as implementation base). + + This layer creates a convolution kernel that is convolved + (actually cross-correlated) with the layer input to produce a tensor of + outputs. If `use_bias` is True (and a `bias_initializer` is provided), + a bias vector is created and added to the outputs. Finally, if + `activation` is not `None`, it is applied to the outputs as well. + + Arguments: + rank: An integer, the rank of the convolution, e.g. "2" for 2D convolution. + filters: Integer, the dimensionality of the output space (i.e. the number + of filters in the convolution). + kernel_size: An integer or tuple/list of n integers, specifying the + length of the convolution window. + strides: An integer or tuple/list of n integers, + specifying the stride length of the convolution. + Specifying any stride value != 1 is incompatible with specifying + any `dilation_rate` value != 1. + padding: One of `"valid"` or `"same"` (case-insensitive). + data_format: A string, one of `channels_last` (default) or `channels_first`. + The ordering of the dimensions in the inputs. + `channels_last` corresponds to inputs with shape + `(batch, ..., channels)` while `channels_first` corresponds to + inputs with shape `(batch, channels, ...)`. + dilation_rate: An integer or tuple/list of n integers, specifying + the dilation rate to use for dilated convolution. + Currently, specifying any `dilation_rate` value != 1 is + incompatible with specifying any `strides` value != 1. + activation: Activation function. Set it to None to maintain a + linear activation. + use_bias: Boolean, whether the layer uses a bias. + kernel_initializer: An initializer for the convolution kernel. + bias_initializer: An initializer for the bias vector. If None, the default + initializer will be used. + kernel_regularizer: Optional regularizer for the convolution kernel. + bias_regularizer: Optional regularizer for the bias vector. + activity_regularizer: Optional regularizer function for the output. + kernel_constraint: Optional projection function to be applied to the + kernel after being updated by an `Optimizer` (e.g. used to implement + norm constraints or value constraints for layer weights). The function + must take as input the unprojected variable and must return the + projected variable (which must have the same shape). Constraints are + not safe to use when doing asynchronous distributed training. + bias_constraint: Optional projection function to be applied to the + bias after being updated by an `Optimizer`. + trainable: Boolean, if `True` also add variables to the graph collection + `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`). + name: A string, the name of the layer. + """ + + def __init__(self, rank, + filters, + kernel_size, + strides=1, + padding='valid', + data_format=None, + dilation_rate=1, + activation=None, + use_bias=True, + kernel_initializer='glorot_uniform', + bias_initializer='zeros', + kernel_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + trainable=True, + name=None, + **kwargs): + super(Conv, self).__init__( + trainable=trainable, + name=name, + activity_regularizer=regularizers.get(activity_regularizer), + **kwargs) + self.rank = rank + self.filters = filters + self.kernel_size = conv_utils.normalize_tuple( + kernel_size, rank, 'kernel_size') + self.strides = conv_utils.normalize_tuple(strides, rank, 'strides') + self.padding = conv_utils.normalize_padding(padding) + self.data_format = conv_utils.normalize_data_format(data_format) + self.dilation_rate = conv_utils.normalize_tuple( + dilation_rate, rank, 'dilation_rate') + self.activation = activations.get(activation) + self.use_bias = use_bias + self.kernel_initializer = initializers.get(kernel_initializer) + self.bias_initializer = initializers.get(bias_initializer) + self.kernel_regularizer = regularizers.get(kernel_regularizer) + self.bias_regularizer = regularizers.get(bias_regularizer) + self.kernel_constraint = constraints.get(kernel_constraint) + self.bias_constraint = constraints.get(bias_constraint) + self.input_spec = InputSpec(ndim=self.rank + 2) + + def build(self, input_shape): + input_shape = tensor_shape.TensorShape(input_shape) + if self.data_format == 'channels_first': + channel_axis = 1 + else: + channel_axis = -1 + if input_shape[channel_axis].value is None: + raise ValueError('The channel dimension of the inputs ' + 'should be defined. Found `None`.') + input_dim = input_shape[channel_axis].value + kernel_shape = self.kernel_size + (input_dim, self.filters) + + self.kernel = self.add_variable(name='kernel', + shape=kernel_shape, + initializer=self.kernel_initializer, + regularizer=self.kernel_regularizer, + constraint=self.kernel_constraint, + trainable=True, + dtype=self.dtype) + if self.use_bias: + self.bias = self.add_variable(name='bias', + shape=(self.filters,), + initializer=self.bias_initializer, + regularizer=self.bias_regularizer, + constraint=self.bias_constraint, + trainable=True, + dtype=self.dtype) + else: + self.bias = None + self.input_spec = InputSpec(ndim=self.rank + 2, + axes={channel_axis: input_dim}) + self._convolution_op = nn_ops.Convolution( + input_shape, + filter_shape=self.kernel.get_shape(), + dilation_rate=self.dilation_rate, + strides=self.strides, + padding=self.padding.upper(), + data_format=conv_utils.convert_data_format(self.data_format, + self.rank + 2)) + self.built = True + + def call(self, inputs): + outputs = self._convolution_op(inputs, self.kernel) + + if self.use_bias: + if self.data_format == 'channels_first': + if self.rank == 1: + # nn.bias_add does not accept a 1D input tensor. + bias = array_ops.reshape(self.bias, (1, self.filters, 1)) + outputs += bias + if self.rank == 2: + outputs = nn.bias_add(outputs, self.bias, data_format='NCHW') + if self.rank == 3: + # As of Mar 2017, direct addition is significantly slower than + # bias_add when computing gradients. To use bias_add, we collapse Z + # and Y into a single dimension to obtain a 4D input tensor. + outputs_shape = outputs.shape.as_list() + if outputs_shape[0] is None: + outputs_shape[0] = -1 + outputs_4d = array_ops.reshape(outputs, + [outputs_shape[0], outputs_shape[1], + outputs_shape[2] * outputs_shape[3], + outputs_shape[4]]) + outputs_4d = nn.bias_add(outputs_4d, self.bias, data_format='NCHW') + outputs = array_ops.reshape(outputs_4d, outputs_shape) + else: + outputs = nn.bias_add(outputs, self.bias, data_format='NHWC') + + if self.activation is not None: + return self.activation(outputs) + return outputs + + def compute_output_shape(self, input_shape): + input_shape = tensor_shape.TensorShape(input_shape).as_list() + if self.data_format == 'channels_last': + space = input_shape[1:-1] + new_space = [] + for i in range(len(space)): + new_dim = conv_utils.conv_output_length( + space[i], + self.kernel_size[i], + padding=self.padding, + stride=self.strides[i], + dilation=self.dilation_rate[i]) + new_space.append(new_dim) + return tensor_shape.TensorShape([input_shape[0]] + new_space + + [self.filters]) + else: + space = input_shape[2:] + new_space = [] + for i in range(len(space)): + new_dim = conv_utils.conv_output_length( + space[i], + self.kernel_size[i], + padding=self.padding, + stride=self.strides[i], + dilation=self.dilation_rate[i]) + new_space.append(new_dim) + return tensor_shape.TensorShape([input_shape[0], self.filters] + + new_space) + + def get_config(self): + config = { + 'filters': self.filters, + 'kernel_size': self.kernel_size, + 'strides': self.strides, + 'padding': self.padding, + 'data_format': self.data_format, + 'dilation_rate': self.dilation_rate, + 'activation': activations.serialize(self.activation), + 'use_bias': self.use_bias, + 'kernel_initializer': initializers.serialize(self.kernel_initializer), + 'bias_initializer': initializers.serialize(self.bias_initializer), + 'kernel_regularizer': regularizers.serialize(self.kernel_regularizer), + 'bias_regularizer': regularizers.serialize(self.bias_regularizer), + 'activity_regularizer': + regularizers.serialize(self.activity_regularizer), + 'kernel_constraint': constraints.serialize(self.kernel_constraint), + 'bias_constraint': constraints.serialize(self.bias_constraint) + } + base_config = super(Conv, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + @tf_export('keras.layers.Conv1D', 'keras.layers.Convolution1D') -class Conv1D(tf_convolutional_layers.Conv1D, Layer): +class Conv1D(Conv): """1D convolution layer (e.g. temporal convolution). This layer creates a convolution kernel that is convolved @@ -74,6 +295,8 @@ class Conv1D(tf_convolutional_layers.Conv1D, Layer): where the model should not violate the temporal order. See [WaveNet: A Generative Model for Raw Audio, section 2.1](https://arxiv.org/abs/1609.03499). + data_format: A string, + one of `channels_last` (default) or `channels_first`. dilation_rate: an integer or tuple/list of a single integer, specifying the dilation rate to use for dilated convolution. Currently, specifying any `dilation_rate` value != 1 is @@ -105,6 +328,7 @@ class Conv1D(tf_convolutional_layers.Conv1D, Layer): kernel_size, strides=1, padding='valid', + data_format='channels_last', dilation_rate=1, activation=None, use_bias=True, @@ -117,11 +341,12 @@ class Conv1D(tf_convolutional_layers.Conv1D, Layer): bias_constraint=None, **kwargs): super(Conv1D, self).__init__( + rank=1, filters=filters, kernel_size=kernel_size, strides=strides, padding=padding, - data_format='channels_last', + data_format=data_format, dilation_rate=dilation_rate, activation=activations.get(activation), use_bias=use_bias, @@ -134,30 +359,9 @@ class Conv1D(tf_convolutional_layers.Conv1D, Layer): bias_constraint=constraints.get(bias_constraint), **kwargs) - def get_config(self): - config = { - 'filters': self.filters, - 'kernel_size': self.kernel_size, - 'strides': self.strides, - 'padding': self.padding, - 'dilation_rate': self.dilation_rate, - 'activation': activations.serialize(self.activation), - 'use_bias': self.use_bias, - 'kernel_initializer': initializers.serialize(self.kernel_initializer), - 'bias_initializer': initializers.serialize(self.bias_initializer), - 'kernel_regularizer': regularizers.serialize(self.kernel_regularizer), - 'bias_regularizer': regularizers.serialize(self.bias_regularizer), - 'activity_regularizer': - regularizers.serialize(self.activity_regularizer), - 'kernel_constraint': constraints.serialize(self.kernel_constraint), - 'bias_constraint': constraints.serialize(self.bias_constraint) - } - base_config = super(Conv1D, self).get_config() - return dict(list(base_config.items()) + list(config.items())) - @tf_export('keras.layers.Conv2D', 'keras.layers.Convolution2D') -class Conv2D(tf_convolutional_layers.Conv2D, Layer): +class Conv2D(Conv): """2D convolution layer (e.g. spatial convolution over images). This layer creates a convolution kernel that is convolved @@ -247,9 +451,8 @@ class Conv2D(tf_convolutional_layers.Conv2D, Layer): kernel_constraint=None, bias_constraint=None, **kwargs): - if data_format is None: - data_format = K.image_data_format() super(Conv2D, self).__init__( + rank=2, filters=filters, kernel_size=kernel_size, strides=strides, @@ -267,31 +470,9 @@ class Conv2D(tf_convolutional_layers.Conv2D, Layer): bias_constraint=constraints.get(bias_constraint), **kwargs) - def get_config(self): - config = { - 'filters': self.filters, - 'kernel_size': self.kernel_size, - 'strides': self.strides, - 'padding': self.padding, - 'data_format': self.data_format, - 'dilation_rate': self.dilation_rate, - 'activation': activations.serialize(self.activation), - 'use_bias': self.use_bias, - 'kernel_initializer': initializers.serialize(self.kernel_initializer), - 'bias_initializer': initializers.serialize(self.bias_initializer), - 'kernel_regularizer': regularizers.serialize(self.kernel_regularizer), - 'bias_regularizer': regularizers.serialize(self.bias_regularizer), - 'activity_regularizer': - regularizers.serialize(self.activity_regularizer), - 'kernel_constraint': constraints.serialize(self.kernel_constraint), - 'bias_constraint': constraints.serialize(self.bias_constraint) - } - base_config = super(Conv2D, self).get_config() - return dict(list(base_config.items()) + list(config.items())) - @tf_export('keras.layers.Conv3D', 'keras.layers.Convolution3D') -class Conv3D(tf_convolutional_layers.Conv3D, Layer): +class Conv3D(Conv): """3D convolution layer (e.g. spatial convolution over volumes). This layer creates a convolution kernel that is convolved @@ -388,9 +569,8 @@ class Conv3D(tf_convolutional_layers.Conv3D, Layer): kernel_constraint=None, bias_constraint=None, **kwargs): - if data_format is None: - data_format = K.image_data_format() super(Conv3D, self).__init__( + rank=3, filters=filters, kernel_size=kernel_size, strides=strides, @@ -408,32 +588,10 @@ class Conv3D(tf_convolutional_layers.Conv3D, Layer): bias_constraint=constraints.get(bias_constraint), **kwargs) - def get_config(self): - config = { - 'filters': self.filters, - 'kernel_size': self.kernel_size, - 'strides': self.strides, - 'padding': self.padding, - 'data_format': self.data_format, - 'dilation_rate': self.dilation_rate, - 'activation': activations.serialize(self.activation), - 'use_bias': self.use_bias, - 'kernel_initializer': initializers.serialize(self.kernel_initializer), - 'bias_initializer': initializers.serialize(self.bias_initializer), - 'kernel_regularizer': regularizers.serialize(self.kernel_regularizer), - 'bias_regularizer': regularizers.serialize(self.bias_regularizer), - 'activity_regularizer': - regularizers.serialize(self.activity_regularizer), - 'kernel_constraint': constraints.serialize(self.kernel_constraint), - 'bias_constraint': constraints.serialize(self.bias_constraint) - } - base_config = super(Conv3D, self).get_config() - return dict(list(base_config.items()) + list(config.items())) - @tf_export('keras.layers.Conv2DTranspose', 'keras.layers.Convolution2DTranspose') -class Conv2DTranspose(tf_convolutional_layers.Conv2DTranspose, Layer): +class Conv2DTranspose(Conv2D): """Transposed convolution layer (sometimes called Deconvolution). The need for transposed convolutions generally arises @@ -529,8 +687,6 @@ class Conv2DTranspose(tf_convolutional_layers.Conv2DTranspose, Layer): kernel_constraint=None, bias_constraint=None, **kwargs): - if data_format is None: - data_format = K.image_data_format() super(Conv2DTranspose, self).__init__( filters=filters, kernel_size=kernel_size, @@ -548,31 +704,123 @@ class Conv2DTranspose(tf_convolutional_layers.Conv2DTranspose, Layer): bias_constraint=constraints.get(bias_constraint), **kwargs) - def get_config(self): - config = { - 'filters': self.filters, - 'kernel_size': self.kernel_size, - 'strides': self.strides, - 'padding': self.padding, - 'data_format': self.data_format, - 'activation': activations.serialize(self.activation), - 'use_bias': self.use_bias, - 'kernel_initializer': initializers.serialize(self.kernel_initializer), - 'bias_initializer': initializers.serialize(self.bias_initializer), - 'kernel_regularizer': regularizers.serialize(self.kernel_regularizer), - 'bias_regularizer': regularizers.serialize(self.bias_regularizer), - 'activity_regularizer': - regularizers.serialize(self.activity_regularizer), - 'kernel_constraint': constraints.serialize(self.kernel_constraint), - 'bias_constraint': constraints.serialize(self.bias_constraint) - } - base_config = super(Conv2DTranspose, self).get_config() - return dict(list(base_config.items()) + list(config.items())) + def build(self, input_shape): + if len(input_shape) != 4: + raise ValueError('Inputs should have rank 4. Received input shape: ' + + str(input_shape)) + if self.data_format == 'channels_first': + channel_axis = 1 + else: + channel_axis = -1 + if input_shape[channel_axis] is None: + raise ValueError('The channel dimension of the inputs ' + 'should be defined. Found `None`.') + input_dim = input_shape[channel_axis] + self.input_spec = InputSpec(ndim=4, axes={channel_axis: input_dim}) + kernel_shape = self.kernel_size + (self.filters, input_dim) + + self.kernel = self.add_variable(name='kernel', + shape=kernel_shape, + initializer=self.kernel_initializer, + regularizer=self.kernel_regularizer, + constraint=self.kernel_constraint, + trainable=True, + dtype=self.dtype) + if self.use_bias: + self.bias = self.add_variable(name='bias', + shape=(self.filters,), + initializer=self.bias_initializer, + regularizer=self.bias_regularizer, + constraint=self.bias_constraint, + trainable=True, + dtype=self.dtype) + else: + self.bias = None + self.built = True + + def call(self, inputs): + inputs_shape = array_ops.shape(inputs) + batch_size = inputs_shape[0] + if self.data_format == 'channels_first': + c_axis, h_axis, w_axis = 1, 2, 3 + else: + c_axis, h_axis, w_axis = 3, 1, 2 + + height, width = inputs_shape[h_axis], inputs_shape[w_axis] + kernel_h, kernel_w = self.kernel_size + stride_h, stride_w = self.strides + + # Infer the dynamic output shape: + out_height = conv_utils.deconv_output_length(height, + kernel_h, + self.padding, + stride_h) + out_width = conv_utils.deconv_output_length(width, + kernel_w, + self.padding, + stride_w) + if self.data_format == 'channels_first': + output_shape = (batch_size, self.filters, out_height, out_width) + strides = (1, 1, stride_h, stride_w) + else: + output_shape = (batch_size, out_height, out_width, self.filters) + strides = (1, stride_h, stride_w, 1) + + output_shape_tensor = array_ops.stack(output_shape) + outputs = nn.conv2d_transpose( + inputs, + self.kernel, + output_shape_tensor, + strides, + padding=self.padding.upper(), + data_format=conv_utils.convert_data_format(self.data_format, ndim=4)) + + if not context.executing_eagerly(): + # Infer the static output shape: + out_shape = inputs.get_shape().as_list() + out_shape[c_axis] = self.filters + out_shape[h_axis] = conv_utils.deconv_output_length(out_shape[h_axis], + kernel_h, + self.padding, + stride_h) + out_shape[w_axis] = conv_utils.deconv_output_length(out_shape[w_axis], + kernel_w, + self.padding, + stride_w) + outputs.set_shape(out_shape) + + if self.use_bias: + outputs = nn.bias_add( + outputs, + self.bias, + data_format=conv_utils.convert_data_format(self.data_format, ndim=4)) + + if self.activation is not None: + return self.activation(outputs) + return outputs + + def compute_output_shape(self, input_shape): + input_shape = tensor_shape.TensorShape(input_shape).as_list() + output_shape = list(input_shape) + if self.data_format == 'channels_first': + c_axis, h_axis, w_axis = 1, 2, 3 + else: + c_axis, h_axis, w_axis = 3, 1, 2 + + kernel_h, kernel_w = self.kernel_size + stride_h, stride_w = self.strides + + output_shape[c_axis] = self.filters + output_shape[h_axis] = conv_utils.deconv_output_length( + output_shape[h_axis], kernel_h, self.padding, stride_h) + output_shape[w_axis] = conv_utils.deconv_output_length( + output_shape[w_axis], kernel_w, self.padding, stride_w) + return tensor_shape.TensorShape(output_shape) @tf_export('keras.layers.Conv3DTranspose', 'keras.layers.Convolution3DTranspose') -class Conv3DTranspose(tf_convolutional_layers.Conv3DTranspose, Layer): +class Conv3DTranspose(Conv3D): """Transposed convolution layer (sometimes called Deconvolution). The need for transposed convolutions generally arises @@ -679,8 +927,6 @@ class Conv3DTranspose(tf_convolutional_layers.Conv3DTranspose, Layer): kernel_constraint=None, bias_constraint=None, **kwargs): - if data_format is None: - data_format = K.image_data_format() super(Conv3DTranspose, self).__init__( filters=filters, kernel_size=kernel_size, @@ -698,6 +944,313 @@ class Conv3DTranspose(tf_convolutional_layers.Conv3DTranspose, Layer): bias_constraint=constraints.get(bias_constraint), **kwargs) + def build(self, input_shape): + if len(input_shape) != 5: + raise ValueError('Inputs should have rank 5, received input shape:', + str(input_shape)) + if self.data_format == 'channels_first': + channel_axis = 1 + else: + channel_axis = -1 + if input_shape[channel_axis] is None: + raise ValueError('The channel dimension of the inputs ' + 'should be defined, found None: ' + str(input_shape)) + input_dim = input_shape[channel_axis] + kernel_shape = self.kernel_size + (self.filters, input_dim) + self.input_spec = InputSpec(ndim=5, axes={channel_axis: input_dim}) + + self.kernel = self.add_variable( + 'kernel', + shape=kernel_shape, + initializer=self.kernel_initializer, + regularizer=self.kernel_regularizer, + constraint=self.kernel_constraint, + trainable=True, + dtype=self.dtype) + if self.use_bias: + self.bias = self.add_variable( + 'bias', + shape=(self.filters,), + initializer=self.bias_initializer, + regularizer=self.bias_regularizer, + constraint=self.bias_constraint, + trainable=True, + dtype=self.dtype) + else: + self.bias = None + self.built = True + + def call(self, inputs): + inputs_shape = array_ops.shape(inputs) + batch_size = inputs_shape[0] + if self.data_format == 'channels_first': + c_axis, d_axis, h_axis, w_axis = 1, 2, 3, 4 + else: + c_axis, d_axis, h_axis, w_axis = 4, 1, 2, 3 + + self.input_spec = InputSpec(ndim=5, axes={c_axis: inputs_shape[c_axis]}) + + depth = inputs_shape[d_axis] + height = inputs_shape[h_axis] + width = inputs_shape[w_axis] + + kernel_d, kernel_h, kernel_w = self.kernel_size + stride_d, stride_h, stride_w = self.strides + + # Infer the dynamic output shape: + out_depth = conv_utils.deconv_output_length(depth, + kernel_d, + self.padding, + stride_d) + out_height = conv_utils.deconv_output_length(height, + kernel_h, + self.padding, + stride_h) + out_width = conv_utils.deconv_output_length(width, + kernel_w, + self.padding, + stride_w) + if self.data_format == 'channels_first': + output_shape = (batch_size, self.filters, out_depth, out_height, + out_width) + strides = (1, 1, stride_d, stride_h, stride_w) + else: + output_shape = (batch_size, out_depth, out_height, out_width, + self.filters) + strides = (1, stride_d, stride_h, stride_w, 1) + + output_shape_tensor = array_ops.stack(output_shape) + outputs = nn.conv3d_transpose( + inputs, + self.kernel, + output_shape_tensor, + strides, + data_format=conv_utils.convert_data_format(self.data_format, ndim=5), + padding=self.padding.upper()) + + if not context.executing_eagerly(): + # Infer the static output shape: + out_shape = inputs.get_shape().as_list() + out_shape[c_axis] = self.filters + out_shape[d_axis] = conv_utils.deconv_output_length(out_shape[d_axis], + kernel_d, + self.padding, + stride_d) + out_shape[h_axis] = conv_utils.deconv_output_length(out_shape[h_axis], + kernel_h, + self.padding, + stride_h) + out_shape[w_axis] = conv_utils.deconv_output_length(out_shape[w_axis], + kernel_w, + self.padding, + stride_w) + outputs.set_shape(out_shape) + + if self.use_bias: + outputs_shape = outputs.shape.as_list() + if outputs_shape[0] is None: + outputs_shape[0] = -1 + if self.data_format == 'channels_first': + outputs_4d = array_ops.reshape(outputs, [ + outputs_shape[0], outputs_shape[1], + outputs_shape[2] * outputs_shape[3], outputs_shape[4] + ]) + else: + outputs_4d = array_ops.reshape(outputs, [ + outputs_shape[0], outputs_shape[1] * outputs_shape[2], + outputs_shape[3], outputs_shape[4] + ]) + outputs_4d = nn.bias_add( + outputs_4d, + self.bias, + data_format=conv_utils.convert_data_format(self.data_format, ndim=4)) + outputs = array_ops.reshape(outputs_4d, outputs_shape) + + if self.activation is not None: + return self.activation(outputs) + return outputs + + def compute_output_shape(self, input_shape): + input_shape = tensor_shape.TensorShape(input_shape).as_list() + output_shape = list(input_shape) + if self.data_format == 'channels_first': + c_axis, d_axis, h_axis, w_axis = 1, 2, 3, 4 + else: + c_axis, d_axis, h_axis, w_axis = 4, 1, 2, 3 + + kernel_d, kernel_h, kernel_w = self.kernel_size + stride_d, stride_h, stride_w = self.strides + + output_shape[c_axis] = self.filters + output_shape[d_axis] = conv_utils.deconv_output_length( + output_shape[d_axis], kernel_d, self.padding, stride_d) + output_shape[h_axis] = conv_utils.deconv_output_length( + output_shape[h_axis], kernel_h, self.padding, stride_h) + output_shape[w_axis] = conv_utils.deconv_output_length( + output_shape[w_axis], kernel_w, self.padding, stride_w) + return tensor_shape.TensorShape(output_shape) + + +class SeparableConv(Conv): + """Abstract base layer for separable nD convolution. + + This layer performs a depthwise convolution that acts separately on + channels, followed by a pointwise convolution that mixes channels. + If `use_bias` is True and a bias initializer is provided, + it adds a bias vector to the output. + It then optionally applies an activation function to produce the final output. + + Arguments: + rank: An integer, the rank of the convolution, e.g. "2" for 2D convolution. + filters: Integer, the dimensionality of the output space (i.e. the number + of filters in the convolution). + kernel_size: A tuple or list of integers specifying the spatial + dimensions of the filters. Can be a single integer to specify the same + value for all spatial dimensions. + strides: A tuple or list of integers specifying the strides + of the convolution. Can be a single integer to specify the same value for + all spatial dimensions. + Specifying any `stride` value != 1 is incompatible with specifying + any `dilation_rate` value != 1. + padding: One of `"valid"` or `"same"` (case-insensitive). + data_format: A string, one of `channels_last` (default) or `channels_first`. + The ordering of the dimensions in the inputs. + `channels_last` corresponds to inputs with shape + `(batch, ..., channels)` while `channels_first` corresponds to + inputs with shape `(batch, channels, ...)`. + dilation_rate: An integer or tuple/list of 2 integers, specifying + the dilation rate to use for dilated convolution. + Can be a single integer to specify the same value for + all spatial dimensions. + Currently, specifying any `dilation_rate` value != 1 is + incompatible with specifying any stride value != 1. + depth_multiplier: The number of depthwise convolution output channels for + each input channel. The total number of depthwise convolution output + channels will be equal to `num_filters_in * depth_multiplier`. + activation: Activation function. Set it to None to maintain a + linear activation. + use_bias: Boolean, whether the layer uses a bias. + depthwise_initializer: An initializer for the depthwise convolution kernel. + pointwise_initializer: An initializer for the pointwise convolution kernel. + bias_initializer: An initializer for the bias vector. If None, the default + initializer will be used. + depthwise_regularizer: Optional regularizer for the depthwise + convolution kernel. + pointwise_regularizer: Optional regularizer for the pointwise + convolution kernel. + bias_regularizer: Optional regularizer for the bias vector. + activity_regularizer: Optional regularizer function for the output. + depthwise_constraint: Optional projection function to be applied to the + depthwise kernel after being updated by an `Optimizer` (e.g. used for + norm constraints or value constraints for layer weights). The function + must take as input the unprojected variable and must return the + projected variable (which must have the same shape). Constraints are + not safe to use when doing asynchronous distributed training. + pointwise_constraint: Optional projection function to be applied to the + pointwise kernel after being updated by an `Optimizer`. + bias_constraint: Optional projection function to be applied to the + bias after being updated by an `Optimizer`. + trainable: Boolean, if `True` also add variables to the graph collection + `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`). + name: A string, the name of the layer. + """ + + def __init__(self, + rank, + filters, + kernel_size, + strides=1, + padding='valid', + data_format=None, + dilation_rate=1, + depth_multiplier=1, + activation=None, + use_bias=True, + depthwise_initializer='glorot_uniform', + pointwise_initializer='glorot_uniform', + bias_initializer='zeros', + depthwise_regularizer=None, + pointwise_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + depthwise_constraint=None, + pointwise_constraint=None, + bias_constraint=None, + trainable=True, + name=None, + **kwargs): + super(SeparableConv, self).__init__( + rank=rank, + filters=filters, + kernel_size=kernel_size, + strides=strides, + padding=padding, + data_format=data_format, + dilation_rate=dilation_rate, + activation=activations.get(activation), + use_bias=use_bias, + bias_regularizer=regularizers.get(bias_regularizer), + activity_regularizer=regularizers.get(activity_regularizer), + bias_constraint=bias_constraint, + trainable=trainable, + name=name, + **kwargs) + self.depth_multiplier = depth_multiplier + self.depthwise_initializer = initializers.get(depthwise_initializer) + self.pointwise_initializer = initializers.get(pointwise_initializer) + self.depthwise_regularizer = regularizers.get(depthwise_regularizer) + self.pointwise_regularizer = regularizers.get(pointwise_regularizer) + self.depthwise_constraint = constraints.get(depthwise_constraint) + self.pointwise_constraint = constraints.get(pointwise_constraint) + + def build(self, input_shape): + input_shape = tensor_shape.TensorShape(input_shape) + if self.data_format == 'channels_first': + channel_axis = 1 + else: + channel_axis = -1 + if input_shape[channel_axis].value is None: + raise ValueError('The channel dimension of the inputs ' + 'should be defined. Found `None`.') + input_dim = input_shape[channel_axis].value + self.input_spec = InputSpec(ndim=self.rank + 2, + axes={channel_axis: input_dim}) + depthwise_kernel_shape = self.kernel_size + (input_dim, + self.depth_multiplier) + pointwise_kernel_shape = ( + 1,) * self.rank + (self.depth_multiplier * input_dim, self.filters) + + self.depthwise_kernel = self.add_variable( + name='depthwise_kernel', + shape=depthwise_kernel_shape, + initializer=self.depthwise_initializer, + regularizer=self.depthwise_regularizer, + constraint=self.depthwise_constraint, + trainable=True, + dtype=self.dtype) + self.pointwise_kernel = self.add_variable( + name='pointwise_kernel', + shape=pointwise_kernel_shape, + initializer=self.pointwise_initializer, + regularizer=self.pointwise_regularizer, + constraint=self.pointwise_constraint, + trainable=True, + dtype=self.dtype) + if self.use_bias: + self.bias = self.add_variable(name='bias', + shape=(self.filters,), + initializer=self.bias_initializer, + regularizer=self.bias_regularizer, + constraint=self.bias_constraint, + trainable=True, + dtype=self.dtype) + else: + self.bias = None + self.built = True + + def call(self, inputs): + raise NotImplementedError + def get_config(self): config = { 'filters': self.filters, @@ -705,24 +1258,34 @@ class Conv3DTranspose(tf_convolutional_layers.Conv3DTranspose, Layer): 'strides': self.strides, 'padding': self.padding, 'data_format': self.data_format, + 'dilation_rate': self.dilation_rate, 'activation': activations.serialize(self.activation), 'use_bias': self.use_bias, - 'kernel_initializer': initializers.serialize(self.kernel_initializer), + 'depthwise_initializer': + initializers.serialize(self.depthwise_initializer), + 'pointwise_initializer': + initializers.serialize(self.pointwise_initializer), 'bias_initializer': initializers.serialize(self.bias_initializer), - 'kernel_regularizer': regularizers.serialize(self.kernel_regularizer), + 'depthwise_regularizer': + regularizers.serialize(self.depthwise_regularizer), + 'pointwise_regularizer': + regularizers.serialize(self.pointwise_regularizer), 'bias_regularizer': regularizers.serialize(self.bias_regularizer), 'activity_regularizer': regularizers.serialize(self.activity_regularizer), - 'kernel_constraint': constraints.serialize(self.kernel_constraint), + 'depthwise_constraint': + constraints.serialize(self.depthwise_constraint), + 'pointwise_constraint': + constraints.serialize(self.pointwise_constraint), 'bias_constraint': constraints.serialize(self.bias_constraint) } - base_config = super(Conv3DTranspose, self).get_config() + base_config = super(SeparableConv, self).get_config() return dict(list(base_config.items()) + list(config.items())) @tf_export('keras.layers.SeparableConv1D', 'keras.layers.SeparableConvolution1D') -class SeparableConv1D(tf_convolutional_layers.SeparableConv1D, Layer): +class SeparableConv1D(SeparableConv): """Depthwise separable 1D convolution. This layer performs a depthwise convolution that acts separately on @@ -802,15 +1365,15 @@ class SeparableConv1D(tf_convolutional_layers.SeparableConv1D, Layer): pointwise_constraint=None, bias_constraint=None, **kwargs): - if data_format is None: - data_format = K.image_data_format() super(SeparableConv1D, self).__init__( + rank=1, filters=filters, kernel_size=kernel_size, strides=strides, padding=padding, data_format=data_format, dilation_rate=dilation_rate, + depth_multiplier=depth_multiplier, activation=activations.get(activation), use_bias=use_bias, depthwise_initializer=initializers.get(depthwise_initializer), @@ -825,44 +1388,46 @@ class SeparableConv1D(tf_convolutional_layers.SeparableConv1D, Layer): bias_constraint=constraints.get(bias_constraint), **kwargs) - def get_config(self): - config = { - 'filters': self.filters, - 'kernel_size': self.kernel_size, - 'strides': self.strides, - 'padding': self.padding, - 'data_format': self.data_format, - 'dilation_rate': self.dilation_rate, - 'activation': activations.serialize(self.activation), - 'use_bias': self.use_bias, - 'depthwise_initializer': - initializers.serialize(self.depthwise_initializer), - 'pointwise_initializer': - initializers.serialize(self.pointwise_initializer), - 'bias_initializer': - initializers.serialize(self.bias_initializer), - 'depthwise_regularizer': - regularizers.serialize(self.depthwise_regularizer), - 'pointwise_regularizer': - regularizers.serialize(self.pointwise_regularizer), - 'bias_regularizer': - regularizers.serialize(self.bias_regularizer), - 'activity_regularizer': - regularizers.serialize(self.activity_regularizer), - 'depthwise_constraint': - constraints.serialize(self.depthwise_constraint), - 'pointwise_constraint': - constraints.serialize(self.pointwise_constraint), - 'bias_constraint': - constraints.serialize(self.bias_constraint) - } - base_config = super(SeparableConv1D, self).get_config() - return dict(list(base_config.items()) + list(config.items())) + def call(self, inputs): + if self.data_format == 'channels_last': + strides = (1,) + self.strides * 2 + (1,) + spatial_start_dim = 1 + else: + strides = (1, 1) + self.strides * 2 + spatial_start_dim = 2 + + # Explicitly broadcast inputs and kernels to 4D. + # TODO(fchollet): refactor when a native separable_conv1d op is available. + inputs = array_ops.expand_dims(inputs, spatial_start_dim) + depthwise_kernel = array_ops.expand_dims(self.depthwise_kernel, 0) + pointwise_kernel = array_ops.expand_dims(self.pointwise_kernel, 0) + dilation_rate = (1,) + self.dilation_rate + + outputs = nn.separable_conv2d( + inputs, + depthwise_kernel, + pointwise_kernel, + strides=strides, + padding=self.padding.upper(), + rate=dilation_rate, + data_format=conv_utils.convert_data_format(self.data_format, ndim=4)) + + if self.use_bias: + outputs = nn.bias_add( + outputs, + self.bias, + data_format=conv_utils.convert_data_format(self.data_format, ndim=4)) + + outputs = array_ops.squeeze(outputs, [spatial_start_dim]) + + if self.activation is not None: + return self.activation(outputs) + return outputs @tf_export('keras.layers.SeparableConv2D', 'keras.layers.SeparableConvolution2D') -class SeparableConv2D(tf_convolutional_layers.SeparableConv2D, Layer): +class SeparableConv2D(SeparableConv): """Depthwise separable 2D convolution. Separable convolutions consist in first performing @@ -959,15 +1524,15 @@ class SeparableConv2D(tf_convolutional_layers.SeparableConv2D, Layer): pointwise_constraint=None, bias_constraint=None, **kwargs): - if data_format is None: - data_format = K.image_data_format() super(SeparableConv2D, self).__init__( + rank=2, filters=filters, kernel_size=kernel_size, strides=strides, padding=padding, data_format=data_format, dilation_rate=dilation_rate, + depth_multiplier=depth_multiplier, activation=activations.get(activation), use_bias=use_bias, depthwise_initializer=initializers.get(depthwise_initializer), @@ -982,47 +1547,30 @@ class SeparableConv2D(tf_convolutional_layers.SeparableConv2D, Layer): bias_constraint=constraints.get(bias_constraint), **kwargs) - def get_config(self): - config = { - 'filters': - self.filters, - 'kernel_size': - self.kernel_size, - 'strides': - self.strides, - 'padding': - self.padding, - 'data_format': - self.data_format, - 'dilation_rate': - self.dilation_rate, - 'activation': - activations.serialize(self.activation), - 'use_bias': - self.use_bias, - 'depthwise_initializer': - initializers.serialize(self.depthwise_initializer), - 'pointwise_initializer': - initializers.serialize(self.pointwise_initializer), - 'bias_initializer': - initializers.serialize(self.bias_initializer), - 'depthwise_regularizer': - regularizers.serialize(self.depthwise_regularizer), - 'pointwise_regularizer': - regularizers.serialize(self.pointwise_regularizer), - 'bias_regularizer': - regularizers.serialize(self.bias_regularizer), - 'activity_regularizer': - regularizers.serialize(self.activity_regularizer), - 'depthwise_constraint': - constraints.serialize(self.depthwise_constraint), - 'pointwise_constraint': - constraints.serialize(self.pointwise_constraint), - 'bias_constraint': - constraints.serialize(self.bias_constraint) - } - base_config = super(SeparableConv2D, self).get_config() - return dict(list(base_config.items()) + list(config.items())) + def call(self, inputs): + # Apply the actual ops. + if self.data_format == 'channels_last': + strides = (1,) + self.strides + (1,) + else: + strides = (1, 1) + self.strides + outputs = nn.separable_conv2d( + inputs, + self.depthwise_kernel, + self.pointwise_kernel, + strides=strides, + padding=self.padding.upper(), + rate=self.dilation_rate, + data_format=conv_utils.convert_data_format(self.data_format, ndim=4)) + + if self.use_bias: + outputs = nn.bias_add( + outputs, + self.bias, + data_format=conv_utils.convert_data_format(self.data_format, ndim=4)) + + if self.activation is not None: + return self.activation(outputs) + return outputs @tf_export('keras.layers.DepthwiseConv2D') @@ -1162,7 +1710,7 @@ class DepthwiseConv2D(Conv2D): self.built = True def call(self, inputs, training=None): - outputs = K.depthwise_conv2d( + outputs = backend.depthwise_conv2d( inputs, self.depthwise_kernel, strides=self.strides, @@ -1171,7 +1719,7 @@ class DepthwiseConv2D(Conv2D): data_format=self.data_format) if self.bias: - outputs = K.bias_add( + outputs = backend.bias_add( outputs, self.bias, data_format=self.data_format) @@ -1246,7 +1794,7 @@ class UpSampling1D(Layer): return tensor_shape.TensorShape([input_shape[0], size, input_shape[2]]) def call(self, inputs): - output = K.repeat_elements(inputs, self.size, axis=1) + output = backend.repeat_elements(inputs, self.size, axis=1) return output def get_config(self): @@ -1315,7 +1863,8 @@ class UpSampling2D(Layer): [input_shape[0], height, width, input_shape[3]]) def call(self, inputs): - return K.resize_images(inputs, self.size[0], self.size[1], self.data_format) + return backend.resize_images( + inputs, self.size[0], self.size[1], self.data_format) def get_config(self): config = {'size': self.size, 'data_format': self.data_format} @@ -1387,8 +1936,8 @@ class UpSampling3D(Layer): [input_shape[0], dim1, dim2, dim3, input_shape[4]]) def call(self, inputs): - return K.resize_volumes(inputs, self.size[0], self.size[1], self.size[2], - self.data_format) + return backend.resize_volumes( + inputs, self.size[0], self.size[1], self.size[2], self.data_format) def get_config(self): config = {'size': self.size, 'data_format': self.data_format} @@ -1429,7 +1978,7 @@ class ZeroPadding1D(Layer): return tensor_shape.TensorShape([input_shape[0], length, input_shape[2]]) def call(self, inputs): - return K.temporal_padding(inputs, padding=self.padding) + return backend.temporal_padding(inputs, padding=self.padding) def get_config(self): config = {'padding': self.padding} @@ -1530,7 +2079,7 @@ class ZeroPadding2D(Layer): [input_shape[0], rows, cols, input_shape[3]]) def call(self, inputs): - return K.spatial_2d_padding( + return backend.spatial_2d_padding( inputs, padding=self.padding, data_format=self.data_format) def get_config(self): @@ -1648,7 +2197,7 @@ class ZeroPadding3D(Layer): [input_shape[0], dim1, dim2, dim3, input_shape[4]]) def call(self, inputs): - return K.spatial_3d_padding( + return backend.spatial_3d_padding( inputs, padding=self.padding, data_format=self.data_format) def get_config(self): diff --git a/tensorflow/python/keras/_impl/keras/layers/core.py b/tensorflow/python/keras/_impl/keras/layers/core.py index c74fc1e4c0..87b997232e 100644 --- a/tensorflow/python/keras/_impl/keras/layers/core.py +++ b/tensorflow/python/keras/_impl/keras/layers/core.py @@ -24,6 +24,7 @@ import types as python_types import numpy as np from tensorflow.python.eager import context +from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.keras._impl.keras import activations from tensorflow.python.keras._impl.keras import backend as K @@ -32,13 +33,14 @@ from tensorflow.python.keras._impl.keras import initializers from tensorflow.python.keras._impl.keras import regularizers from tensorflow.python.keras._impl.keras.engine import InputSpec from tensorflow.python.keras._impl.keras.engine import Layer -from tensorflow.python.keras._impl.keras.utils.generic_utils import deserialize_keras_object -from tensorflow.python.keras._impl.keras.utils.generic_utils import func_dump -from tensorflow.python.keras._impl.keras.utils.generic_utils import func_load -from tensorflow.python.keras._impl.keras.utils.generic_utils import has_arg -from tensorflow.python.layers import core as tf_core_layers +from tensorflow.python.keras._impl.keras.utils import generic_utils +from tensorflow.python.keras._impl.keras.utils import tf_utils from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import standard_ops from tensorflow.python.util.tf_export import tf_export @@ -94,7 +96,7 @@ class Masking(Layer): @tf_export('keras.layers.Dropout') -class Dropout(tf_core_layers.Dropout, Layer): +class Dropout(Layer): """Applies Dropout to the input. Dropout consists in randomly setting @@ -113,23 +115,39 @@ class Dropout(tf_core_layers.Dropout, Layer): """ def __init__(self, rate, noise_shape=None, seed=None, **kwargs): - # Inheritance call order: - # 1) tf.layers.Dropout, 2) keras.layers.Layer, 3) tf.layers.Layer - super(Dropout, self).__init__(rate=rate, - noise_shape=noise_shape, - seed=seed, - **kwargs) + super(Dropout, self).__init__(**kwargs) + self.rate = rate + self.noise_shape = noise_shape + self.seed = seed self.supports_masking = True + def _get_noise_shape(self, inputs): + # Subclasses of `Dropout` may implement `_get_noise_shape(self, inputs)`, + # which will override `self.noise_shape`, and allows for custom noise + # shapes with dynamically sized inputs. + if self.noise_shape is None: + return self.noise_shape + return nn_ops._get_noise_shape(inputs, self.noise_shape) # pylint: disable=protected-access + def call(self, inputs, training=None): if training is None: training = K.learning_phase() - output = super(Dropout, self).call(inputs, training=training) + + def dropped_inputs(): + return nn.dropout(inputs, 1 - self.rate, + noise_shape=self._get_noise_shape(inputs), + seed=self.seed) + output = tf_utils.smart_cond(training, + dropped_inputs, + lambda: array_ops.identity(inputs)) # EagerTensor object has no attribute _uses_learning_phase if not context.executing_eagerly() and training is K.learning_phase(): output._uses_learning_phase = True # pylint: disable=protected-access return output + def compute_output_shape(self, input_shape): + return input_shape + def get_config(self): config = { 'rate': self.rate, @@ -479,7 +497,7 @@ class Permute(Layer): @tf_export('keras.layers.Flatten') -class Flatten(tf_core_layers.Flatten, Layer): +class Flatten(Layer): """Flattens the input. Does not affect the batch size. Example: @@ -495,7 +513,25 @@ class Flatten(tf_core_layers.Flatten, Layer): # now: model.output_shape == (None, 65536) ``` """ - pass + + def __init__(self, **kwargs): + super(Flatten, self).__init__(**kwargs) + self.input_spec = InputSpec(min_ndim=2) + + def call(self, inputs): + outputs = array_ops.reshape(inputs, (array_ops.shape(inputs)[0], -1)) + if not context.executing_eagerly(): + outputs.set_shape(self.compute_output_shape(inputs.get_shape())) + return outputs + + def compute_output_shape(self, input_shape): + input_shape = tensor_shape.TensorShape(input_shape).as_list() + output_shape = [input_shape[0]] + if all(input_shape[1:]): + output_shape += [np.prod(input_shape[1:])] + else: + output_shape += [None] + return tensor_shape.TensorShape(output_shape) @tf_export('keras.layers.RepeatVector') @@ -611,10 +647,12 @@ class Lambda(Layer): 'must be a list, a tuple, or a function.') self._output_shape = output_shape - def _compute_output_shape(self, input_shape): + def compute_output_shape(self, input_shape): input_shape = tuple(tensor_shape.TensorShape(input_shape).as_list()) if self._output_shape is None: + if context.executing_eagerly(): + raise NotImplementedError x = K.placeholder(shape=input_shape) x = self.call(x) if isinstance(x, list): @@ -640,7 +678,7 @@ class Lambda(Layer): def call(self, inputs, mask=None): arguments = self.arguments - if has_arg(self.function, 'mask'): + if generic_utils.has_arg(self.function, 'mask'): arguments['mask'] = mask return self.function(inputs, **arguments) @@ -651,14 +689,14 @@ class Lambda(Layer): def get_config(self): if isinstance(self.function, python_types.LambdaType): - function = func_dump(self.function) + function = generic_utils.func_dump(self.function) function_type = 'lambda' else: function = self.function.__name__ function_type = 'function' if isinstance(self._output_shape, python_types.LambdaType): - output_shape = func_dump(self._output_shape) + output_shape = generic_utils.func_dump(self._output_shape) output_shape_type = 'lambda' elif callable(self._output_shape): output_shape = self._output_shape.__name__ @@ -686,26 +724,27 @@ class Lambda(Layer): function_type = config.pop('function_type') if function_type == 'function': # Simple lookup in custom objects - function = deserialize_keras_object( + function = generic_utils.deserialize_keras_object( config['function'], custom_objects=custom_objects, printable_module_name='function in Lambda layer') elif function_type == 'lambda': # Unsafe deserialization from bytecode - function = func_load(config['function'], globs=globs) + function = generic_utils.func_load(config['function'], globs=globs) else: raise TypeError('Unknown function type:', function_type) output_shape_type = config.pop('output_shape_type') if output_shape_type == 'function': # Simple lookup in custom objects - output_shape = deserialize_keras_object( + output_shape = generic_utils.deserialize_keras_object( config['output_shape'], custom_objects=custom_objects, printable_module_name='output_shape function in Lambda layer') elif output_shape_type == 'lambda': # Unsafe deserialization from bytecode - output_shape = func_load(config['output_shape'], globs=globs) + output_shape = generic_utils.func_load(config['output_shape'], + globs=globs) else: output_shape = config['output_shape'] @@ -725,7 +764,7 @@ class Lambda(Layer): @tf_export('keras.layers.Dense') -class Dense(tf_core_layers.Dense, Layer): +class Dense(Layer): """Just your regular densely-connected NN layer. `Dense` implements the operation: @@ -795,21 +834,74 @@ class Dense(tf_core_layers.Dense, Layer): if 'input_shape' not in kwargs and 'input_dim' in kwargs: kwargs['input_shape'] = (kwargs.pop('input_dim'),) - # Inheritance call order: - # 1) tf.layers.Dense, 2) keras.layers.Layer, 3) tf.layers.Layer super(Dense, self).__init__( - units, - activation=activations.get(activation), - use_bias=use_bias, - kernel_initializer=initializers.get(kernel_initializer), - bias_initializer=initializers.get(bias_initializer), - kernel_regularizer=regularizers.get(kernel_regularizer), - bias_regularizer=regularizers.get(bias_regularizer), - activity_regularizer=regularizers.get(activity_regularizer), - kernel_constraint=constraints.get(kernel_constraint), - bias_constraint=constraints.get(bias_constraint), - **kwargs) + activity_regularizer=regularizers.get(activity_regularizer), **kwargs) + self.units = units + self.activation = activations.get(activation) + self.use_bias = use_bias + self.kernel_initializer = initializers.get(kernel_initializer) + self.bias_initializer = initializers.get(bias_initializer) + self.kernel_regularizer = regularizers.get(kernel_regularizer) + self.bias_regularizer = regularizers.get(bias_regularizer) + self.kernel_constraint = constraints.get(kernel_constraint) + self.bias_constraint = constraints.get(bias_constraint) + self.supports_masking = True + self.input_spec = InputSpec(min_ndim=2) + + def build(self, input_shape): + input_shape = tensor_shape.TensorShape(input_shape) + if input_shape[-1].value is None: + raise ValueError('The last dimension of the inputs to `Dense` ' + 'should be defined. Found `None`.') + self.input_spec = InputSpec(min_ndim=2, + axes={-1: input_shape[-1].value}) + self.kernel = self.add_variable('kernel', + shape=[input_shape[-1].value, self.units], + initializer=self.kernel_initializer, + regularizer=self.kernel_regularizer, + constraint=self.kernel_constraint, + dtype=self.dtype, + trainable=True) + if self.use_bias: + self.bias = self.add_variable('bias', + shape=[self.units,], + initializer=self.bias_initializer, + regularizer=self.bias_regularizer, + constraint=self.bias_constraint, + dtype=self.dtype, + trainable=True) + else: + self.bias = None + self.built = True + + def call(self, inputs): + inputs = ops.convert_to_tensor(inputs, dtype=self.dtype) + shape = inputs.get_shape().as_list() + if len(shape) > 2: + # Broadcasting is required for the inputs. + outputs = standard_ops.tensordot(inputs, self.kernel, [[len(shape) - 1], + [0]]) + # Reshape the output back to the original ndim of the input. + if not context.executing_eagerly(): + output_shape = shape[:-1] + [self.units] + outputs.set_shape(output_shape) + else: + outputs = gen_math_ops.mat_mul(inputs, self.kernel) + if self.use_bias: + outputs = nn.bias_add(outputs, self.bias) + if self.activation is not None: + return self.activation(outputs) # pylint: disable=not-callable + return outputs + + def compute_output_shape(self, input_shape): + input_shape = tensor_shape.TensorShape(input_shape) + input_shape = input_shape.with_rank_at_least(2) + if input_shape[-1].value is None: + raise ValueError( + 'The innermost dimension of input_shape must be defined, but saw: %s' + % input_shape) + return input_shape[:-1].concatenate(self.units) def get_config(self): config = { diff --git a/tensorflow/python/keras/_impl/keras/layers/core_test.py b/tensorflow/python/keras/_impl/keras/layers/core_test.py index 551d1b1c3a..d22d8d12dc 100644 --- a/tensorflow/python/keras/_impl/keras/layers/core_test.py +++ b/tensorflow/python/keras/_impl/keras/layers/core_test.py @@ -129,7 +129,6 @@ class CoreLayersTest(test.TestCase): testing_utils.layer_test( keras.layers.RepeatVector, kwargs={'n': 3}, input_shape=(3, 2)) - @tf_test_util.run_in_graph_and_eager_modes() def test_lambda(self): testing_utils.layer_test( keras.layers.Lambda, diff --git a/tensorflow/python/keras/_impl/keras/layers/embeddings.py b/tensorflow/python/keras/_impl/keras/layers/embeddings.py index 540e2d945c..591bab7cd8 100644 --- a/tensorflow/python/keras/_impl/keras/layers/embeddings.py +++ b/tensorflow/python/keras/_impl/keras/layers/embeddings.py @@ -102,7 +102,8 @@ class Embedding(Layer): kwargs['input_shape'] = (input_length,) else: kwargs['input_shape'] = (None,) - super(Embedding, self).__init__(**kwargs) + dtype = kwargs.pop('dtype', K.floatx()) + super(Embedding, self).__init__(dtype=dtype, **kwargs) self.input_dim = input_dim self.output_dim = output_dim @@ -120,8 +121,7 @@ class Embedding(Layer): initializer=self.embeddings_initializer, name='embeddings', regularizer=self.embeddings_regularizer, - constraint=self.embeddings_constraint, - dtype=self.dtype) + constraint=self.embeddings_constraint) self.built = True def compute_mask(self, inputs, mask=None): diff --git a/tensorflow/python/keras/_impl/keras/layers/normalization.py b/tensorflow/python/keras/_impl/keras/layers/normalization.py index 3b44b20bf8..b60d864ae5 100644 --- a/tensorflow/python/keras/_impl/keras/layers/normalization.py +++ b/tensorflow/python/keras/_impl/keras/layers/normalization.py @@ -19,17 +19,29 @@ from __future__ import division from __future__ import print_function from tensorflow.python.eager import context +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_shape from tensorflow.python.keras._impl.keras import backend as K from tensorflow.python.keras._impl.keras import constraints from tensorflow.python.keras._impl.keras import initializers from tensorflow.python.keras._impl.keras import regularizers +from tensorflow.python.keras._impl.keras.engine import InputSpec from tensorflow.python.keras._impl.keras.engine import Layer -from tensorflow.python.layers import normalization as tf_normalization_layers +from tensorflow.python.keras._impl.keras.utils import tf_utils +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import state_ops +from tensorflow.python.platform import tf_logging as logging +from tensorflow.python.training import distribute as distribute_lib from tensorflow.python.util.tf_export import tf_export @tf_export('keras.layers.BatchNormalization') -class BatchNormalization(tf_normalization_layers.BatchNormalization, Layer): +class BatchNormalization(Layer): """Batch normalization layer (Ioffe and Szegedy, 2014). Normalize the activations of the previous layer at each batch, @@ -37,28 +49,63 @@ class BatchNormalization(tf_normalization_layers.BatchNormalization, Layer): close to 0 and the activation standard deviation close to 1. Arguments: - axis: Integer, the axis that should be normalized - (typically the features axis). - For instance, after a `Conv2D` layer with - `data_format="channels_first"`, - set `axis=1` in `BatchNormalization`. - momentum: Momentum for the moving average. - epsilon: Small float added to variance to avoid dividing by zero. - center: If True, add offset of `beta` to normalized tensor. - If False, `beta` is ignored. - scale: If True, multiply by `gamma`. - If False, `gamma` is not used. - When the next layer is linear (also e.g. `nn.relu`), - this can be disabled since the scaling - will be done by the next layer. - beta_initializer: Initializer for the beta weight. - gamma_initializer: Initializer for the gamma weight. - moving_mean_initializer: Initializer for the moving mean. - moving_variance_initializer: Initializer for the moving variance. - beta_regularizer: Optional regularizer for the beta weight. - gamma_regularizer: Optional regularizer for the gamma weight. - beta_constraint: Optional constraint for the beta weight. - gamma_constraint: Optional constraint for the gamma weight. + axis: Integer, the axis that should be normalized + (typically the features axis). + For instance, after a `Conv2D` layer with + `data_format="channels_first"`, + set `axis=1` in `BatchNormalization`. + momentum: Momentum for the moving average. + epsilon: Small float added to variance to avoid dividing by zero. + center: If True, add offset of `beta` to normalized tensor. + If False, `beta` is ignored. + scale: If True, multiply by `gamma`. + If False, `gamma` is not used. + When the next layer is linear (also e.g. `nn.relu`), + this can be disabled since the scaling + will be done by the next layer. + beta_initializer: Initializer for the beta weight. + gamma_initializer: Initializer for the gamma weight. + moving_mean_initializer: Initializer for the moving mean. + moving_variance_initializer: Initializer for the moving variance. + beta_regularizer: Optional regularizer for the beta weight. + gamma_regularizer: Optional regularizer for the gamma weight. + beta_constraint: Optional constraint for the beta weight. + gamma_constraint: Optional constraint for the gamma weight. + renorm: Whether to use Batch Renormalization + (https://arxiv.org/abs/1702.03275). This adds extra variables during + training. The inference is the same for either value of this parameter. + renorm_clipping: A dictionary that may map keys 'rmax', 'rmin', 'dmax' to + scalar `Tensors` used to clip the renorm correction. The correction + `(r, d)` is used as `corrected_value = normalized_value * r + d`, with + `r` clipped to [rmin, rmax], and `d` to [-dmax, dmax]. Missing rmax, rmin, + dmax are set to inf, 0, inf, respectively. + renorm_momentum: Momentum used to update the moving means and standard + deviations with renorm. Unlike `momentum`, this affects training + and should be neither too small (which would add noise) nor too large + (which would give stale estimates). Note that `momentum` is still applied + to get the means and variances for inference. + fused: if `None` or `True`, use a faster, fused implementation if possible. + If `False`, use the system recommended implementation. + trainable: Boolean, if `True` also add variables to the graph collection + `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable). + virtual_batch_size: An `int`. By default, `virtual_batch_size` is `None`, + which means batch normalization is performed across the whole batch. When + `virtual_batch_size` is not `None`, instead perform "Ghost Batch + Normalization", which creates virtual sub-batches which are each + normalized separately (with shared gamma, beta, and moving statistics). + Must divide the actual batch size during execution. + adjustment: A function taking the `Tensor` containing the (dynamic) shape of + the input tensor and returning a pair (scale, bias) to apply to the + normalized values (before gamma and beta), only during training. For + example, if axis==-1, + `adjustment = lambda shape: ( + tf.random_uniform(shape[-1:], 0.93, 1.07), + tf.random_uniform(shape[-1:], -0.1, 0.1))` + will scale the normalized value by up to 7% up or down, then shift the + result by up to 0.1 (with independent scaling and bias for each feature + but shared across all examples), and finally apply gamma and/or beta. If + `None`, no adjustment is applied. Cannot be specified if + virtual_batch_size is specified. Input shape: Arbitrary. Use the keyword argument `input_shape` @@ -87,33 +134,537 @@ class BatchNormalization(tf_normalization_layers.BatchNormalization, Layer): gamma_regularizer=None, beta_constraint=None, gamma_constraint=None, + renorm=False, + renorm_clipping=None, + renorm_momentum=0.99, + fused=None, + trainable=True, + virtual_batch_size=None, + adjustment=None, + name=None, **kwargs): - self.supports_masking = True super(BatchNormalization, self).__init__( - axis=axis, - momentum=momentum, - epsilon=epsilon, - center=center, - scale=scale, - beta_initializer=initializers.get(beta_initializer), - gamma_initializer=initializers.get(gamma_initializer), - moving_mean_initializer=initializers.get(moving_mean_initializer), - moving_variance_initializer=initializers.get( - moving_variance_initializer), - beta_regularizer=regularizers.get(beta_regularizer), - gamma_regularizer=regularizers.get(gamma_regularizer), - beta_constraint=constraints.get(beta_constraint), - gamma_constraint=constraints.get(gamma_constraint), - **kwargs - ) + name=name, trainable=trainable, **kwargs) + if isinstance(axis, list): + self.axis = axis[:] + else: + self.axis = axis + self.momentum = momentum + self.epsilon = epsilon + self.center = center + self.scale = scale + self.beta_initializer = initializers.get(beta_initializer) + self.gamma_initializer = initializers.get(gamma_initializer) + self.moving_mean_initializer = initializers.get(moving_mean_initializer) + self.moving_variance_initializer = initializers.get( + moving_variance_initializer) + self.beta_regularizer = regularizers.get(beta_regularizer) + self.gamma_regularizer = regularizers.get(gamma_regularizer) + self.beta_constraint = constraints.get(beta_constraint) + self.gamma_constraint = constraints.get(gamma_constraint) + self.renorm = renorm + self.virtual_batch_size = virtual_batch_size + self.adjustment = adjustment + if fused is None: + fused = True + self.supports_masking = True + + self.fused = fused + self._bessels_correction_test_only = True + self._use_resource_variables = None + + if renorm: + renorm_clipping = renorm_clipping or {} + keys = ['rmax', 'rmin', 'dmax'] + if set(renorm_clipping) - set(keys): + raise ValueError('renorm_clipping %s contains keys not in %s' % + (renorm_clipping, keys)) + self.renorm_clipping = renorm_clipping + self.renorm_momentum = renorm_momentum + + def _add_tower_local_variable(self, *args, **kwargs): + tower_context = distribute_lib.get_tower_context() + with tower_context.tower_local_var_scope('mean'): + return self.add_variable(*args, **kwargs) + + def build(self, input_shape): + input_shape = tensor_shape.TensorShape(input_shape) + if not input_shape.ndims: + raise ValueError('Input has undefined rank:', input_shape) + ndims = len(input_shape) + + # Convert axis to list and resolve negatives + if isinstance(self.axis, int): + self.axis = [self.axis] + + if not isinstance(self.axis, list): + raise TypeError('axis must be int or list, type given: %s' + % type(self.axis)) + + for idx, x in enumerate(self.axis): + if x < 0: + self.axis[idx] = ndims + x + + # Validate axes + for x in self.axis: + if x < 0 or x >= ndims: + raise ValueError('Invalid axis: %d' % x) + if len(self.axis) != len(set(self.axis)): + raise ValueError('Duplicate axis: %s' % self.axis) + + if self.virtual_batch_size is not None: + if self.virtual_batch_size <= 0: + raise ValueError('virtual_batch_size must be a positive integer that ' + 'divides the true batch size of the input Tensor') + # If using virtual batches, the first dimension must be the batch + # dimension and cannot be the batch norm axis + if 0 in self.axis: + raise ValueError('When using virtual_batch_size, the batch dimension ' + 'must be 0 and thus axis cannot include 0') + if self.adjustment is not None: + raise ValueError('When using virtual_batch_size, adjustment cannot ' + 'be specified') + + if self.fused: + # Currently fused batch norm doesn't support renorm. It also only supports + # an input tensor of rank 4 and a channel dimension on axis 1 or 3. + # TODO(yaozhang): if input is not 4D, reshape it to 4D and reshape the + # output back to its original shape accordingly. + self.fused = (not self.renorm and + ndims == 4 and + self.axis in [[1], [3]] and + self.virtual_batch_size is None and + self.adjustment is None) + # TODO(chrisying): fused batch norm is currently not supported for + # multi-axis batch norm and by extension virtual batches. In some cases, + # it might be possible to use fused batch norm but would require reshaping + # the Tensor to 4D with the axis in 1 or 3 (preferred 1) which is + # particularly tricky. A compromise might be to just support the most + # common use case (turning 5D w/ virtual batch to NCHW) + + if self.fused: + if self.axis == [1]: + self._data_format = 'NCHW' + elif self.axis == [3]: + self._data_format = 'NHWC' + else: + raise ValueError('Unsupported axis, fused batch norm only supports ' + 'axis == [1] or axis == [3]') + + # Raise parameters of fp16 batch norm to fp32 + if self.dtype == dtypes.float16 or self.dtype == dtypes.bfloat16: + param_dtype = dtypes.float32 + else: + param_dtype = self.dtype or dtypes.float32 + + axis_to_dim = {x: input_shape[x].value for x in self.axis} + for x in axis_to_dim: + if axis_to_dim[x] is None: + raise ValueError('Input has undefined `axis` dimension. Input shape: ', + input_shape) + self.input_spec = InputSpec(ndim=ndims, axes=axis_to_dim) + + if len(axis_to_dim) == 1 and self.virtual_batch_size is None: + # Single axis batch norm (most common/default use-case) + param_shape = (list(axis_to_dim.values())[0],) + else: + # Parameter shape is the original shape but with 1 in all non-axis dims + param_shape = [axis_to_dim[i] if i in axis_to_dim + else 1 for i in range(ndims)] + if self.virtual_batch_size is not None: + # When using virtual batches, add an extra dim at index 1 + param_shape.insert(1, 1) + for idx, x in enumerate(self.axis): + self.axis[idx] = x + 1 # Account for added dimension + + # BUG: when using fused BN with Resource Variables with a dynamic + # `training` argument in call, the cond + # `smart_cond( + # training, + # _fused_batch_norm_training, + # _fused_batch_norm_inference)` triggers None gradients for the + # variables gamma and beta. + # In this case we choose to force normal variables when possible. + # The bug will not occur of `training` is static, or when + # not using fused BN, or when in eager execution. + # TODO(fchollet): remove code below when bug is fixed. + use_resource = False + if context.executing_eagerly(): + use_resource = True # Eager execution requires resource variables. + elif not self.fused: + use_resource = True # Issue only exists with fused BN. + elif self._use_resource_variables is True: + use_resource = True # Case of a subclassed model, always use RVs. + if hasattr(self, '_scope'): + use_resource = None # Legacy layers, leave it to `add_weight`. + + if self.scale: + self.gamma = self.add_variable( + name='gamma', + shape=param_shape, + dtype=param_dtype, + initializer=self.gamma_initializer, + regularizer=self.gamma_regularizer, + constraint=self.gamma_constraint, + use_resource=use_resource, + trainable=True) + else: + self.gamma = None + if self.fused: + self._gamma_const = array_ops.constant( + 1.0, dtype=param_dtype, shape=param_shape) + + if self.center: + self.beta = self.add_variable( + name='beta', + shape=param_shape, + dtype=param_dtype, + initializer=self.beta_initializer, + regularizer=self.beta_regularizer, + constraint=self.beta_constraint, + use_resource=use_resource, + trainable=True) + else: + self.beta = None + if self.fused: + self._beta_const = array_ops.constant( + 0.0, dtype=param_dtype, shape=param_shape) + + try: + # Disable variable partitioning when creating the moving mean and variance + if hasattr(self, '_scope') and self._scope: + partitioner = self._scope.partitioner + self._scope.set_partitioner(None) + else: + partitioner = None + self.moving_mean = self._add_tower_local_variable( + name='moving_mean', + shape=param_shape, + dtype=param_dtype, + initializer=self.moving_mean_initializer, + trainable=False) + + self.moving_variance = self._add_tower_local_variable( + name='moving_variance', + shape=param_shape, + dtype=param_dtype, + initializer=self.moving_variance_initializer, + trainable=False) + + if self.renorm: + # Create variables to maintain the moving mean and standard deviation. + # These are used in training and thus are different from the moving + # averages above. The renorm variables are colocated with moving_mean + # and moving_variance. + # NOTE: below, the outer `with device` block causes the current device + # stack to be cleared. The nested ones use a `lambda` to set the desired + # device and ignore any devices that may be set by the custom getter. + def _renorm_variable(name, shape): + var = self._add_tower_local_variable( + name=name, + shape=shape, + dtype=param_dtype, + initializer=init_ops.zeros_initializer(), + trainable=False) + return var + + with distribute_lib.get_distribution_strategy().colocate_vars_with( + self.moving_mean): + self.renorm_mean = _renorm_variable('renorm_mean', param_shape) + self.renorm_mean_weight = _renorm_variable('renorm_mean_weight', ()) + # We initialize renorm_stddev to 0, and maintain the (0-initialized) + # renorm_stddev_weight. This allows us to (1) mix the average + # stddev with the minibatch stddev early in training, and (2) compute + # the unbiased average stddev by dividing renorm_stddev by the weight. + with distribute_lib.get_distribution_strategy().colocate_vars_with( + self.moving_variance): + self.renorm_stddev = _renorm_variable('renorm_stddev', param_shape) + self.renorm_stddev_weight = _renorm_variable('renorm_stddev_weight', + ()) + finally: + if partitioner: + self._scope.set_partitioner(partitioner) + self.built = True + + def _assign_moving_average(self, variable, value, momentum): + with ops.name_scope(None, 'AssignMovingAvg', + [variable, value, momentum]) as scope: + decay = ops.convert_to_tensor(1.0 - momentum, name='decay') + if decay.dtype != variable.dtype.base_dtype: + decay = math_ops.cast(decay, variable.dtype.base_dtype) + update_delta = (variable - value) * decay + return state_ops.assign_sub(variable, update_delta, name=scope) + + def _fused_batch_norm(self, inputs, training): + """Returns the output of fused batch norm.""" + beta = self.beta if self.center else self._beta_const + gamma = self.gamma if self.scale else self._gamma_const + + def _fused_batch_norm_training(): + return nn.fused_batch_norm( + inputs, + gamma, + beta, + epsilon=self.epsilon, + data_format=self._data_format) + + def _fused_batch_norm_inference(): + return nn.fused_batch_norm( + inputs, + gamma, + beta, + mean=self.moving_mean, + variance=self.moving_variance, + epsilon=self.epsilon, + is_training=False, + data_format=self._data_format) + + output, mean, variance = tf_utils.smart_cond( + training, _fused_batch_norm_training, _fused_batch_norm_inference) + if not self._bessels_correction_test_only: + # Remove Bessel's correction to be consistent with non-fused batch norm. + # Note that the variance computed by fused batch norm is + # with Bessel's correction. + sample_size = math_ops.cast( + array_ops.size(inputs) / array_ops.size(variance), variance.dtype) + factor = (sample_size - math_ops.cast(1.0, variance.dtype)) / sample_size + variance *= factor + + training_value = tf_utils.constant_value(training) + if training_value is None: + momentum = tf_utils.smart_cond(training, + lambda: self.momentum, + lambda: 1.0) + else: + momentum = ops.convert_to_tensor(self.momentum) + if training_value or training_value is None: + mean_update = self._assign_moving_average(self.moving_mean, mean, + momentum) + variance_update = self._assign_moving_average(self.moving_variance, + variance, momentum) + self.add_update(mean_update, inputs=True) + self.add_update(variance_update, inputs=True) + + return output + + def _renorm_correction_and_moments(self, mean, variance, training): + """Returns the correction and update values for renorm.""" + stddev = math_ops.sqrt(variance + self.epsilon) + # Compute the average mean and standard deviation, as if they were + # initialized with this batch's moments. + mixed_renorm_mean = (self.renorm_mean + + (1. - self.renorm_mean_weight) * mean) + mixed_renorm_stddev = (self.renorm_stddev + + (1. - self.renorm_stddev_weight) * stddev) + # Compute the corrections for batch renorm. + r = stddev / mixed_renorm_stddev + d = (mean - mixed_renorm_mean) / mixed_renorm_stddev + # Ensure the corrections use pre-update moving averages. + with ops.control_dependencies([r, d]): + mean = array_ops.identity(mean) + stddev = array_ops.identity(stddev) + rmin, rmax, dmax = [self.renorm_clipping.get(key) + for key in ['rmin', 'rmax', 'dmax']] + if rmin is not None: + r = math_ops.maximum(r, rmin) + if rmax is not None: + r = math_ops.minimum(r, rmax) + if dmax is not None: + d = math_ops.maximum(d, -dmax) + d = math_ops.minimum(d, dmax) + # When not training, use r=1, d=0. + r = tf_utils.smart_cond(training, lambda: r, lambda: array_ops.ones_like(r)) + d = tf_utils.smart_cond(training, + lambda: d, + lambda: array_ops.zeros_like(d)) + + def _update_renorm_variable(var, weight, value): + """Updates a moving average and weight, returns the unbiased value.""" + value = array_ops.identity(value) + def _do_update(): + """Updates the var and weight, returns their updated ratio.""" + # Update the variables without zero debiasing. The debiasing will be + # accomplished by dividing the exponential moving average by the weight. + # For example, after a single update, the moving average would be + # (1-decay) * value. and the weight will be 1-decay, with their ratio + # giving the value. + # Make sure the weight is not updated until before r and d computation. + with ops.control_dependencies([value]): + weight_value = array_ops.constant(1., dtype=weight.dtype) + new_var = self._assign_moving_average(var, value, self.renorm_momentum) + new_weight = self._assign_moving_average(weight, weight_value, + self.renorm_momentum) + # TODO(yuefengz): the updates to var and weighted can not be batched + # together if we fetch their updated values here. Consider calculating + # new values and delaying the updates. + return new_var / new_weight + + def _fake_update(): + return array_ops.identity(var) + return tf_utils.smart_cond(training, _do_update, _fake_update) + + # TODO(yuefengz): colocate the operations + new_mean = _update_renorm_variable(self.renorm_mean, + self.renorm_mean_weight, mean) + new_stddev = _update_renorm_variable(self.renorm_stddev, + self.renorm_stddev_weight, stddev) + # Make sqrt(moving_variance + epsilon) = new_stddev. + new_variance = math_ops.square(new_stddev) - self.epsilon + + return (r, d, new_mean, new_variance) def call(self, inputs, training=None): if training is None: training = K.learning_phase() - output = super(BatchNormalization, self).call(inputs, training=training) + + in_eager_mode = context.executing_eagerly() + if self.virtual_batch_size is not None: + # Virtual batches (aka ghost batches) can be simulated by reshaping the + # Tensor and reusing the existing batch norm implementation + original_shape = [-1] + inputs.shape.as_list()[1:] + expanded_shape = [self.virtual_batch_size, -1] + original_shape[1:] + + # Will cause errors if virtual_batch_size does not divide the batch size + inputs = array_ops.reshape(inputs, expanded_shape) + + def undo_virtual_batching(outputs): + outputs = array_ops.reshape(outputs, original_shape) + return outputs + + # Gradient bug when using fused BN with dynamic `training` and resource + # variables. TODO(fchollet): remove workaround when bug fixed. + use_fused_bn = ( + self.fused and + (tf_utils.constant_value(training) is not None or + not isinstance(self.gamma, resource_variable_ops.ResourceVariable))) + if use_fused_bn: + outputs = self._fused_batch_norm(inputs, training=training) + if self.virtual_batch_size is not None: + # Currently never reaches here since fused_batch_norm does not support + # virtual batching + outputs = undo_virtual_batching(outputs) + if not context.executing_eagerly() and training is K.learning_phase(): + outputs._uses_learning_phase = True # pylint: disable=protected-access + return outputs + + # Compute the axes along which to reduce the mean / variance + input_shape = inputs.get_shape() + ndims = len(input_shape) + reduction_axes = [i for i in range(ndims) if i not in self.axis] + if self.virtual_batch_size is not None: + del reduction_axes[1] # Do not reduce along virtual batch dim + + # Broadcasting only necessary for single-axis batch norm where the axis is + # not the last dimension + broadcast_shape = [1] * ndims + broadcast_shape[self.axis[0]] = input_shape[self.axis[0]].value + def _broadcast(v): + if (v is not None and + len(v.get_shape()) != ndims and + reduction_axes != list(range(ndims - 1))): + return array_ops.reshape(v, broadcast_shape) + return v + + scale, offset = _broadcast(self.gamma), _broadcast(self.beta) + + def _compose_transforms(scale, offset, then_scale, then_offset): + if then_scale is not None: + scale *= then_scale + offset *= then_scale + if then_offset is not None: + offset += then_offset + return (scale, offset) + + # Determine a boolean value for `training`: could be True, False, or None. + training_value = tf_utils.constant_value(training) + if training_value is not False: + if self.adjustment: + adj_scale, adj_bias = self.adjustment(array_ops.shape(inputs)) + # Adjust only during training. + adj_scale = tf_utils.smart_cond(training, + lambda: adj_scale, + lambda: array_ops.ones_like(adj_scale)) + adj_bias = tf_utils.smart_cond(training, + lambda: adj_bias, + lambda: array_ops.zeros_like(adj_bias)) + scale, offset = _compose_transforms(adj_scale, adj_bias, scale, offset) + + # Some of the computations here are not necessary when training==False + # but not a constant. However, this makes the code simpler. + keep_dims = self.virtual_batch_size is not None or len(self.axis) > 1 + mean, variance = nn.moments(inputs, reduction_axes, keep_dims=keep_dims) + + moving_mean = self.moving_mean + moving_variance = self.moving_variance + + mean = tf_utils.smart_cond(training, + lambda: mean, + lambda: moving_mean) + variance = tf_utils.smart_cond(training, + lambda: variance, + lambda: moving_variance) + + if self.renorm: + r, d, new_mean, new_variance = self._renorm_correction_and_moments( + mean, variance, training) + # When training, the normalized values (say, x) will be transformed as + # x * gamma + beta without renorm, and (x * r + d) * gamma + beta + # = x * (r * gamma) + (d * gamma + beta) with renorm. + r = _broadcast(array_ops.stop_gradient(r, name='renorm_r')) + d = _broadcast(array_ops.stop_gradient(d, name='renorm_d')) + scale, offset = _compose_transforms(r, d, scale, offset) + else: + new_mean, new_variance = mean, variance + + if self.virtual_batch_size is not None: + # This isn't strictly correct since in ghost batch norm, you are + # supposed to sequentially update the moving_mean and moving_variance + # with each sub-batch. However, since the moving statistics are only + # used during evaluation, it is more efficient to just update in one + # step and should not make a significant difference in the result. + new_mean = math_ops.reduce_mean(new_mean, + axis=1, keep_dims=True) + new_variance = math_ops.reduce_mean(new_variance, + axis=1, keep_dims=True) + + def _do_update(var, value): + if in_eager_mode and not self.trainable: + return + + return self._assign_moving_average(var, value, self.momentum) + + mean_update = tf_utils.smart_cond( + training, + lambda: _do_update(self.moving_mean, new_mean), + lambda: self.moving_mean) + variance_update = tf_utils.smart_cond( + training, + lambda: _do_update(self.moving_variance, new_variance), + lambda: self.moving_variance) + if not context.executing_eagerly(): + self.add_update(mean_update, inputs=True) + self.add_update(variance_update, inputs=True) + + else: + mean, variance = self.moving_mean, self.moving_variance + + outputs = nn.batch_normalization(inputs, + _broadcast(mean), + _broadcast(variance), + offset, + scale, + self.epsilon) + # If some components of the shape got lost due to adjustments, fix that. + outputs.set_shape(input_shape) + + if self.virtual_batch_size is not None: + outputs = undo_virtual_batching(outputs) if not context.executing_eagerly() and training is K.learning_phase(): - output._uses_learning_phase = True # pylint: disable=protected-access - return output + outputs._uses_learning_phase = True # pylint: disable=protected-access + return outputs + + def compute_output_shape(self, input_shape): + return input_shape def get_config(self): config = { @@ -133,5 +684,19 @@ class BatchNormalization(tf_normalization_layers.BatchNormalization, Layer): 'beta_constraint': constraints.serialize(self.beta_constraint), 'gamma_constraint': constraints.serialize(self.gamma_constraint) } + # Only add TensorFlow-specific parameters if they are set, so as to preserve + # model compatibility with external Keras. + if self.renorm: + config['renorm'] = True + config['renorm_clipping'] = self.renorm_clipping + config['renorm_momentum'] = self.renorm_momentum + if self.virtual_batch_size is not None: + config['virtual_batch_size'] = self.virtual_batch_size + # Note: adjustment is not serializable. + if self.adjustment is not None: + logging.warning('The `adjustment` function of this `BatchNormalization` ' + 'layer cannot be serialized and has been omitted from ' + 'the layer config. It will not be included when ' + 're-creating the layer from the saved config.') base_config = super(BatchNormalization, self).get_config() return dict(list(base_config.items()) + list(config.items())) diff --git a/tensorflow/python/keras/_impl/keras/layers/normalization_test.py b/tensorflow/python/keras/_impl/keras/layers/normalization_test.py index 2b3628c3f1..fa9277e3d1 100644 --- a/tensorflow/python/keras/_impl/keras/layers/normalization_test.py +++ b/tensorflow/python/keras/_impl/keras/layers/normalization_test.py @@ -114,6 +114,26 @@ class NormalizationLayersTest(test.TestCase): np.testing.assert_allclose(np.mean(out, axis=(0, 2, 3)), 0.0, atol=1e-1) np.testing.assert_allclose(np.std(out, axis=(0, 2, 3)), 1.0, atol=1e-1) + def test_batchnorm_convnet_channel_last(self): + with self.test_session(): + # keras.backend.set_learning_phase(True) + + model = keras.models.Sequential() + norm = keras.layers.BatchNormalization( + axis=-1, input_shape=(4, 4, 3), momentum=0.8) + model.add(norm) + model.compile(loss='mse', optimizer='sgd') + + # centered on 5.0, variance 10.0 + x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 4, 4, 3)) + model.fit(x, x, epochs=4, verbose=0) + out = model.predict(x) + out -= np.reshape(keras.backend.eval(norm.beta), (1, 1, 1, 3)) + out /= np.reshape(keras.backend.eval(norm.gamma), (1, 1, 1, 3)) + + np.testing.assert_allclose(np.mean(out, axis=(0, 1, 2)), 0.0, atol=1e-1) + np.testing.assert_allclose(np.std(out, axis=(0, 1, 2)), 1.0, atol=1e-1) + def test_shared_batchnorm(self): """Test that a BN layer can be shared across different data streams. """ diff --git a/tensorflow/python/keras/_impl/keras/layers/pooling.py b/tensorflow/python/keras/_impl/keras/layers/pooling.py index 15d5337976..86bc8a680a 100644 --- a/tensorflow/python/keras/_impl/keras/layers/pooling.py +++ b/tensorflow/python/keras/_impl/keras/layers/pooling.py @@ -19,16 +19,98 @@ from __future__ import division from __future__ import print_function from tensorflow.python.framework import tensor_shape -from tensorflow.python.keras._impl.keras import backend as K +from tensorflow.python.keras._impl.keras import backend from tensorflow.python.keras._impl.keras.engine import InputSpec from tensorflow.python.keras._impl.keras.engine import Layer from tensorflow.python.keras._impl.keras.utils import conv_utils -from tensorflow.python.layers import pooling as tf_pooling_layers +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn from tensorflow.python.util.tf_export import tf_export +class Pooling1D(Layer): + """Pooling layer for arbitrary pooling functions, for 1D inputs. + + This class only exists for code reuse. It will never be an exposed API. + + Arguments: + pool_function: The pooling function to apply, e.g. `tf.nn.max_pool`. + pool_size: An integer or tuple/list of a single integer, + representing the size of the pooling window. + strides: An integer or tuple/list of a single integer, specifying the + strides of the pooling operation. + padding: A string. The padding method, either 'valid' or 'same'. + Case-insensitive. + data_format: A string, one of `channels_last` (default) or `channels_first`. + The ordering of the dimensions in the inputs. + `channels_last` corresponds to inputs with shape + `(batch, length, channels)` while `channels_first` corresponds to + inputs with shape `(batch, channels, length)`. + name: A string, the name of the layer. + """ + + def __init__(self, pool_function, pool_size, strides, + padding='valid', data_format=None, + name=None, **kwargs): + super(Pooling1D, self).__init__(name=name, **kwargs) + if data_format is None: + data_format = backend.image_data_format() + if strides is None: + strides = pool_size + self.pool_function = pool_function + self.pool_size = conv_utils.normalize_tuple(pool_size, 1, 'pool_size') + self.strides = conv_utils.normalize_tuple(strides, 1, 'strides') + self.padding = conv_utils.normalize_padding(padding) + self.data_format = conv_utils.normalize_data_format(data_format) + self.input_spec = InputSpec(ndim=3) + + def call(self, inputs): + # There is no TF op for 1D pooling, hence we make the inputs 4D. + if self.data_format == 'channels_last': + # input is NWC, make it NHWC + inputs = array_ops.expand_dims(inputs, 1) + # pool on the W dim + pool_shape = (1, 1) + self.pool_size + (1,) + strides = (1, 1) + self.strides + (1,) + data_format = 'NHWC' + else: + # input is NCW, make it NCHW + inputs = array_ops.expand_dims(inputs, 2) + # pool on the W dim + pool_shape = (1, 1, 1) + self.pool_size + strides = (1, 1, 1) + self.strides + data_format = 'NCHW' + + outputs = self.pool_function( + inputs, + ksize=pool_shape, + strides=strides, + padding=self.padding.upper(), + data_format=data_format) + + if self.data_format == 'channels_last': + return array_ops.squeeze(outputs, 1) + else: + return array_ops.squeeze(outputs, 2) + + def compute_output_shape(self, input_shape): + input_shape = tensor_shape.TensorShape(input_shape).as_list() + length = conv_utils.conv_output_length(input_shape[1], self.pool_size[0], + self.padding, self.strides[0]) + return tensor_shape.TensorShape([input_shape[0], length, input_shape[2]]) + + def get_config(self): + config = { + 'strides': self.strides, + 'pool_size': self.pool_size, + 'padding': self.padding + } + base_config = super(Pooling1D, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + @tf_export('keras.layers.MaxPool1D', 'keras.layers.MaxPooling1D') -class MaxPooling1D(tf_pooling_layers.MaxPooling1D, Layer): +class MaxPooling1D(Pooling1D): """Max pooling operation for temporal data. Arguments: @@ -45,23 +127,20 @@ class MaxPooling1D(tf_pooling_layers.MaxPooling1D, Layer): 3D tensor with shape: `(batch_size, downsampled_steps, features)`. """ - def __init__(self, pool_size=2, strides=None, padding='valid', **kwargs): - if strides is None: - strides = pool_size - super(MaxPooling1D, self).__init__(pool_size, strides, padding, **kwargs) + def __init__(self, pool_size=2, strides=None, + padding='valid', data_format=None, **kwargs): - def get_config(self): - config = { - 'strides': self.strides, - 'pool_size': self.pool_size, - 'padding': self.padding - } - base_config = super(MaxPooling1D, self).get_config() - return dict(list(base_config.items()) + list(config.items())) + super(MaxPooling1D, self).__init__( + nn.max_pool, + pool_size=pool_size, + strides=strides, + padding=padding, + data_format=data_format, + **kwargs) @tf_export('keras.layers.AveragePooling1D', 'keras.layers.AvgPool1D') -class AveragePooling1D(tf_pooling_layers.AveragePooling1D, Layer): +class AveragePooling1D(Pooling1D): """Average pooling for temporal data. Arguments: @@ -78,24 +157,104 @@ class AveragePooling1D(tf_pooling_layers.AveragePooling1D, Layer): 3D tensor with shape: `(batch_size, downsampled_steps, features)`. """ - def __init__(self, pool_size=2, strides=None, padding='valid', **kwargs): + def __init__(self, pool_size=2, strides=None, + padding='valid', data_format=None, **kwargs): + super(AveragePooling1D, self).__init__( + nn.avg_pool, + pool_size=pool_size, + strides=strides, + padding=padding, + data_format=data_format, + **kwargs) + + +class Pooling2D(Layer): + """Pooling layer for arbitrary pooling functions, for 2D inputs (e.g. images). + + This class only exists for code reuse. It will never be an exposed API. + + Arguments: + pool_function: The pooling function to apply, e.g. `tf.nn.max_pool`. + pool_size: An integer or tuple/list of 2 integers: (pool_height, pool_width) + specifying the size of the pooling window. + Can be a single integer to specify the same value for + all spatial dimensions. + strides: An integer or tuple/list of 2 integers, + specifying the strides of the pooling operation. + Can be a single integer to specify the same value for + all spatial dimensions. + padding: A string. The padding method, either 'valid' or 'same'. + Case-insensitive. + data_format: A string, one of `channels_last` (default) or `channels_first`. + The ordering of the dimensions in the inputs. + `channels_last` corresponds to inputs with shape + `(batch, height, width, channels)` while `channels_first` corresponds to + inputs with shape `(batch, channels, height, width)`. + name: A string, the name of the layer. + """ + + def __init__(self, pool_function, pool_size, strides, + padding='valid', data_format=None, + name=None, **kwargs): + super(Pooling2D, self).__init__(name=name, **kwargs) + if data_format is None: + data_format = backend.image_data_format() if strides is None: strides = pool_size - super(AveragePooling1D, self).__init__(pool_size, strides, padding, - **kwargs) + self.pool_function = pool_function + self.pool_size = conv_utils.normalize_tuple(pool_size, 2, 'pool_size') + self.strides = conv_utils.normalize_tuple(strides, 2, 'strides') + self.padding = conv_utils.normalize_padding(padding) + self.data_format = conv_utils.normalize_data_format(data_format) + self.input_spec = InputSpec(ndim=4) + + def call(self, inputs): + if self.data_format == 'channels_last': + pool_shape = (1,) + self.pool_size + (1,) + strides = (1,) + self.strides + (1,) + else: + pool_shape = (1, 1) + self.pool_size + strides = (1, 1) + self.strides + outputs = self.pool_function( + inputs, + ksize=pool_shape, + strides=strides, + padding=self.padding.upper(), + data_format=conv_utils.convert_data_format(self.data_format, 4)) + return outputs + + def compute_output_shape(self, input_shape): + input_shape = tensor_shape.TensorShape(input_shape).as_list() + if self.data_format == 'channels_first': + rows = input_shape[2] + cols = input_shape[3] + else: + rows = input_shape[1] + cols = input_shape[2] + rows = conv_utils.conv_output_length(rows, self.pool_size[0], self.padding, + self.strides[0]) + cols = conv_utils.conv_output_length(cols, self.pool_size[1], self.padding, + self.strides[1]) + if self.data_format == 'channels_first': + return tensor_shape.TensorShape( + [input_shape[0], input_shape[1], rows, cols]) + else: + return tensor_shape.TensorShape( + [input_shape[0], rows, cols, input_shape[3]]) def get_config(self): config = { - 'strides': self.strides, 'pool_size': self.pool_size, - 'padding': self.padding + 'padding': self.padding, + 'strides': self.strides, + 'data_format': self.data_format } - base_config = super(AveragePooling1D, self).get_config() + base_config = super(Pooling2D, self).get_config() return dict(list(base_config.items()) + list(config.items())) @tf_export('keras.layers.MaxPool2D', 'keras.layers.MaxPooling2D') -class MaxPooling2D(tf_pooling_layers.MaxPooling2D, Layer): +class MaxPooling2D(Pooling2D): """Max pooling operation for spatial data. Arguments: @@ -142,26 +301,14 @@ class MaxPooling2D(tf_pooling_layers.MaxPooling2D, Layer): padding='valid', data_format=None, **kwargs): - if data_format is None: - data_format = K.image_data_format() - if strides is None: - strides = pool_size - super(MaxPooling2D, self).__init__(pool_size, strides, padding, data_format, - **kwargs) - - def get_config(self): - config = { - 'pool_size': self.pool_size, - 'padding': self.padding, - 'strides': self.strides, - 'data_format': self.data_format - } - base_config = super(MaxPooling2D, self).get_config() - return dict(list(base_config.items()) + list(config.items())) + super(MaxPooling2D, self).__init__( + nn.max_pool, + pool_size=pool_size, strides=strides, + padding=padding, data_format=data_format, **kwargs) @tf_export('keras.layers.AveragePooling2D', 'keras.layers.AvgPool2D') -class AveragePooling2D(tf_pooling_layers.AveragePooling2D, Layer): +class AveragePooling2D(Pooling2D): """Average pooling operation for spatial data. Arguments: @@ -208,12 +355,96 @@ class AveragePooling2D(tf_pooling_layers.AveragePooling2D, Layer): padding='valid', data_format=None, **kwargs): + super(AveragePooling2D, self).__init__( + nn.avg_pool, + pool_size=pool_size, strides=strides, + padding=padding, data_format=data_format, **kwargs) + + +class Pooling3D(Layer): + """Pooling layer for arbitrary pooling functions, for 3D inputs. + + This class only exists for code reuse. It will never be an exposed API. + + Arguments: + pool_function: The pooling function to apply, e.g. `tf.nn.max_pool`. + pool_size: An integer or tuple/list of 3 integers: + (pool_depth, pool_height, pool_width) + specifying the size of the pooling window. + Can be a single integer to specify the same value for + all spatial dimensions. + strides: An integer or tuple/list of 3 integers, + specifying the strides of the pooling operation. + Can be a single integer to specify the same value for + all spatial dimensions. + padding: A string. The padding method, either 'valid' or 'same'. + Case-insensitive. + data_format: A string, one of `channels_last` (default) or `channels_first`. + The ordering of the dimensions in the inputs. + `channels_last` corresponds to inputs with shape + `(batch, depth, height, width, channels)` + while `channels_first` corresponds to + inputs with shape `(batch, channels, depth, height, width)`. + name: A string, the name of the layer. + """ + + def __init__(self, pool_function, pool_size, strides, + padding='valid', data_format='channels_last', + name=None, **kwargs): + super(Pooling3D, self).__init__(name=name, **kwargs) if data_format is None: - data_format = K.image_data_format() + data_format = backend.image_data_format() if strides is None: strides = pool_size - super(AveragePooling2D, self).__init__(pool_size, strides, padding, - data_format, **kwargs) + self.pool_function = pool_function + self.pool_size = conv_utils.normalize_tuple(pool_size, 3, 'pool_size') + self.strides = conv_utils.normalize_tuple(strides, 3, 'strides') + self.padding = conv_utils.normalize_padding(padding) + self.data_format = conv_utils.normalize_data_format(data_format) + self.input_spec = InputSpec(ndim=5) + + def call(self, inputs): + pool_shape = (1,) + self.pool_size + (1,) + strides = (1,) + self.strides + (1,) + + if self.data_format == 'channels_first': + # TF does not support `channels_first` with 3D pooling operations, + # so we must handle this case manually. + # TODO(fchollet): remove this when TF pooling is feature-complete. + inputs = array_ops.transpose(inputs, (0, 2, 3, 4, 1)) + + outputs = self.pool_function( + inputs, + ksize=pool_shape, + strides=strides, + padding=self.padding.upper()) + + if self.data_format == 'channels_first': + outputs = array_ops.transpose(outputs, (0, 4, 1, 2, 3)) + return outputs + + def compute_output_shape(self, input_shape): + input_shape = tensor_shape.TensorShape(input_shape).as_list() + if self.data_format == 'channels_first': + len_dim1 = input_shape[2] + len_dim2 = input_shape[3] + len_dim3 = input_shape[4] + else: + len_dim1 = input_shape[1] + len_dim2 = input_shape[2] + len_dim3 = input_shape[3] + len_dim1 = conv_utils.conv_output_length(len_dim1, self.pool_size[0], + self.padding, self.strides[0]) + len_dim2 = conv_utils.conv_output_length(len_dim2, self.pool_size[1], + self.padding, self.strides[1]) + len_dim3 = conv_utils.conv_output_length(len_dim3, self.pool_size[2], + self.padding, self.strides[2]) + if self.data_format == 'channels_first': + return tensor_shape.TensorShape( + [input_shape[0], input_shape[1], len_dim1, len_dim2, len_dim3]) + else: + return tensor_shape.TensorShape( + [input_shape[0], len_dim1, len_dim2, len_dim3, input_shape[4]]) def get_config(self): config = { @@ -222,12 +453,12 @@ class AveragePooling2D(tf_pooling_layers.AveragePooling2D, Layer): 'strides': self.strides, 'data_format': self.data_format } - base_config = super(AveragePooling2D, self).get_config() + base_config = super(Pooling3D, self).get_config() return dict(list(base_config.items()) + list(config.items())) @tf_export('keras.layers.MaxPool3D', 'keras.layers.MaxPooling3D') -class MaxPooling3D(tf_pooling_layers.MaxPooling3D, Layer): +class MaxPooling3D(Pooling3D): """Max pooling operation for 3D data (spatial or spatio-temporal). Arguments: @@ -270,26 +501,14 @@ class MaxPooling3D(tf_pooling_layers.MaxPooling3D, Layer): padding='valid', data_format=None, **kwargs): - if data_format is None: - data_format = K.image_data_format() - if strides is None: - strides = pool_size - super(MaxPooling3D, self).__init__(pool_size, strides, padding, data_format, - **kwargs) - - def get_config(self): - config = { - 'pool_size': self.pool_size, - 'padding': self.padding, - 'strides': self.strides, - 'data_format': self.data_format - } - base_config = super(MaxPooling3D, self).get_config() - return dict(list(base_config.items()) + list(config.items())) + super(MaxPooling3D, self).__init__( + nn.max_pool3d, + pool_size=pool_size, strides=strides, + padding=padding, data_format=data_format, **kwargs) @tf_export('keras.layers.AveragePooling3D', 'keras.layers.AvgPool3D') -class AveragePooling3D(tf_pooling_layers.AveragePooling3D, Layer): +class AveragePooling3D(Pooling3D): """Average pooling operation for 3D data (spatial or spatio-temporal). Arguments: @@ -332,30 +551,18 @@ class AveragePooling3D(tf_pooling_layers.AveragePooling3D, Layer): padding='valid', data_format=None, **kwargs): - if data_format is None: - data_format = K.image_data_format() - if strides is None: - strides = pool_size - super(AveragePooling3D, self).__init__(pool_size, strides, padding, - data_format, **kwargs) - - def get_config(self): - config = { - 'pool_size': self.pool_size, - 'padding': self.padding, - 'strides': self.strides, - 'data_format': self.data_format - } - base_config = super(AveragePooling3D, self).get_config() - return dict(list(base_config.items()) + list(config.items())) + super(AveragePooling3D, self).__init__( + nn.avg_pool3d, + pool_size=pool_size, strides=strides, + padding=padding, data_format=data_format, **kwargs) -class _GlobalPooling1D(Layer): +class GlobalPooling1D(Layer): """Abstract class for different global pooling 1D layers. """ def __init__(self, **kwargs): - super(_GlobalPooling1D, self).__init__(**kwargs) + super(GlobalPooling1D, self).__init__(**kwargs) self.input_spec = InputSpec(ndim=3) def compute_output_shape(self, input_shape): @@ -368,7 +575,7 @@ class _GlobalPooling1D(Layer): @tf_export('keras.layers.GlobalAveragePooling1D', 'keras.layers.GlobalAvgPool1D') -class GlobalAveragePooling1D(_GlobalPooling1D): +class GlobalAveragePooling1D(GlobalPooling1D): """Global average pooling operation for temporal data. Input shape: @@ -380,11 +587,11 @@ class GlobalAveragePooling1D(_GlobalPooling1D): """ def call(self, inputs): - return K.mean(inputs, axis=1) + return backend.mean(inputs, axis=1) @tf_export('keras.layers.GlobalMaxPool1D', 'keras.layers.GlobalMaxPooling1D') -class GlobalMaxPooling1D(_GlobalPooling1D): +class GlobalMaxPooling1D(GlobalPooling1D): """Global max pooling operation for temporal data. Input shape: @@ -396,15 +603,15 @@ class GlobalMaxPooling1D(_GlobalPooling1D): """ def call(self, inputs): - return K.max(inputs, axis=1) + return backend.max(inputs, axis=1) -class _GlobalPooling2D(Layer): +class GlobalPooling2D(Layer): """Abstract class for different global pooling 2D layers. """ def __init__(self, data_format=None, **kwargs): - super(_GlobalPooling2D, self).__init__(**kwargs) + super(GlobalPooling2D, self).__init__(**kwargs) self.data_format = conv_utils.normalize_data_format(data_format) self.input_spec = InputSpec(ndim=4) @@ -420,13 +627,13 @@ class _GlobalPooling2D(Layer): def get_config(self): config = {'data_format': self.data_format} - base_config = super(_GlobalPooling2D, self).get_config() + base_config = super(GlobalPooling2D, self).get_config() return dict(list(base_config.items()) + list(config.items())) @tf_export('keras.layers.GlobalAveragePooling2D', 'keras.layers.GlobalAvgPool2D') -class GlobalAveragePooling2D(_GlobalPooling2D): +class GlobalAveragePooling2D(GlobalPooling2D): """Global average pooling operation for spatial data. Arguments: @@ -456,13 +663,13 @@ class GlobalAveragePooling2D(_GlobalPooling2D): def call(self, inputs): if self.data_format == 'channels_last': - return K.mean(inputs, axis=[1, 2]) + return backend.mean(inputs, axis=[1, 2]) else: - return K.mean(inputs, axis=[2, 3]) + return backend.mean(inputs, axis=[2, 3]) @tf_export('keras.layers.GlobalMaxPool2D', 'keras.layers.GlobalMaxPooling2D') -class GlobalMaxPooling2D(_GlobalPooling2D): +class GlobalMaxPooling2D(GlobalPooling2D): """Global max pooling operation for spatial data. Arguments: @@ -492,17 +699,17 @@ class GlobalMaxPooling2D(_GlobalPooling2D): def call(self, inputs): if self.data_format == 'channels_last': - return K.max(inputs, axis=[1, 2]) + return backend.max(inputs, axis=[1, 2]) else: - return K.max(inputs, axis=[2, 3]) + return backend.max(inputs, axis=[2, 3]) -class _GlobalPooling3D(Layer): +class GlobalPooling3D(Layer): """Abstract class for different global pooling 3D layers. """ def __init__(self, data_format=None, **kwargs): - super(_GlobalPooling3D, self).__init__(**kwargs) + super(GlobalPooling3D, self).__init__(**kwargs) self.data_format = conv_utils.normalize_data_format(data_format) self.input_spec = InputSpec(ndim=5) @@ -518,13 +725,13 @@ class _GlobalPooling3D(Layer): def get_config(self): config = {'data_format': self.data_format} - base_config = super(_GlobalPooling3D, self).get_config() + base_config = super(GlobalPooling3D, self).get_config() return dict(list(base_config.items()) + list(config.items())) @tf_export('keras.layers.GlobalAveragePooling3D', 'keras.layers.GlobalAvgPool3D') -class GlobalAveragePooling3D(_GlobalPooling3D): +class GlobalAveragePooling3D(GlobalPooling3D): """Global Average pooling operation for 3D data. Arguments: @@ -554,13 +761,13 @@ class GlobalAveragePooling3D(_GlobalPooling3D): def call(self, inputs): if self.data_format == 'channels_last': - return K.mean(inputs, axis=[1, 2, 3]) + return backend.mean(inputs, axis=[1, 2, 3]) else: - return K.mean(inputs, axis=[2, 3, 4]) + return backend.mean(inputs, axis=[2, 3, 4]) @tf_export('keras.layers.GlobalMaxPool3D', 'keras.layers.GlobalMaxPooling3D') -class GlobalMaxPooling3D(_GlobalPooling3D): +class GlobalMaxPooling3D(GlobalPooling3D): """Global Max pooling operation for 3D data. Arguments: @@ -590,9 +797,9 @@ class GlobalMaxPooling3D(_GlobalPooling3D): def call(self, inputs): if self.data_format == 'channels_last': - return K.max(inputs, axis=[1, 2, 3]) + return backend.max(inputs, axis=[1, 2, 3]) else: - return K.max(inputs, axis=[2, 3, 4]) + return backend.max(inputs, axis=[2, 3, 4]) # Aliases diff --git a/tensorflow/python/keras/_impl/keras/layers/recurrent_test.py b/tensorflow/python/keras/_impl/keras/layers/recurrent_test.py index 641b563a25..4c68c18825 100644 --- a/tensorflow/python/keras/_impl/keras/layers/recurrent_test.py +++ b/tensorflow/python/keras/_impl/keras/layers/recurrent_test.py @@ -435,8 +435,8 @@ class RNNTest(test.TestCase): cells[0].add_update(update_1, inputs=x) cells[0].add_update(update_2) self.assertEqual(len(layer.updates), 2) - self.assertEqual(layer.get_updates_for(None), [update_2]) - self.assertEqual(layer.get_updates_for(x), [update_1]) + self.assertEqual(len(layer.get_updates_for(None)), 1) + self.assertEqual(len(layer.get_updates_for(x)), 1) def test_rnn_dynamic_trainability(self): layer_class = keras.layers.SimpleRNN diff --git a/tensorflow/python/keras/_impl/keras/layers/wrappers.py b/tensorflow/python/keras/_impl/keras/layers/wrappers.py index c510e464ae..9aee5f03b6 100644 --- a/tensorflow/python/keras/_impl/keras/layers/wrappers.py +++ b/tensorflow/python/keras/_impl/keras/layers/wrappers.py @@ -23,11 +23,11 @@ import copy from tensorflow.python.framework import tensor_shape from tensorflow.python.keras._impl.keras import backend as K +from tensorflow.python.keras._impl.keras.engine import base_layer from tensorflow.python.keras._impl.keras.engine import InputSpec from tensorflow.python.keras._impl.keras.engine import Layer from tensorflow.python.keras._impl.keras.engine.base_layer import shape_type_conversion from tensorflow.python.keras._impl.keras.utils.generic_utils import has_arg -from tensorflow.python.layers import utils as tf_layers_util from tensorflow.python.ops import array_ops from tensorflow.python.util.tf_export import tf_export @@ -213,7 +213,7 @@ class TimeDistributed(Wrapper): input_length = array_ops.shape(inputs)[1] # Shape: (num_samples * timesteps, ...). And track the # transformation in self._input_map. - input_uid = tf_layers_util.object_list_uid(inputs) + input_uid = base_layer.object_list_uid(inputs) inputs = array_ops.reshape(inputs, (-1,) + input_shape[2:]) self._input_map[input_uid] = inputs # (num_samples * timesteps, ...) diff --git a/tensorflow/python/keras/_impl/keras/model_subclassing_test.py b/tensorflow/python/keras/_impl/keras/model_subclassing_test.py index 4445900330..bc8698f235 100644 --- a/tensorflow/python/keras/_impl/keras/model_subclassing_test.py +++ b/tensorflow/python/keras/_impl/keras/model_subclassing_test.py @@ -607,12 +607,6 @@ class CustomCallSignatureTests(test.TestCase): self.assertAllClose(10. * expected_output, self.evaluate(output)) output = model(first, second=second, training=False) self.assertAllClose(expected_output, self.evaluate(output)) - if not context.executing_eagerly(): - six.assertCountEqual(self, [first, second], model.inputs) - with self.assertRaises(TypeError): - # tf.layers.Layer expects an "inputs" argument, so all-keywords doesn't - # work at the moment. - model(first=first, second=second, fiddle_with_output='yes') @test_util.run_in_graph_and_eager_modes() def test_inputs_in_signature(self): @@ -622,10 +616,14 @@ class CustomCallSignatureTests(test.TestCase): def call(self, inputs, some_other_arg, training=False): return inputs + def compute_output_shape(self, input_shape): + return input_shape + model = HasInputsAndOtherPositional() with self.assertRaisesRegexp( TypeError, 'everything else as a keyword argument'): - model(array_ops.ones([]), array_ops.ones([])) + x1, x2 = keras.Input((1, 1)), keras.Input((1, 1)) + model(x1, x2) @test_util.run_in_graph_and_eager_modes() def test_kwargs_in_signature(self): @@ -649,13 +647,14 @@ class CustomCallSignatureTests(test.TestCase): def call(self, x, *args, **kwargs): return [x] + list(args) + def compute_output_shape(self, input_shape): + return input_shape + model = HasArgs() - arg1 = array_ops.ones([]) - arg2 = array_ops.ones([]) - arg3 = array_ops.ones([]) - model(arg1, arg2, arg3, a=3) + x1, x2, x3 = keras.Input((1, 1)), keras.Input((1, 1)), keras.Input((1, 1)) + model(x1, x2, x3, a=3) if not context.executing_eagerly(): - six.assertCountEqual(self, [arg1, arg2, arg3], model.inputs) + six.assertCountEqual(self, [x1, x2, x3], model.inputs) def test_args_and_keywords_in_signature(self): @@ -666,11 +665,9 @@ class CustomCallSignatureTests(test.TestCase): with context.graph_mode(): model = HasArgs() - arg1 = array_ops.ones([]) - arg2 = array_ops.ones([]) - arg3 = array_ops.ones([]) + x1, x2, x3 = keras.Input((1, 1)), keras.Input((1, 1)), keras.Input((1, 1)) with self.assertRaisesRegexp(TypeError, 'args and arguments with'): - model(arg1, arg2, arg3, a=3) + model(x1, x2, x3, a=3) def test_training_no_default(self): @@ -694,11 +691,9 @@ class CustomCallSignatureTests(test.TestCase): with context.graph_mode(): model = TrainingNoDefaultWithPositional() - arg1 = array_ops.ones([]) - arg2 = array_ops.ones([]) - arg3 = array_ops.ones([]) + x1, x2, x3 = keras.Input((1, 1)), keras.Input((1, 1)), keras.Input((1, 1)) with self.assertRaisesRegexp(TypeError, 'after a non-input'): - model(arg1, arg2, arg3) + model(x1, x2, x3) if __name__ == '__main__': test.main() diff --git a/tensorflow/python/keras/_impl/keras/utils/conv_utils.py b/tensorflow/python/keras/_impl/keras/utils/conv_utils.py index 583079d962..8882a3a46b 100644 --- a/tensorflow/python/keras/_impl/keras/utils/conv_utils.py +++ b/tensorflow/python/keras/_impl/keras/utils/conv_utils.py @@ -21,17 +21,146 @@ from __future__ import print_function import numpy as np from six.moves import range # pylint: disable=redefined-builtin -# pylint: disable=unused-import -from tensorflow.python.keras._impl.keras import backend as K -from tensorflow.python.layers.utils import conv_input_length -from tensorflow.python.layers.utils import conv_output_length -from tensorflow.python.layers.utils import deconv_output_length as deconv_length -from tensorflow.python.layers.utils import normalize_tuple +from tensorflow.python.keras._impl.keras import backend + + +def convert_data_format(data_format, ndim): + if data_format == 'channels_last': + if ndim == 3: + return 'NWC' + elif ndim == 4: + return 'NHWC' + elif ndim == 5: + return 'NDHWC' + else: + raise ValueError('Input rank not supported:', ndim) + elif data_format == 'channels_first': + if ndim == 3: + return 'NCW' + elif ndim == 4: + return 'NCHW' + elif ndim == 5: + return 'NCDHW' + else: + raise ValueError('Input rank not supported:', ndim) + else: + raise ValueError('Invalid data_format:', data_format) + + +def normalize_tuple(value, n, name): + """Transforms a single integer or iterable of integers into an integer tuple. + + Arguments: + value: The value to validate and convert. Could an int, or any iterable + of ints. + n: The size of the tuple to be returned. + name: The name of the argument being validated, e.g. "strides" or + "kernel_size". This is only used to format error messages. + + Returns: + A tuple of n integers. + + Raises: + ValueError: If something else than an int/long or iterable thereof was + passed. + """ + if isinstance(value, int): + return (value,) * n + else: + try: + value_tuple = tuple(value) + except TypeError: + raise ValueError('The `' + name + '` argument must be a tuple of ' + + str(n) + ' integers. Received: ' + str(value)) + if len(value_tuple) != n: + raise ValueError('The `' + name + '` argument must be a tuple of ' + + str(n) + ' integers. Received: ' + str(value)) + for single_value in value_tuple: + try: + int(single_value) + except (ValueError, TypeError): + raise ValueError('The `' + name + '` argument must be a tuple of ' + + str(n) + ' integers. Received: ' + str(value) + ' ' + 'including element ' + str(single_value) + ' of type' + + ' ' + str(type(single_value))) + return value_tuple + + +def conv_output_length(input_length, filter_size, padding, stride, dilation=1): + """Determines output length of a convolution given input length. + + Arguments: + input_length: integer. + filter_size: integer. + padding: one of "same", "valid", "full". + stride: integer. + dilation: dilation rate, integer. + + Returns: + The output length (integer). + """ + if input_length is None: + return None + assert padding in {'same', 'valid', 'full'} + dilated_filter_size = filter_size + (filter_size - 1) * (dilation - 1) + if padding == 'same': + output_length = input_length + elif padding == 'valid': + output_length = input_length - dilated_filter_size + 1 + elif padding == 'full': + output_length = input_length + dilated_filter_size - 1 + return (output_length + stride - 1) // stride + + +def conv_input_length(output_length, filter_size, padding, stride): + """Determines input length of a convolution given output length. + + Arguments: + output_length: integer. + filter_size: integer. + padding: one of "same", "valid", "full". + stride: integer. + + Returns: + The input length (integer). + """ + if output_length is None: + return None + assert padding in {'same', 'valid', 'full'} + if padding == 'same': + pad = filter_size // 2 + elif padding == 'valid': + pad = 0 + elif padding == 'full': + pad = filter_size - 1 + return (output_length - 1) * stride - 2 * pad + filter_size + + +def deconv_output_length(input_length, filter_size, padding, stride): + """Determines output length of a transposed convolution given input length. + + Arguments: + input_length: integer. + filter_size: integer. + padding: one of "same", "valid", "full". + stride: integer. + + Returns: + The output length (integer). + """ + if input_length is None: + return None + input_length *= stride + if padding == 'valid': + input_length += max(filter_size - stride, 0) + elif padding == 'full': + input_length -= (stride + filter_size - 2) + return input_length def normalize_data_format(value): if value is None: - value = K.image_data_format() + value = backend.image_data_format() data_format = value.lower() if data_format not in {'channels_first', 'channels_last'}: raise ValueError('The `data_format` argument must be one of ' diff --git a/tensorflow/python/keras/_impl/keras/utils/tf_utils.py b/tensorflow/python/keras/_impl/keras/utils/tf_utils.py new file mode 100644 index 0000000000..8da5f77777 --- /dev/null +++ b/tensorflow/python/keras/_impl/keras/utils/tf_utils.py @@ -0,0 +1,74 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""TensorFlow-related utilities.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.framework import smart_cond as smart_module +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import variables + + +def smart_cond(pred, true_fn=None, false_fn=None, name=None): + """Return either `true_fn()` if predicate `pred` is true else `false_fn()`. + + If `pred` is a bool or has a constant value, we return either `true_fn()` + or `false_fn()`, otherwise we use `tf.cond` to dynamically route to both. + + Arguments: + pred: A scalar determining whether to return the result of `true_fn` or + `false_fn`. + true_fn: The callable to be performed if pred is true. + false_fn: The callable to be performed if pred is false. + name: Optional name prefix when using `tf.cond`. + + Returns: + Tensors returned by the call to either `true_fn` or `false_fn`. + + Raises: + TypeError: If `true_fn` or `false_fn` is not callable. + """ + if isinstance(pred, variables.Variable): + return control_flow_ops.cond( + pred, true_fn=true_fn, false_fn=false_fn, name=name) + return smart_module.smart_cond( + pred, true_fn=true_fn, false_fn=false_fn, name=name) + + +def constant_value(pred): + """Return the bool value for `pred`, or None if `pred` had a dynamic value. + + Arguments: + pred: A scalar, either a Python bool or a TensorFlow boolean variable + or tensor, or the Python integer 1 or 0. + + Returns: + True or False if `pred` has a constant boolean value, None otherwise. + + Raises: + TypeError: If `pred` is not a Variable, Tensor or bool, or Python + integer 1 or 0. + """ + # Allow integer booleans. + if isinstance(pred, int): + if pred == 1: + pred = True + elif pred == 0: + pred = False + + if isinstance(pred, variables.Variable): + return None + return smart_module.smart_constant_value(pred) diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py index ec741d3265..64db49c900 100644 --- a/tensorflow/python/layers/base.py +++ b/tensorflow/python/layers/base.py @@ -12,148 +12,91 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================= - -# pylint: disable=unused-import,g-bad-import-order """Contains the base Layer class, from which all layers inherit.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function -import collections import copy -import re -import weakref -import numpy as np from tensorflow.python.eager import context from tensorflow.python.estimator import util as estimator_util from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.layers import utils as layers_util -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops +from tensorflow.python.keras._impl.keras.engine import base_layer from tensorflow.python.ops import variable_scope as vs from tensorflow.python.ops import variables as tf_variables -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import checkpointable from tensorflow.python.util import nest from tensorflow.python.util.tf_export import tf_export -@tf_export('layers.Layer') -class Layer(checkpointable.CheckpointableBase): - """Base layer class. +InputSpec = base_layer.InputSpec # pylint: disable=invalid-name - This is the class from which all layers inherit, implementing common - infrastructure functionality. - A layer is a class implementing common neural networks operations, such - as convolution, batch norm, etc. These operations require managing variables, - losses, and updates, as well as applying TensorFlow ops to input tensors. +@tf_export('layers.Layer') +class Layer(base_layer.Layer): + """Base layer class. - Users will just instantiate it and then treat it as a callable. + It is considered legacy, and we recommend the use of `tf.keras.layers.Layer` + instead. - We recommend that descendants of Layer implement the following methods: - * `__init__()`: Save configuration in member variables - * `build()`: Called once from `__call__`, when we know the shapes of inputs - and `dtype`. Should have the calls to `add_variable()`, and then - call the super's `build()` (which sets `self.built = True`, which is - nice in case the user wants to call `build()` manually before the - first `__call__`). - * `call()`: Called in `__call__` after making sure `build()` has been called - once. Should actually perform the logic of applying the layer to the - input tensors (which should be passed in as the first argument). + Arguments: + trainable: Boolean, whether the layer's variables should be trainable. + name: String name of the layer. + dtype: Default dtype of the layer's weights (default of `None` means use the + type of the first input). Read-only properties: - `name`: The name of the layer (string). - `dtype`: Default dtype of the layer (default of `None` means use the + name: The name of the layer (string). + dtype: Default dtype of the layer's weights (default of `None` means use the type of the first input). - `trainable_variables`: List of trainable variables. - `non_trainable_variables`: List of non-trainable variables. - `variables`: List of all variables of this layer, trainable and + trainable_variables: List of trainable variables. + non_trainable_variables: List of non-trainable variables. + variables: List of all variables of this layer, trainable and non-trainable. - `updates`: List of update ops of this layer. - `losses`: List of losses added by this layer. + updates: List of update ops of this layer. + losses: List of losses added by this layer. + trainable_weights: List of variables to be included in backprop. + non_trainable_weights: List of variables that should not be + included in backprop. + weights: The concatenation of the lists trainable_weights and + non_trainable_weights (in this order). Mutable properties: - `trainable`: Whether the layer should be trained (boolean). - `input_spec`: Optional (list of) `InputSpec` object(s) specifying the + trainable: Whether the layer should be trained (boolean). + input_spec: Optional (list of) `InputSpec` object(s) specifying the constraints on inputs that can be accepted by the layer. """ def __init__(self, trainable=True, name=None, dtype=None, - activity_regularizer=None, **kwargs): - # We use a kwargs dict here because these kwargs only exist - # for compatibility reasons. - # The list of kwargs is subject to changes in the future. - # We do not want to commit to it or to expose the list to users at all. - # Note this is exactly as safe as defining kwargs in the function signature, - # the only difference being that the list of valid kwargs is defined - # below rather rather in the signature, and default values are defined - # in calls to kwargs.get(). - allowed_kwargs = { - '_scope', - '_reuse', - 'input_shape', # For compatibility with Keras `Sequential` model. - 'batch_size', # For compatibility with Keras `Sequential` model. - } - for kwarg in kwargs: - if kwarg not in allowed_kwargs: - raise TypeError('Keyword argument not understood:', kwarg) - - # Mutable properties - # Indicates whether the layer's weights are updated during training - # and whether the layer's updates are run during training - self.trainable = trainable - # A stateful layer is a layer whose updates are run during inference too, - # for instance stateful RNNs. - self.stateful = False - # Indicates whether `build` needs to be called upon layer call, to create - # the layer's weights. - self.built = False - # Provides information about which inputs are compatible with the layer. - self.input_spec = None - - if activity_regularizer and context.executing_eagerly(): - raise ValueError( - ('Activity regularization is not supported when executing eagerly. ' - 'Got activity_regularizer=%s') % (activity_regularizer,)) - self._activity_regularizer = activity_regularizer + **kwargs): + # For backwards compatibility, legacy layers do not use `ResourceVariable` + # by default. + self._use_resource_variables = False + scope = kwargs.pop('_scope', None) + self._reuse = kwargs.pop('_reuse', None) + + # Avoid an incorrect lint error self._trainable_weights = [] - self._non_trainable_weights = [] - self._updates = [] - # When executing eagerly, _losses is a list of zero-argument lambdas which - # return tensors. When using graph execution, _losses is a list of ops. - self._losses = [] - self._reuse = kwargs.get('_reuse') - self._graph = None # Will be set at build time. - self._dtype = None if dtype is None else dtypes.as_dtype(dtype).name - self._call_fn_args = estimator_util.fn_args(self.call) - self._compute_previous_mask = ('mask' in self._call_fn_args or - hasattr(self, 'compute_mask')) - self._call_has_scope_arg = 'scope' in self._call_fn_args - - # These lists will be filled via successive calls - # to self._add_inbound_node(). - self._inbound_nodes = [] - self._outbound_nodes = [] + self.built = False - self._init_set_name(name) + super(Layer, self).__init__(trainable=trainable, name=name, dtype=dtype, + **kwargs) - # Determine variable scope. - scope = kwargs.get('_scope') + self._graph = None + self._call_has_scope_arg = 'scope' in self._call_fn_args if scope: with vs.variable_scope(scope) as captured_scope: self._scope = captured_scope else: self._scope = None + self._current_scope = None - # Set `_batch_input_shape` attribute - # for compatibility with Keras `Sequential` model. - if 'input_shape' in kwargs: - batch_size = kwargs.get('batch_size') - self._batch_input_shape = (batch_size,) + tuple(kwargs['input_shape']) + @property + def graph(self): + if context.executing_eagerly(): + raise RuntimeError('Layer.graph not supported when executing eagerly.') + return self._graph def _init_set_name(self, name): # Determine layer name (non-unique). @@ -166,18 +109,15 @@ class Layer(checkpointable.CheckpointableBase): self._name, base_name = self._make_unique_name() self._base_name = base_name - @property - def dtype(self): - return self._dtype - - @property - def name(self): - return self._name - - @property - def activity_regularizer(self): - """Optional regularizer function for the output of this layer.""" - return self._activity_regularizer + def _make_unique_name(self, name_uid_map=None, avoid_names=None, + namespace='', zero_based=False): + base_name = base_layer.to_snake_case(self.__class__.__name__) + name = base_layer.unique_layer_name(base_name, + name_uid_map=name_uid_map, + avoid_names=avoid_names, + namespace=namespace, + zero_based=zero_based) + return (name, base_name) @property def scope_name(self): @@ -189,271 +129,16 @@ class Layer(checkpointable.CheckpointableBase): 'querying `scope_name`.') return self._scope.name - @property - def trainable_weights(self): - return self._trainable_weights if self.trainable else [] - - @property - def non_trainable_weights(self): - if self.trainable: - return self._non_trainable_weights - else: - return self._trainable_weights + self._non_trainable_weights - - @property - def trainable_variables(self): - return self.trainable_weights - - @property - def non_trainable_variables(self): - return self.non_trainable_weights - - @property - def weights(self): - """Returns the list of all layer variables/weights. - - Returns: - A list of variables. - """ - return self.trainable_weights + self.non_trainable_weights - - @property - def variables(self): - """Returns the list of all layer variables/weights. - - Returns: - A list of variables. - """ - return self.weights - - @property - def updates(self): - if context.executing_eagerly(): - raise RuntimeError('Layer.updates not supported in Eager mode.') - if not self.trainable and not self.stateful: - return [] - return self._updates - - def add_update(self, updates, inputs=None): - """Add update op(s), potentially dependent on layer inputs. - - Weight updates (for instance, the updates of the moving mean and variance - in a BatchNormalization layer) may be dependent on the inputs passed - when calling a layer. Hence, when reusing the same layer on - different inputs `a` and `b`, some entries in `layer.updates` may be - dependent on `a` and some on `b`. This method automatically keeps track - of dependencies. - - The `get_updates_for` method allows to retrieve the updates relevant to a - specific set of inputs. - - This call is ignored in Eager mode. - - Arguments: - updates: Update op, or list/tuple of update ops. - inputs: If anything other than None is passed, it signals the updates - are conditional on some of the layer's inputs, - and thus they should only be run where these inputs are available. - This is the case for BatchNormalization updates, for instance. - If None, the updates will be taken into account unconditionally, - and you are responsible for making sure that any dependency they might - have is available at runtime. - A step counter might fall into this category. - """ - if context.executing_eagerly(): - return # Updates already applied when in eager mode. - - updates = _to_list(updates) - updates = [x if isinstance(x, ops.Operation) - else ops.convert_to_tensor(x) for x in updates] - self._updates += updates - if inputs is None: - for u in updates: - u._unconditional_update = True # pylint: disable=protected-access - else: - for u in updates: - u._unconditional_update = False # pylint: disable=protected-access - - def get_updates_for(self, inputs): - """Retrieves updates relevant to a specific set of inputs. - - Arguments: - inputs: Input tensor or list/tuple of input tensors. - - Returns: - List of update ops of the layer that depend on `inputs`. - - Raises: - RuntimeError: If called in Eager mode. - """ - if context.executing_eagerly(): - raise RuntimeError('`get_updates_for()` not supported in Eager mode.') - - # Updates disabled if layer is not trainable and not explicitly stateful. - if not self.trainable and not self.stateful: - return [] - - if inputs is None: - # Requesting unconditional updates. - return [x for x in self.updates if x._unconditional_update] # pylint: disable=protected-access - - # Requesting input-conditional updates. - inputs = nest.flatten(inputs) - reachable = layers_util.get_reachable_from_inputs(inputs, self.updates) - updates = [] - for update in self.updates: - if update in reachable: - updates.append(update) - return updates - - @property - def losses(self): - """Losses which are associated with this `Layer`. - - Note that when executing eagerly, getting this property evaluates - regularizers. When using graph execution, variable regularization ops have - already been created and are simply returned here. - - Returns: - A list of tensors. - """ - if context.executing_eagerly(): - # _losses may only contain variable regularization losses when executing - # eagerly, and they have been saved as lambdas to be executed when - # requested. - return [regularizer() for regularizer in self._losses] - else: - return self._losses - def add_loss(self, losses, inputs=None): - """Add loss tensor(s), potentially dependent on layer inputs. - - Some losses (for instance, activity regularization losses) may be dependent - on the inputs passed when calling a layer. Hence, when reusing the same - layer on different inputs `a` and `b`, some entries in `layer.losses` may - be dependent on `a` and some on `b`. This method automatically keeps track - of dependencies. - - The `get_losses_for` method allows to retrieve the losses relevant to a - specific set of inputs. - - Note that `add_loss` is not supported when executing eagerly. Instead, - variable regularizers may be added through `add_variable`. Activity - regularization is not supported directly (but such losses may be returned - from `Layer.call()`). - - Arguments: - losses: Loss tensor, or list/tuple of tensors. - inputs: If anything other than None is passed, it signals the losses - are conditional on some of the layer's inputs, - and thus they should only be run where these inputs are available. - This is the case for activity regularization losses, for instance. - If `None` is passed, the losses are assumed - to be unconditional, and will apply across all dataflows of the layer - (e.g. weight regularization losses). - - Raises: - RuntimeError: If called in Eager mode. - """ - if context.executing_eagerly(): - # TODO(fchollet): it should be possible (and highly desirable) to support - # `add_loss` in eager mode. This allows great convenience and flexibility - # in defining custom losses on the fly (e.g. in VAEs). - # Simply appending the loss value to `self._losses` - # is the correct behavior. - # The only caveat is that we need to force the user to only call - # `add_loss` from inside a model or Layer's `call` method - # (otherwise the loss computation cannot be backproped through). - raise RuntimeError('Layer.add_loss not supported in Eager mode.') - - losses = _to_list(losses) - self._losses += losses - if inputs is None: - for loss in losses: - loss._unconditional_loss = True # pylint: disable=protected-access - else: - for loss in losses: - loss._unconditional_loss = False # pylint: disable=protected-access + previous_losses_length = len(self._losses) + super(Layer, self).add_loss(losses, inputs=inputs) # TODO(fchollet): deprecate collection below. - _add_elements_to_collection(losses, ops.GraphKeys.REGULARIZATION_LOSSES) - - def get_losses_for(self, inputs): - """Retrieves losses relevant to a specific set of inputs. - - Arguments: - inputs: Input tensor or list/tuple of input tensors. - - Returns: - List of loss tensors of the layer that depend on `inputs`. - - Raises: - RuntimeError: If called in Eager mode. - """ - if context.executing_eagerly(): - raise RuntimeError('Layer.get_losses_for not supported in Eager mode.') - - if inputs is None: - # Requesting unconditional losses. - return [x for x in self.losses if x._unconditional_loss] # pylint: disable=protected-access - - # Requesting input-conditional losses. - inputs = nest.flatten(inputs) - # Retrieve the set of tensors in the TF graph that depend on `inputs`. - # The losses we want to return will be part of this set. - # To avoid unnecessary work, we stop the search in case all of - # `self.losses` have been retrieved. - reachable = layers_util.get_reachable_from_inputs(inputs, self.losses) - losses = [] - for loss in self.losses: - if loss in reachable: - losses.append(loss) - return losses - - def build(self, _): - """Creates the variables of the layer.""" - self.built = True - - def call(self, inputs, **kwargs): # pylint: disable=unused-argument - """The logic of the layer lives here. + new_losses = self._losses[previous_losses_length:] + _add_elements_to_collection(new_losses, ops.GraphKeys.REGULARIZATION_LOSSES) - Arguments: - inputs: input tensor(s). - **kwargs: additional keyword arguments. - - Returns: - Output tensor(s). - """ - return inputs - - def _name_scope_name(self, current_variable_scope): + def _name_scope(self): """Determines op naming for the Layer.""" - return current_variable_scope.original_name_scope - - def compute_output_shape(self, input_shape): - """Computes the output shape of the layer given the input shape. - - Args: - input_shape: A (possibly nested tuple of) `TensorShape`. It need not - be fully defined (e.g. the batch size may be unknown). - - Returns: - A (possibly nested tuple of) `TensorShape`. - - Raises: - TypeError: if `input_shape` is not a (possibly nested tuple of) - `TensorShape`. - ValueError: if `input_shape` is incomplete or is incompatible with the - the layer. - """ - raise NotImplementedError - - def _make_unique_name(self, name_uid_map=None, avoid_names=None, - namespace='', zero_based=False): - base_name = _to_snake_case(self.__class__.__name__) - name = _unique_layer_name(base_name, name_uid_map=name_uid_map, - avoid_names=avoid_names, namespace=namespace, - zero_based=zero_based) - return (name, base_name) + return self._current_scope.original_name_scope def _set_scope(self, scope=None): if self._scope is None: @@ -467,10 +152,11 @@ class Layer(checkpointable.CheckpointableBase): scope, default_name=self._base_name) as captured_scope: self._scope = captured_scope - def add_variable(self, name, shape, dtype=None, - initializer=None, regularizer=None, - trainable=True, constraint=None, - partitioner=None): + def add_weight(self, name, shape, dtype=None, + initializer=None, regularizer=None, + trainable=True, constraint=None, + use_resource=None, + partitioner=None): """Adds a new variable to the layer, or gets an existing one; returns it. Arguments: @@ -486,6 +172,7 @@ class Layer(checkpointable.CheckpointableBase): then this parameter is ignored and any added variables are also marked as non-trainable. constraint: constraint instance (callable). + use_resource: Whether to use `ResourceVariable`. partitioner: (optional) partitioner instance (callable). If provided, when the requested variable is created it will be split into multiple partitions according to `partitioner`. In this case, @@ -504,10 +191,6 @@ class Layer(checkpointable.CheckpointableBase): RuntimeError: If called with partioned variable regularization and eager execution is enabled. """ - - # `init_graph` should point to the graph in which variable initialization - # will occur; it should be None if and only if initialization will take - # place in the eager context. init_graph = None if not context.executing_eagerly(): default_graph = ops.get_default_graph() @@ -530,71 +213,43 @@ class Layer(checkpointable.CheckpointableBase): self._set_scope(None) reuse = self.built or self._reuse + prev_len_trainable = len(self._trainable_weights) with vs.variable_scope( self._scope, reuse=reuse, auxiliary_name_scope=False) as scope: - with ops.name_scope(self._name_scope_name(scope)): - variable = self._add_variable_with_custom_getter( - name=name, - shape=shape, - getter=vs.get_variable, - # Manage errors in Layer rather than Checkpointable. - overwrite=True, - initializer=initializer, + self._current_scope = scope + with ops.name_scope(self._name_scope()): + use_resource = (use_resource or + self._use_resource_variables or + scope.use_resource) + variable = super(Layer, self).add_weight( + name, + shape, dtype=dtypes.as_dtype(dtype), + initializer=initializer or scope.initializer, + trainable=trainable, constraint=constraint, - trainable=trainable and self.trainable, - partitioner=partitioner) - - if init_graph is not None: # pylint: disable=protected-access - # The variable was created and initialized in a graph. - - if variable in existing_variables: - # To match the behavior of tf.get_variable(), we only apply - # regularization if the variable is newly created. - return variable - + partitioner=partitioner, + use_resource=use_resource, + getter=vs.get_variable) + + if regularizer: + if context.executing_eagerly() or variable not in existing_variables: + self._handle_weight_regularization(name, variable, regularizer) + + if init_graph is not None: + # Handle edge case where a custom getter has overridden `trainable`. + # There is one known occurrence of this, in unit test + # testBasicRNNCellNotTrainable in + # contrib.rnn.python.kernel_tests.core_rnn_cell_test with init_graph.as_default(): trainable_variables = tf_variables.trainable_variables() if (trainable and self.trainable and variable not in trainable_variables): # A custom getter / variable scope overrode the trainable flag. - trainable = False - - if regularizer: - if isinstance(variable, tf_variables.PartitionedVariable): - for v in variable: - with ops.colocate_with(v.op): - with ops.name_scope(name + '/Regularizer'): - regularization = regularizer(v) - if regularization is not None: - self.add_loss(regularization) - else: - with ops.colocate_with(variable.op): - with ops.name_scope(name + '/Regularizer'): - regularization = regularizer(variable) - if regularization is not None: - self.add_loss(regularization) - elif regularizer: # and initialization took place in an eager context - if isinstance(variable, tf_variables.PartitionedVariable): - raise RuntimeError( - 'Partitioned variable regularization is not yet ' - 'supported when executing eagerly. File a feature request ' - 'if this is important to you.') - # Save a zero-argument lambda which runs the regularizer on the - # variable, to be executed when `Layer.losses` is requested. - # This makes losses responsive to variable updates when executing - # eagerly. - # - # TODO(akshayka): Do the same for graphs as well, so that losses - # collected in a while_loop can be run outside its control flow - # context and so that losses won't be swallowed up by graph functions - # (i.e., `.losses()` should always create regularizers). - self._losses.append(lambda: regularizer(variable)) - - if trainable: - self._trainable_weights.append(variable) - else: - self._non_trainable_weights.append(variable) + extra_trainable_vars = self._trainable_weights[prev_len_trainable:] + self._trainable_weights = self._trainable_weights[ + :prev_len_trainable] + self._non_trainable_weights += extra_trainable_vars return variable def __call__(self, inputs, *args, **kwargs): @@ -622,35 +277,14 @@ class Layer(checkpointable.CheckpointableBase): ValueError: if the layer's `call` method returns None (an invalid value). """ self._set_scope(kwargs.pop('scope', None)) - input_list = nest.flatten(inputs) - build_graph = not context.executing_eagerly() - # TODO(fchollet, allenl): Make deferred mode work with subclassed Models - # which don't use an "inputs" argument. - in_deferred_mode = isinstance(input_list[0], _DeferredTensor) - # Ensure the Layer, if being reused, is working with inputs from - # the same graph as where it was created. - if build_graph: + if not context.executing_eagerly(): try: # Set layer's "graph" at build time - self._graph = ops._get_graph_from_inputs(input_list, graph=self._graph) # pylint: disable=protected-access + self._graph = ops._get_graph_from_inputs(nest.flatten(inputs), # pylint: disable=protected-access + graph=self._graph) except ValueError as e: raise ValueError('Input graph and Layer graph are not the same: %s' % e) - if build_graph or in_deferred_mode: - user_kwargs = copy.copy(kwargs) - - # Handle Keras mask propagation from previous layer to current layer. - previous_mask = None - if (not hasattr(self, '_compute_previous_mask') or - self._compute_previous_mask): - previous_mask = _collect_previous_mask(inputs) - if not hasattr(self, '_call_fn_args'): - self._call_fn_args = estimator_util.fn_args(self.call) - if ('mask' in self._call_fn_args and 'mask' not in kwargs and - not _is_all_none(previous_mask)): - # The previous layer generated a mask, and mask was not explicitly pass - # to __call__, hence we set previous_mask as the default value. - kwargs['mask'] = previous_mask if self.built: try: @@ -667,134 +301,27 @@ class Layer(checkpointable.CheckpointableBase): else: scope_context_manager = vs.variable_scope( self._scope, reuse=self._reuse, auxiliary_name_scope=False) - input_shapes = None - with scope_context_manager as scope: - with ops.name_scope(self._name_scope_name(scope)): - if not self.built: - if not build_graph: - # Activity regularization is currently unsupported in Eager mode. - if self._activity_regularizer: - raise ValueError( - 'activity_regularizer currently unsupported with ' - 'eager execution enabled. Found an activity_regularizer in ' - '%s(%s).' % (self.__class__.__name__, self)) - if not build_graph and not in_deferred_mode: - # TODO(agarwal): support _keras_history in Eager mode. - for x in input_list: - if hasattr(x, '_keras_history'): - raise ValueError('_keras_history currently unsupported in ' - 'Eager mode. Found _keras_history in %s while ' - 'executing __call__ for %s(%s)' % - (x, self.__class_.__name__, self)) - - # Check input assumptions set before layer building, e.g. input rank. - self._assert_input_compatibility(inputs) - if input_list and self._dtype is None: - try: - self._dtype = input_list[0].dtype.base_dtype.name - except AttributeError: - pass - if all(hasattr(x, 'get_shape') for x in input_list): - input_shapes = nest.map_structure(lambda x: x.get_shape(), inputs) - self.build(input_shapes) - try: - # Note: not all sub-classes of Layer call Layer.__init__ (especially - # the ones under tensorflow/python/keras). Hence we recompute this - # attribute here if it is not set. - # TODO(agarwal): Fix the sub-classes and avoid this complexity. - call_has_scope_arg = self._call_has_scope_arg - except AttributeError: - self._call_fn_args = estimator_util.fn_args(self.call) - self._call_has_scope_arg = 'scope' in self._call_fn_args - call_has_scope_arg = self._call_has_scope_arg - if call_has_scope_arg: - kwargs['scope'] = scope - # Check input assumptions set after layer building, e.g. input shape. - if build_graph or in_deferred_mode: - self._assert_input_compatibility(inputs) - - if not in_deferred_mode: - outputs = self.call(inputs, *args, **kwargs) - if outputs is None: - raise ValueError('A layer\'s `call` method should return a Tensor ' - 'or a list of Tensors, not None.') - else: - # Deferred mode behavior: use `compute_output_shape` to - # infer the number of outputs of the layer and their shapes. - if input_shapes is None: - input_shapes = nest.map_structure(lambda x: x.get_shape(), inputs) - - output_shapes = self.compute_output_shape(input_shapes) - output_shapes = nest.flatten(output_shapes) - outputs = [ - # TODO(fchollet): name the deferred tensors? - _DeferredTensor(shape=shape, dtype=self._dtype) - for shape in output_shapes - ] - if len(outputs) == 1: - outputs = outputs[0] - if build_graph: - # Apply activity regularization. - # Note that it should be applied every time the layer creates a new - # output, since it is output-specific. - if self._activity_regularizer: - output_list = nest.flatten(outputs) - for output in output_list: - with ops.name_scope('ActivityRegularizer'): - activity_regularization = self._activity_regularizer(output) - self.add_loss(activity_regularization, inputs=inputs) + with scope_context_manager as scope: + self._current_scope = scope - # TODO(fchollet): consider enabling masking for Eager mode. - if hasattr(self, 'compute_mask'): - output_mask = self.compute_mask(inputs, previous_mask) - if isinstance(outputs, (list, tuple)): - if output_mask is None: - output_mask = [None for _ in range(len(outputs))] - for x, m in zip(outputs, output_mask): - x._keras_mask = m # pylint: disable=protected-access - else: - outputs._keras_mask = output_mask # pylint: disable=protected-access + try: + call_has_scope_arg = self._call_has_scope_arg + except AttributeError: + self._call_fn_args = estimator_util.fn_args(self.call) + self._call_has_scope_arg = 'scope' in self._call_fn_args + call_has_scope_arg = self._call_has_scope_arg + if call_has_scope_arg: + kwargs['scope'] = scope - if build_graph: - # If all input tensors have history metadata, - # we update the output tensors - # with corresponding history metadata, thus eventually allowing to use - # these tensors to instantiate a Network. - if _have_all_keras_metadata(inputs): - # If the layer returns tensors from its inputs, unmodified, - # we copy them to avoid loss of tensor metadata. - output_ls = nest.flatten(outputs) - output_ls_copy = [] - for x in output_ls: - if x in input_list: - with ops.name_scope(scope.original_name_scope): - x = array_ops.identity(x) - output_ls_copy.append(x) - if len(output_ls_copy) == 1: - outputs = output_ls_copy[0] - else: - outputs = output_ls_copy + # Actually call layer + outputs = super(Layer, self).__call__(inputs, *args, **kwargs) + if not context.executing_eagerly(): # Update global default collections. _add_elements_to_collection(self.updates, ops.GraphKeys.UPDATE_OPS) - - if in_deferred_mode or build_graph: - if _have_all_keras_metadata(inputs): - # Add an inbound node to the layer, so it can keep track of this call. - # This updates the layer history of the output tensor(s). - self._add_inbound_node( - input_tensors=inputs, output_tensors=outputs, arguments=user_kwargs) - - self.built = True return outputs - @property - def graph(self): - if context.executing_eagerly(): - raise RuntimeError('Layer.graph not supported in Eager mode.') - return self._graph - def __deepcopy__(self, memo): no_copy = set(['_graph']) shallow_copy = set(['_scope', '_always_reuse_variable_scope']) @@ -806,658 +333,12 @@ class Layer(checkpointable.CheckpointableBase): setattr(result, k, v) elif k in shallow_copy: setattr(result, k, copy.copy(v)) - elif _is_tensor_or_tensor_list(v): + elif base_layer.is_tensor_or_tensor_list(v): setattr(result, k, v) else: setattr(result, k, copy.deepcopy(v, memo)) return result - def apply(self, inputs, *args, **kwargs): - """Apply the layer on a input. - - This simply wraps `self.__call__`. - - Arguments: - inputs: Input tensor(s). - *args: additional positional arguments to be passed to `self.call`. - **kwargs: additional keyword arguments to be passed to `self.call`. - - Returns: - Output tensor(s). - """ - return self.__call__(inputs, *args, **kwargs) - - def _add_inbound_node(self, - input_tensors, - output_tensors, - arguments=None): - """Internal method to create an inbound node for the layer. - - Arguments: - input_tensors: list of input tensors. - output_tensors: list of output tensors. - arguments: dictionary of keyword arguments that were passed to the - `call` method of the layer at the call that created the node. - """ - input_tensors = nest.flatten(input_tensors) - output_tensors = nest.flatten(output_tensors) - - # Collect input tensor(s) coordinates. - inbound_layers = [] - node_indices = [] - tensor_indices = [] - for x in input_tensors: - assert hasattr(x, '_keras_history') - inbound_layer, node_index, tensor_index = x._keras_history # pylint: disable=protected-access - inbound_layers.append(inbound_layer) - node_indices.append(node_index) - tensor_indices.append(tensor_index) - - # Create node, add it to inbound nodes. - Node( - self, - inbound_layers=inbound_layers, - node_indices=node_indices, - tensor_indices=tensor_indices, - input_tensors=input_tensors, - output_tensors=output_tensors, - arguments=arguments) - - # Update tensor history metadata. - for i in range(len(output_tensors)): - # The metadata attribute consists of 1) a layer instance - # 2) a node index for the layer, 3) a tensor index for the node. - # The allows layer reuse (multiple nodes per layer) and multi-output - # or multi-input layers (e.g. a layer can return multiple tensors, - # and each can be sent to a different layer). - output_tensors[i]._keras_history = (self, len(self._inbound_nodes) - 1, i) # pylint: disable=protected-access - - def _get_node_attribute_at_index(self, node_index, attr, attr_name): - """Private utility to retrieves an attribute (e.g. inputs) from a node. - - This is used to implement the methods: - - get_input_shape_at - - get_output_shape_at - - get_input_at - etc... - - Arguments: - node_index: Integer index of the node from which - to retrieve the attribute. - attr: Exact node attribute name. - attr_name: Human-readable attribute name, for error messages. - - Returns: - The layer's attribute `attr` at the node of index `node_index`. - - Raises: - RuntimeError: If the layer has no inbound nodes, or if called in Eager - mode. - ValueError: If the index provided does not match any node. - """ - if not self._inbound_nodes: - raise RuntimeError('The layer has never been called ' - 'and thus has no defined ' + attr_name + '.') - if not len(self._inbound_nodes) > node_index: - raise ValueError('Asked to get ' + attr_name + ' at node ' + - str(node_index) + ', but the layer has only ' + - str(len(self._inbound_nodes)) + ' inbound nodes.') - values = getattr(self._inbound_nodes[node_index], attr) - if len(values) == 1: - return values[0] - else: - return values - - def get_input_shape_at(self, node_index): - """Retrieves the input shape(s) of a layer at a given node. - - Arguments: - node_index: Integer, index of the node - from which to retrieve the attribute. - E.g. `node_index=0` will correspond to the - first time the layer was called. - - Returns: - A shape tuple - (or list of shape tuples if the layer has multiple inputs). - - Raises: - RuntimeError: If called in Eager mode. - """ - return self._get_node_attribute_at_index(node_index, 'input_shapes', - 'input shape') - - def get_output_shape_at(self, node_index): - """Retrieves the output shape(s) of a layer at a given node. - - Arguments: - node_index: Integer, index of the node - from which to retrieve the attribute. - E.g. `node_index=0` will correspond to the - first time the layer was called. - - Returns: - A shape tuple - (or list of shape tuples if the layer has multiple outputs). - - Raises: - RuntimeError: If called in Eager mode. - """ - if context.executing_eagerly(): - raise RuntimeError( - 'Layer.get_output_shape_at not supported in Eager mode.') - return self._get_node_attribute_at_index(node_index, 'output_shapes', - 'output shape') - - def get_input_at(self, node_index): - """Retrieves the input tensor(s) of a layer at a given node. - - Arguments: - node_index: Integer, index of the node - from which to retrieve the attribute. - E.g. `node_index=0` will correspond to the - first time the layer was called. - - Returns: - A tensor (or list of tensors if the layer has multiple inputs). - - Raises: - RuntimeError: If called in Eager mode. - """ - if context.executing_eagerly(): - raise RuntimeError('Layer.get_input_at not supported in Eager mode.') - return self._get_node_attribute_at_index(node_index, 'input_tensors', - 'input') - - def get_output_at(self, node_index): - """Retrieves the output tensor(s) of a layer at a given node. - - Arguments: - node_index: Integer, index of the node - from which to retrieve the attribute. - E.g. `node_index=0` will correspond to the - first time the layer was called. - - Returns: - A tensor (or list of tensors if the layer has multiple outputs). - - Raises: - RuntimeError: If called in Eager mode. - """ - return self._get_node_attribute_at_index(node_index, 'output_tensors', - 'output') - - @property - def input(self): - """Retrieves the input tensor(s) of a layer. - - Only applicable if the layer has exactly one input, - i.e. if it is connected to one incoming layer. - - Returns: - Input tensor or list of input tensors. - - Raises: - AttributeError: if the layer is connected to - more than one incoming layers. - - Raises: - RuntimeError: If called in Eager mode. - AttributeError: If no inbound nodes are found. - """ - if not self._inbound_nodes: - raise AttributeError('Layer ' + self.name + - ' is not connected, no input to return.') - return self._get_node_attribute_at_index(0, 'input_tensors', 'input') - - @property - def output(self): - """Retrieves the output tensor(s) of a layer. - - Only applicable if the layer has exactly one output, - i.e. if it is connected to one incoming layer. - - Returns: - Output tensor or list of output tensors. - - Raises: - AttributeError: if the layer is connected to more than one incoming - layers. - RuntimeError: if called in Eager mode. - """ - if not self._inbound_nodes: - raise AttributeError('Layer ' + self.name + ' has no inbound nodes.') - return self._get_node_attribute_at_index(0, 'output_tensors', 'output') - - @property - def input_shape(self): - """Retrieves the input shape(s) of a layer. - - Only applicable if the layer has exactly one input, - i.e. if it is connected to one incoming layer, or if all inputs - have the same shape. - - Returns: - Input shape, as an integer shape tuple - (or list of shape tuples, one tuple per input tensor). - - Raises: - AttributeError: if the layer has no defined input_shape. - RuntimeError: if called in Eager mode. - """ - if not self._inbound_nodes: - raise AttributeError('The layer has never been called ' - 'and thus has no defined input shape.') - all_input_shapes = set( - [str(node.input_shapes) for node in self._inbound_nodes]) - if len(all_input_shapes) == 1: - input_shapes = self._inbound_nodes[0].input_shapes - if len(input_shapes) == 1: - return tuple(tensor_shape.TensorShape(input_shapes[0]).as_list()) - else: - return [ - tuple(tensor_shape.TensorShape(shape).as_list()) - for shape in input_shapes - ] - else: - raise AttributeError('The layer "' + str(self.name) + - ' has multiple inbound nodes, ' - 'with different input shapes. Hence ' - 'the notion of "input shape" is ' - 'ill-defined for the layer. ' - 'Use `get_input_shape_at(node_index)` ' - 'instead.') - - def count_params(self): - """Count the total number of scalars composing the weights. - - Returns: - An integer count. - - Raises: - ValueError: if the layer isn't yet built - (in which case its weights aren't yet defined). - """ - if not self.built: - if self.__class__.__name__ == 'Sequential': - self.build() # pylint: disable=no-value-for-parameter - else: - raise ValueError('You tried to call `count_params` on ' + self.name + - ', but the layer isn\'t built. ' - 'You can build it manually via: `' + self.name + - '.build(batch_input_shape)`.') - weight_shapes = [w.get_shape().as_list() for w in self.weights] - return int(sum([np.prod(w) for w in weight_shapes])) - - @property - def output_shape(self): - """Retrieves the output shape(s) of a layer. - - Only applicable if the layer has one output, - or if all outputs have the same shape. - - Returns: - Output shape, as an integer shape tuple - (or list of shape tuples, one tuple per output tensor). - - Raises: - AttributeError: if the layer has no defined output shape. - RuntimeError: if called in Eager mode. - """ - if not self._inbound_nodes: - raise AttributeError('The layer has never been called ' - 'and thus has no defined output shape.') - all_output_shapes = set( - [str(node.output_shapes) for node in self._inbound_nodes]) - if len(all_output_shapes) == 1: - output_shapes = self._inbound_nodes[0].output_shapes - if len(output_shapes) == 1: - return tuple(tensor_shape.TensorShape(output_shapes[0]).as_list()) - else: - return [ - tuple(tensor_shape.TensorShape(shape).as_list()) - for shape in output_shapes - ] - else: - raise AttributeError('The layer "%s"' - ' has multiple inbound nodes, ' - 'with different output shapes. Hence ' - 'the notion of "output shape" is ' - 'ill-defined for the layer. ' - 'Use `get_output_shape_at(node_index)` ' - 'instead.' % self.name) - - @property - def inbound_nodes(self): - """Deprecated, do NOT use! Only for compatibility with external Keras.""" - return self._inbound_nodes - - @property - def outbound_nodes(self): - """Deprecated, do NOT use! Only for compatibility with external Keras.""" - return self._outbound_nodes - - def _assert_input_compatibility(self, inputs): - """Checks compatibility between the layer and provided inputs. - - This checks that the tensor(s) `inputs` verify the input assumptions - of the layer (if any). If not, a clear and actional exception gets raised. - - Arguments: - inputs: input tensor or list of input tensors. - - Raises: - ValueError: in case of mismatch between - the provided inputs and the expectations of the layer. - """ - if not self.input_spec: - return - if not isinstance(self.input_spec, (list, tuple)): - input_spec = nest.flatten(self.input_spec) - else: - input_spec = self.input_spec - inputs = nest.flatten(inputs) - if len(inputs) != len(input_spec): - raise ValueError('Layer ' + self.name + ' expects ' + - str(len(input_spec)) + ' inputs, ' - 'but it received ' + str(len(inputs)) + - ' input tensors. Inputs received: ' + str(inputs)) - for input_index, (x, spec) in enumerate(zip(inputs, input_spec)): - if spec is None: - continue - - if (spec.ndim is not None or - spec.min_ndim is not None or - spec.max_ndim is not None): - if x.get_shape().ndims is None: - raise ValueError('Input ' + str(input_index) + ' of layer ' + - self.name + ' is incompatible with the layer: ' - 'its rank is undefined, but the layer requires a ' - 'defined rank.') - - # Check ndim. - if spec.ndim is not None: - ndim = x.get_shape().ndims - if ndim != spec.ndim: - raise ValueError('Input ' + str(input_index) + ' of layer ' + - self.name + ' is incompatible with the layer: ' - 'expected ndim=' + str(spec.ndim) + ', found ndim=' + - str(ndim) + '. Full shape received: ' + - str(x.get_shape().as_list())) - if spec.max_ndim is not None: - ndim = x.get_shape().ndims - if ndim is not None and ndim > spec.max_ndim: - raise ValueError('Input ' + str(input_index) + ' of layer ' + - self.name + ' is incompatible with the layer: ' - 'expected max_ndim=' + str(spec.max_ndim) + - ', found ndim=' + str(ndim)) - if spec.min_ndim is not None: - ndim = x.get_shape().ndims - if ndim is not None and ndim < spec.min_ndim: - raise ValueError('Input ' + str(input_index) + ' of layer ' + - self.name + ' is incompatible with the layer: ' - ': expected min_ndim=' + str(spec.min_ndim) + - ', found ndim=' + str(ndim) + - '. Full shape received: ' + - str(x.get_shape().as_list())) - # Check dtype. - if spec.dtype is not None: - if x.dtype != spec.dtype: - raise ValueError('Input ' + str(input_index) + ' of layer ' + - self.name + ' is incompatible with the layer: ' - 'expected dtype=' + str(spec.dtype) + - ', found dtype=' + str(x.dtype)) - # Check specific shape axes. - if spec.axes: - shape = x.get_shape().as_list() - if shape is not None: - for axis, value in spec.axes.items(): - if hasattr(value, 'value'): - value = value.value - if value is not None and shape[int(axis)] not in {value, None}: - raise ValueError( - 'Input ' + str(input_index) + ' of layer ' + self.name + ' is' - ' incompatible with the layer: expected axis ' + str(axis) + - ' of input shape to have value ' + str(value) + - ' but received input with shape ' + str(shape)) - # Check shape. - if spec.shape is not None: - shape = x.get_shape().as_list() - if shape is not None: - for spec_dim, dim in zip(spec.shape, shape): - if spec_dim is not None and dim is not None: - if spec_dim != dim: - raise ValueError('Input ' + str(input_index) + - ' is incompatible with layer ' + self.name + - ': expected shape=' + str(spec.shape) + - ', found shape=' + str(shape)) - - -@tf_export('keras.layers.InputSpec', 'layers.InputSpec') -class InputSpec(object): - """Specifies the ndim, dtype and shape of every input to a layer. - - Every layer should expose (if appropriate) an `input_spec` attribute: - a list of instances of InputSpec (one per input tensor). - - A None entry in a shape is compatible with any dimension, - a None shape is compatible with any shape. - - Arguments: - dtype: Expected DataType of the input. - shape: Shape tuple, expected shape of the input - (may include None for unchecked axes). - ndim: Integer, expected rank of the input. - max_ndim: Integer, maximum rank of the input. - min_ndim: Integer, minimum rank of the input. - axes: Dictionary mapping integer axes to - a specific dimension value. - """ - - def __init__(self, - dtype=None, - shape=None, - ndim=None, - max_ndim=None, - min_ndim=None, - axes=None): - self.dtype = dtype - self.shape = shape - if shape is not None: - self.ndim = len(shape) - else: - self.ndim = ndim - self.max_ndim = max_ndim - self.min_ndim = min_ndim - self.axes = axes or {} - - def __repr__(self): - spec = [('dtype=' + str(self.dtype)) if self.dtype else '', - ('shape=' + str(self.shape)) if self.shape else '', - ('ndim=' + str(self.ndim)) if self.ndim else '', - ('max_ndim=' + str(self.max_ndim)) if self.max_ndim else '', - ('min_ndim=' + str(self.min_ndim)) if self.min_ndim else '', - ('axes=' + str(self.axes)) if self.axes else ''] - return 'InputSpec(%s)' % ', '.join(x for x in spec if x) - - -class Node(object): - """A `Node` describes the connectivity between two layers. - - Each time a layer is connected to some new input, - a node is added to `layer._inbound_nodes`. - Each time the output of a layer is used by another layer, - a node is added to `layer._outbound_nodes`. - - Arguments: - outbound_layer: the layer that takes - `input_tensors` and turns them into `output_tensors` - (the node gets created when the `call` - method of the layer was called). - inbound_layers: a list of layers, the same length as `input_tensors`, - the layers from where `input_tensors` originate. - node_indices: a list of integers, the same length as `inbound_layers`. - `node_indices[i]` is the origin node of `input_tensors[i]` - (necessary since each inbound layer might have several nodes, - e.g. if the layer is being shared with a different data stream). - tensor_indices: a list of integers, - the same length as `inbound_layers`. - `tensor_indices[i]` is the index of `input_tensors[i]` within the - output of the inbound layer - (necessary since each inbound layer might - have multiple tensor outputs, with each one being - independently manipulable). - input_tensors: list of input tensors. - output_tensors: list of output tensors. - arguments: dictionary of keyword arguments that were passed to the - `call` method of the layer at the call that created the node. - - `node_indices` and `tensor_indices` are basically fine-grained coordinates - describing the origin of the `input_tensors`. - - A node from layer A to layer B is added to: - - A._outbound_nodes - - B._inbound_nodes - """ - - def __init__(self, - outbound_layer, - inbound_layers, - node_indices, - tensor_indices, - input_tensors, - output_tensors, - arguments=None): - # Layer instance (NOT a list). - if isinstance(outbound_layer, list): - raise ValueError( - '`outbound_layer` should be a layer instance, not a list.') - # this is the layer that takes a list of input tensors - # and turns them into a list of output tensors. - # the current node will be added to - # the inbound_nodes of outbound_layer. - self.outbound_layer = outbound_layer - - # The following 3 properties describe where - # the input tensors come from: which layers, - # and for each layer, which node and which - # tensor output of each node. - - # List of layer instances. - self.inbound_layers = inbound_layers - # List of integers, 1:1 mapping with inbound_layers. - self.node_indices = node_indices - # List of integers, 1:1 mapping with inbound_layers. - self.tensor_indices = tensor_indices - - # Following 2 properties: - # tensor inputs and outputs of outbound_layer. - - # List of tensors. 1:1 mapping with inbound_layers. - self.input_tensors = input_tensors - # List of tensors, created by outbound_layer.call(). - self.output_tensors = output_tensors - - # Following 2 properties: input and output shapes. - - # List of shape tuples, shapes of input_tensors. - self.input_shapes = [layers_util.static_shape(x) for x in input_tensors] - # List of shape tuples, shapes of output_tensors. - self.output_shapes = [layers_util.static_shape(x) for x in output_tensors] - - # Optional keyword arguments to layer's `call`. - self.arguments = arguments - - # Add nodes to all layers involved. - for layer in inbound_layers: - if layer is not None: - # For compatibility with external Keras, we use the deprecated - # accessor here. - layer.outbound_nodes.append(self) - # For compatibility with external Keras, we use the deprecated - # accessor here. - outbound_layer.inbound_nodes.append(self) - - def get_config(self): - inbound_names = [] - for layer in self.inbound_layers: - if layer: - inbound_names.append(layer.name) - else: - inbound_names.append(None) - return { - 'outbound_layer': self.outbound_layer.name, - 'inbound_layers': inbound_names, - 'node_indices': self.node_indices, - 'tensor_indices': self.tensor_indices - } - - -class _DeferredTensor(object): - """Tensor-like object used to build graphs of layers in Eager mode. - - When calling a layer on a DeferredTensor, the layer will not perform any - computation and will simply perfom shape inference to return new - DeferredTensors with appropriate shape information. Thus DeferredTensor - behaves like a graph-mode Tensor when manipulated by layers. - """ - - def __init__(self, shape, dtype, name=None): - self.shape = tensor_shape.TensorShape(shape) - if dtype is None: - self.dtype = dtypes.as_dtype(np.float32) - else: - self.dtype = dtypes.as_dtype(dtype) - self.name = name - - def get_shape(self): - return self.shape - - def __str__(self): - return "DeferredTensor('%s', shape=%s, dtype=%s)" % (self.name, - self.get_shape(), - self.dtype.name) - - def __repr__(self): - return "<_DeferredTensor '%s' shape=%s dtype=%s>" % (self.name, - self.get_shape(), - self.dtype.name) - - -def _is_tensor_or_tensor_list(v): - v = nest.flatten(v) - if v and isinstance(v[0], ops.Tensor): - return True - else: - return False - - -def _to_snake_case(name): - intermediate = re.sub('(.)([A-Z][a-z0-9]+)', r'\1_\2', name) - insecure = re.sub('([a-z])([A-Z])', r'\1_\2', intermediate).lower() - # If the class is private the name starts with "_" which is not secure - # for creating scopes. We prefix the name with "private" in this case. - if insecure[0] != '_': - return insecure - return 'private' + insecure - - -def _to_list(x): - """This normalizes a list/tuple or single element into a list. - - If a single element is passed, we return - a list of size 1 containing the element. - - Arguments: - x: list or tuple or single element. - - Returns: - A list. - """ - if isinstance(x, (list, tuple)): - return list(x) - return [x] - def _add_elements_to_collection(elements, collection_list): if context.executing_eagerly(): @@ -1473,105 +354,3 @@ def _add_elements_to_collection(elements, collection_list): if element not in collection_set: collection.append(element) - -def _is_all_none(iterable_or_element): - if not isinstance(iterable_or_element, (list, tuple)): - iterable = [iterable_or_element] - else: - iterable = iterable_or_element - # We cannot use Python's `any` because the iterable may return Tensors. - for element in iterable: - if element is not None: - return False - return True - - -def _have_all_keras_metadata(iterable_or_element): - if not isinstance(iterable_or_element, (list, tuple)): - iterable = [iterable_or_element] - else: - iterable = iterable_or_element - return all([hasattr(x, '_keras_history') for x in iterable]) - - -def _collect_previous_mask(input_tensors): - """Retrieves the output mask(s) of the previous node. - - Arguments: - input_tensors: A tensor or list of tensors. - - Returns: - A mask tensor or list of mask tensors. - """ - input_tensors = nest.flatten(input_tensors) - masks = [] - for x in input_tensors: - if hasattr(x, '_keras_mask'): - mask = x._keras_mask # pylint: disable=protected-access - masks.append(mask) - else: - masks.append(None) - if len(masks) == 1: - return masks[0] - return masks - - -# A global dictionary mapping graph objects to an index of counters used -# for various layer names in each graph. -# Allows to give unique autogenerated names to layers, in a graph-specific way. -PER_GRAPH_LAYER_NAME_UIDS = weakref.WeakKeyDictionary() - - -def _get_default_graph_uid_map(): - graph = ops.get_default_graph() - name_uid_map = PER_GRAPH_LAYER_NAME_UIDS.get(graph, None) - if name_uid_map is None: - name_uid_map = collections.defaultdict(int) - PER_GRAPH_LAYER_NAME_UIDS[graph] = name_uid_map - return name_uid_map - - -def _unique_layer_name(name, name_uid_map=None, avoid_names=None, namespace='', - zero_based=False): - """Makes a layer name (or arbitrary string) unique within a TensorFlow graph. - - Arguments: - name: String name to make unique. - name_uid_map: An optional defaultdict(int) to use when creating unique - names. If None (default), uses a per-Graph dictionary. - avoid_names: An optional set or dict with names which should not be used. If - None (default) does not avoid any names. - namespace: Gets a name which is unique within the (graph, namespace). Layers - which are not Networks use a blank namespace and so get graph-global - names. - zero_based: If True, name sequences start with no suffix (e.g. "dense", - "dense_1"). If False, naming is one-based ("dense_1", "dense_2"). - - Returns: - Unique string name. - - Example: - - ```python - _unique_layer_name('dense') # dense_1 - _unique_layer_name('dense') # dense_2 - ``` - """ - if name_uid_map is None: - name_uid_map = _get_default_graph_uid_map() - if avoid_names is None: - avoid_names = set() - proposed_name = None - while proposed_name is None or proposed_name in avoid_names: - name_key = (namespace, name) - if zero_based: - number = name_uid_map[name_key] - if number: - proposed_name = name + '_' + str(number) - else: - proposed_name = name - name_uid_map[name_key] += 1 - else: - name_uid_map[name_key] += 1 - proposed_name = name + '_' + str(name_uid_map[name_key]) - return proposed_name diff --git a/tensorflow/python/layers/base_test.py b/tensorflow/python/layers/base_test.py index 9ed4afeaba..c05c675263 100644 --- a/tensorflow/python/layers/base_test.py +++ b/tensorflow/python/layers/base_test.py @@ -94,61 +94,6 @@ class BaseLayerTest(test.TestCase): with self.assertRaisesRegexp(ValueError, 'activity_regularizer'): core_layers.Dense(1, activity_regularizer=lambda *args, **kwargs: 0.) - def testGetVariable(self): - with self.test_session(): - - class MyLayer(base_layers.Layer): - - def build(self, input_shape): - self.my_var = self.add_variable( - 'my_var', [2, 2], initializer=init_ops.zeros_initializer()) - - def call(self, inputs): - return inputs * 2 - - layer = MyLayer(name='my_layer') - inputs = random_ops.random_uniform((5,), seed=1) - layer.apply(inputs) - layer.apply(inputs) - self.assertEqual([v.name for v in layer.variables], - ['my_layer/my_var:0']) - - # Creating a layer with no scope leads to lazy construction of - # the scope at apply() time. It uses scope "/base_name" - lazy_layer = MyLayer(_reuse=True) - with variable_scope.variable_scope('new_scope'): - with variable_scope.variable_scope('my_layer'): - variable_scope.get_variable('my_var', [2, 2]) - - # Smoke test: it runs. - lazy_layer.apply(inputs) - # The variables were created outside of the Layer, and - # reuse=True, so the Layer does not own them and they are not - # stored in its collection. - self.assertEqual(lazy_layer.variables, []) - self.assertEqual(lazy_layer._scope.name, 'new_scope/my_layer') - - # Creating a layer with no scope leads to lazy construction of - # the scope at apply() time. If 'scope' argument is passed to - # apply(), it uses that scope when accessing variables. - lazy_layer = MyLayer(_reuse=True) - with variable_scope.variable_scope('new_scope') as new_scope: - variable_scope.get_variable('my_var', [2, 2]) - - # Smoke test: it runs. - lazy_layer.apply(inputs, scope=new_scope) - # The variables were created outside of the Layer, and - # reuse=True, so the Layer does not own them and they are not - # stored in its collection. - self.assertEqual(lazy_layer.variables, []) - self.assertEqual(lazy_layer._scope.name, 'new_scope') - - # Checking for graph equality is only done in GRAPH mode. - with ops.Graph().as_default(): - inputs_ng = random_ops.random_uniform((5,), seed=1) - with self.assertRaisesRegexp(ValueError, r'graph are not the same'): - layer.apply(inputs_ng) - @test_util.run_in_graph_and_eager_modes() def testCall(self): @@ -165,38 +110,6 @@ class BaseLayerTest(test.TestCase): # op is only supported in GRAPH mode self.assertEqual(outputs.op.name, 'my_layer/Square') - def testFirstCallCanCreateVariablesButSecondCanNotWhenBuildEmpty(self): - # Note that this test is only run in Graph mode since with EAGER mode we can - # still create a new variable on second call. - - class MyLayer(base_layers.Layer): - - def build(self, _): - # Do not mark the layer as built. - pass - - def call(self, inputs): - self.my_var = self.add_variable('my_var', [2, 2]) - if self.built: - # Skip creating on the first call; try to create after it's - # built. This is expected to fail. - self.add_variable('this_will_break_on_second_call', [2, 2]) - return inputs + math_ops.square(self.my_var) - - layer = MyLayer(name='my_layer') - inputs = random_ops.random_uniform((2,), seed=1) - outputs = layer.apply(inputs) - self.assertEqual(layer.built, True) - self.assertEqual(outputs.op.name, 'my_layer/add') - self.assertEqual([v.name - for v in layer.variables], ['my_layer/my_var:0']) - with self.assertRaisesRegexp(ValueError, - 'my_layer/this_will_break_on_second_call'): - layer.apply(inputs) - # The list of variables hasn't changed. - self.assertEqual([v.name - for v in layer.variables], ['my_layer/my_var:0']) - @test_util.run_in_graph_and_eager_modes() def testDeepCopy(self): @@ -645,13 +558,14 @@ class BaseLayerTest(test.TestCase): def testLayerGraphSetInFirstApply(self): with ops.Graph().as_default(): - layer = core_layers.Dense(1) # Graph at construction time is ignored + # Graph at construction time is ignored + layer = core_layers.Dense(1) with ops.Graph().as_default(): - layer.apply(constant_op.constant([[1]])) + layer.apply(constant_op.constant([[1.]])) # layer is now bound to second Graph with ops.Graph().as_default(), self.assertRaisesRegexp( ValueError, 'Input graph and Layer graph are not the same'): - layer.apply(constant_op.constant([[1]])) + layer.apply(constant_op.constant([[1.]])) if __name__ == '__main__': diff --git a/tensorflow/python/layers/convolutional.py b/tensorflow/python/layers/convolutional.py index 2d99b1688f..34a1487e74 100644 --- a/tensorflow/python/layers/convolutional.py +++ b/tensorflow/python/layers/convolutional.py @@ -23,6 +23,7 @@ from __future__ import print_function from tensorflow.python.eager import context from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.keras._impl.keras import layers as keras_layers from tensorflow.python.layers import base from tensorflow.python.layers import utils from tensorflow.python.ops import array_ops @@ -32,201 +33,8 @@ from tensorflow.python.ops import nn_ops from tensorflow.python.util.tf_export import tf_export -class _Conv(base.Layer): - """Abstract nD convolution layer (private, used as implementation base). - - This layer creates a convolution kernel that is convolved - (actually cross-correlated) with the layer input to produce a tensor of - outputs. If `use_bias` is True (and a `bias_initializer` is provided), - a bias vector is created and added to the outputs. Finally, if - `activation` is not `None`, it is applied to the outputs as well. - - Arguments: - rank: An integer, the rank of the convolution, e.g. "2" for 2D convolution. - filters: Integer, the dimensionality of the output space (i.e. the number - of filters in the convolution). - kernel_size: An integer or tuple/list of n integers, specifying the - length of the convolution window. - strides: An integer or tuple/list of n integers, - specifying the stride length of the convolution. - Specifying any stride value != 1 is incompatible with specifying - any `dilation_rate` value != 1. - padding: One of `"valid"` or `"same"` (case-insensitive). - data_format: A string, one of `channels_last` (default) or `channels_first`. - The ordering of the dimensions in the inputs. - `channels_last` corresponds to inputs with shape - `(batch, ..., channels)` while `channels_first` corresponds to - inputs with shape `(batch, channels, ...)`. - dilation_rate: An integer or tuple/list of n integers, specifying - the dilation rate to use for dilated convolution. - Currently, specifying any `dilation_rate` value != 1 is - incompatible with specifying any `strides` value != 1. - activation: Activation function. Set it to None to maintain a - linear activation. - use_bias: Boolean, whether the layer uses a bias. - kernel_initializer: An initializer for the convolution kernel. - bias_initializer: An initializer for the bias vector. If None, the default - initializer will be used. - kernel_regularizer: Optional regularizer for the convolution kernel. - bias_regularizer: Optional regularizer for the bias vector. - activity_regularizer: Optional regularizer function for the output. - kernel_constraint: Optional projection function to be applied to the - kernel after being updated by an `Optimizer` (e.g. used to implement - norm constraints or value constraints for layer weights). The function - must take as input the unprojected variable and must return the - projected variable (which must have the same shape). Constraints are - not safe to use when doing asynchronous distributed training. - bias_constraint: Optional projection function to be applied to the - bias after being updated by an `Optimizer`. - trainable: Boolean, if `True` also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`). - name: A string, the name of the layer. - """ - - def __init__(self, rank, - filters, - kernel_size, - strides=1, - padding='valid', - data_format='channels_last', - dilation_rate=1, - activation=None, - use_bias=True, - kernel_initializer=None, - bias_initializer=init_ops.zeros_initializer(), - kernel_regularizer=None, - bias_regularizer=None, - activity_regularizer=None, - kernel_constraint=None, - bias_constraint=None, - trainable=True, - name=None, - **kwargs): - super(_Conv, self).__init__(trainable=trainable, name=name, - activity_regularizer=activity_regularizer, - **kwargs) - self.rank = rank - self.filters = filters - self.kernel_size = utils.normalize_tuple(kernel_size, rank, 'kernel_size') - self.strides = utils.normalize_tuple(strides, rank, 'strides') - self.padding = utils.normalize_padding(padding) - self.data_format = utils.normalize_data_format(data_format) - self.dilation_rate = utils.normalize_tuple( - dilation_rate, rank, 'dilation_rate') - self.activation = activation - self.use_bias = use_bias - self.kernel_initializer = kernel_initializer - self.bias_initializer = bias_initializer - self.kernel_regularizer = kernel_regularizer - self.bias_regularizer = bias_regularizer - self.kernel_constraint = kernel_constraint - self.bias_constraint = bias_constraint - self.input_spec = base.InputSpec(ndim=self.rank + 2) - - def build(self, input_shape): - input_shape = tensor_shape.TensorShape(input_shape) - if self.data_format == 'channels_first': - channel_axis = 1 - else: - channel_axis = -1 - if input_shape[channel_axis].value is None: - raise ValueError('The channel dimension of the inputs ' - 'should be defined. Found `None`.') - input_dim = input_shape[channel_axis].value - kernel_shape = self.kernel_size + (input_dim, self.filters) - - self.kernel = self.add_variable(name='kernel', - shape=kernel_shape, - initializer=self.kernel_initializer, - regularizer=self.kernel_regularizer, - constraint=self.kernel_constraint, - trainable=True, - dtype=self.dtype) - if self.use_bias: - self.bias = self.add_variable(name='bias', - shape=(self.filters,), - initializer=self.bias_initializer, - regularizer=self.bias_regularizer, - constraint=self.bias_constraint, - trainable=True, - dtype=self.dtype) - else: - self.bias = None - self.input_spec = base.InputSpec(ndim=self.rank + 2, - axes={channel_axis: input_dim}) - self._convolution_op = nn_ops.Convolution( - input_shape, - filter_shape=self.kernel.get_shape(), - dilation_rate=self.dilation_rate, - strides=self.strides, - padding=self.padding.upper(), - data_format=utils.convert_data_format(self.data_format, - self.rank + 2)) - self.built = True - - def call(self, inputs): - outputs = self._convolution_op(inputs, self.kernel) - - if self.use_bias: - if self.data_format == 'channels_first': - if self.rank == 1: - # nn.bias_add does not accept a 1D input tensor. - bias = array_ops.reshape(self.bias, (1, self.filters, 1)) - outputs += bias - if self.rank == 2: - outputs = nn.bias_add(outputs, self.bias, data_format='NCHW') - if self.rank == 3: - # As of Mar 2017, direct addition is significantly slower than - # bias_add when computing gradients. To use bias_add, we collapse Z - # and Y into a single dimension to obtain a 4D input tensor. - outputs_shape = outputs.shape.as_list() - if outputs_shape[0] is None: - outputs_shape[0] = -1 - outputs_4d = array_ops.reshape(outputs, - [outputs_shape[0], outputs_shape[1], - outputs_shape[2] * outputs_shape[3], - outputs_shape[4]]) - outputs_4d = nn.bias_add(outputs_4d, self.bias, data_format='NCHW') - outputs = array_ops.reshape(outputs_4d, outputs_shape) - else: - outputs = nn.bias_add(outputs, self.bias, data_format='NHWC') - - if self.activation is not None: - return self.activation(outputs) - return outputs - - def compute_output_shape(self, input_shape): - input_shape = tensor_shape.TensorShape(input_shape).as_list() - if self.data_format == 'channels_last': - space = input_shape[1:-1] - new_space = [] - for i in range(len(space)): - new_dim = utils.conv_output_length( - space[i], - self.kernel_size[i], - padding=self.padding, - stride=self.strides[i], - dilation=self.dilation_rate[i]) - new_space.append(new_dim) - return tensor_shape.TensorShape([input_shape[0]] + new_space + - [self.filters]) - else: - space = input_shape[2:] - new_space = [] - for i in range(len(space)): - new_dim = utils.conv_output_length( - space[i], - self.kernel_size[i], - padding=self.padding, - stride=self.strides[i], - dilation=self.dilation_rate[i]) - new_space.append(new_dim) - return tensor_shape.TensorShape([input_shape[0], self.filters] + - new_space) - - @tf_export('layers.Conv1D') -class Conv1D(_Conv): +class Conv1D(keras_layers.Conv1D, base.Layer): """1D convolution layer (e.g. temporal convolution). This layer creates a convolution kernel that is convolved @@ -294,8 +102,7 @@ class Conv1D(_Conv): trainable=True, name=None, **kwargs): - super(Convolution1D, self).__init__( - rank=1, + super(Conv1D, self).__init__( filters=filters, kernel_size=kernel_size, strides=strides, @@ -417,7 +224,7 @@ def conv1d(inputs, @tf_export('layers.Conv2D') -class Conv2D(_Conv): +class Conv2D(keras_layers.Conv2D, base.Layer): """2D convolution layer (e.g. spatial convolution over images). This layer creates a convolution kernel that is convolved @@ -493,7 +300,6 @@ class Conv2D(_Conv): name=None, **kwargs): super(Conv2D, self).__init__( - rank=2, filters=filters, kernel_size=kernel_size, strides=strides, @@ -622,7 +428,7 @@ def conv2d(inputs, @tf_export('layers.Conv3D') -class Conv3D(_Conv): +class Conv3D(keras_layers.Conv3D, base.Layer): """3D convolution layer (e.g. spatial convolution over volumes). This layer creates a convolution kernel that is convolved @@ -699,7 +505,6 @@ class Conv3D(_Conv): name=None, **kwargs): super(Conv3D, self).__init__( - rank=3, filters=filters, kernel_size=kernel_size, strides=strides, @@ -828,169 +633,8 @@ def conv3d(inputs, return layer.apply(inputs) -class _SeparableConv(_Conv): - """Abstract base layer for separable nD convolution. - - This layer performs a depthwise convolution that acts separately on - channels, followed by a pointwise convolution that mixes channels. - If `use_bias` is True and a bias initializer is provided, - it adds a bias vector to the output. - It then optionally applies an activation function to produce the final output. - - Arguments: - rank: An integer, the rank of the convolution, e.g. "2" for 2D convolution. - filters: Integer, the dimensionality of the output space (i.e. the number - of filters in the convolution). - kernel_size: A tuple or list of integers specifying the spatial - dimensions of the filters. Can be a single integer to specify the same - value for all spatial dimensions. - strides: A tuple or list of integers specifying the strides - of the convolution. Can be a single integer to specify the same value for - all spatial dimensions. - Specifying any `stride` value != 1 is incompatible with specifying - any `dilation_rate` value != 1. - padding: One of `"valid"` or `"same"` (case-insensitive). - data_format: A string, one of `channels_last` (default) or `channels_first`. - The ordering of the dimensions in the inputs. - `channels_last` corresponds to inputs with shape - `(batch, ..., channels)` while `channels_first` corresponds to - inputs with shape `(batch, channels, ...)`. - dilation_rate: An integer or tuple/list of 2 integers, specifying - the dilation rate to use for dilated convolution. - Can be a single integer to specify the same value for - all spatial dimensions. - Currently, specifying any `dilation_rate` value != 1 is - incompatible with specifying any stride value != 1. - depth_multiplier: The number of depthwise convolution output channels for - each input channel. The total number of depthwise convolution output - channels will be equal to `num_filters_in * depth_multiplier`. - activation: Activation function. Set it to None to maintain a - linear activation. - use_bias: Boolean, whether the layer uses a bias. - depthwise_initializer: An initializer for the depthwise convolution kernel. - pointwise_initializer: An initializer for the pointwise convolution kernel. - bias_initializer: An initializer for the bias vector. If None, the default - initializer will be used. - depthwise_regularizer: Optional regularizer for the depthwise - convolution kernel. - pointwise_regularizer: Optional regularizer for the pointwise - convolution kernel. - bias_regularizer: Optional regularizer for the bias vector. - activity_regularizer: Optional regularizer function for the output. - depthwise_constraint: Optional projection function to be applied to the - depthwise kernel after being updated by an `Optimizer` (e.g. used for - norm constraints or value constraints for layer weights). The function - must take as input the unprojected variable and must return the - projected variable (which must have the same shape). Constraints are - not safe to use when doing asynchronous distributed training. - pointwise_constraint: Optional projection function to be applied to the - pointwise kernel after being updated by an `Optimizer`. - bias_constraint: Optional projection function to be applied to the - bias after being updated by an `Optimizer`. - trainable: Boolean, if `True` also add variables to the graph collection - `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`). - name: A string, the name of the layer. - """ - - def __init__(self, - rank, - filters, - kernel_size, - strides=1, - padding='valid', - data_format='channels_last', - dilation_rate=1, - depth_multiplier=1, - activation=None, - use_bias=True, - depthwise_initializer=None, - pointwise_initializer=None, - bias_initializer=init_ops.zeros_initializer(), - depthwise_regularizer=None, - pointwise_regularizer=None, - bias_regularizer=None, - activity_regularizer=None, - depthwise_constraint=None, - pointwise_constraint=None, - bias_constraint=None, - trainable=True, - name=None, - **kwargs): - super(_SeparableConv, self).__init__( - rank=rank, - filters=filters, - kernel_size=kernel_size, - strides=strides, - padding=padding, - data_format=data_format, - dilation_rate=dilation_rate, - activation=activation, - use_bias=use_bias, - bias_regularizer=bias_regularizer, - activity_regularizer=activity_regularizer, - bias_constraint=bias_constraint, - trainable=trainable, - name=name, - **kwargs) - self.depth_multiplier = depth_multiplier - self.depthwise_initializer = depthwise_initializer - self.pointwise_initializer = pointwise_initializer - self.depthwise_regularizer = depthwise_regularizer - self.pointwise_regularizer = pointwise_regularizer - self.depthwise_constraint = depthwise_constraint - self.pointwise_constraint = pointwise_constraint - - def build(self, input_shape): - input_shape = tensor_shape.TensorShape(input_shape) - if self.data_format == 'channels_first': - channel_axis = 1 - else: - channel_axis = -1 - if input_shape[channel_axis].value is None: - raise ValueError('The channel dimension of the inputs ' - 'should be defined. Found `None`.') - input_dim = input_shape[channel_axis].value - self.input_spec = base.InputSpec(ndim=self.rank + 2, - axes={channel_axis: input_dim}) - depthwise_kernel_shape = self.kernel_size + (input_dim, - self.depth_multiplier) - pointwise_kernel_shape = ( - 1,) * self.rank + (self.depth_multiplier * input_dim, self.filters) - - self.depthwise_kernel = self.add_variable( - name='depthwise_kernel', - shape=depthwise_kernel_shape, - initializer=self.depthwise_initializer, - regularizer=self.depthwise_regularizer, - constraint=self.depthwise_constraint, - trainable=True, - dtype=self.dtype) - self.pointwise_kernel = self.add_variable( - name='pointwise_kernel', - shape=pointwise_kernel_shape, - initializer=self.pointwise_initializer, - regularizer=self.pointwise_regularizer, - constraint=self.pointwise_constraint, - trainable=True, - dtype=self.dtype) - if self.use_bias: - self.bias = self.add_variable(name='bias', - shape=(self.filters,), - initializer=self.bias_initializer, - regularizer=self.bias_regularizer, - constraint=self.bias_constraint, - trainable=True, - dtype=self.dtype) - else: - self.bias = None - self.built = True - - def call(self, inputs): - raise NotImplementedError - - @tf_export('layers.SeparableConv1D') -class SeparableConv1D(_SeparableConv): +class SeparableConv1D(keras_layers.SeparableConv1D, base.Layer): """Depthwise separable 1D convolution. This layer performs a depthwise convolution that acts separately on @@ -1072,7 +716,6 @@ class SeparableConv1D(_SeparableConv): name=None, **kwargs): super(SeparableConv1D, self).__init__( - rank=1, filters=filters, kernel_size=kernel_size, strides=strides, @@ -1096,45 +739,9 @@ class SeparableConv1D(_SeparableConv): name=name, **kwargs) - def call(self, inputs): - if self.data_format == 'channels_last': - strides = (1,) + self.strides * 2 + (1,) - spatial_start_dim = 1 - else: - strides = (1, 1) + self.strides * 2 - spatial_start_dim = 2 - - # Explicitly broadcast inputs and kernels to 4D. - # TODO(fchollet): refactor when a native separable_conv1d op is available. - inputs = array_ops.expand_dims(inputs, spatial_start_dim) - depthwise_kernel = array_ops.expand_dims(self.depthwise_kernel, 0) - pointwise_kernel = array_ops.expand_dims(self.pointwise_kernel, 0) - dilation_rate = (1,) + self.dilation_rate - - outputs = nn.separable_conv2d( - inputs, - depthwise_kernel, - pointwise_kernel, - strides=strides, - padding=self.padding.upper(), - rate=dilation_rate, - data_format=utils.convert_data_format(self.data_format, ndim=4)) - - if self.use_bias: - outputs = nn.bias_add( - outputs, - self.bias, - data_format=utils.convert_data_format(self.data_format, ndim=4)) - - outputs = array_ops.squeeze(outputs, [spatial_start_dim]) - - if self.activation is not None: - return self.activation(outputs) - return outputs - @tf_export('layers.SeparableConv2D') -class SeparableConv2D(_SeparableConv): +class SeparableConv2D(keras_layers.SeparableConv2D, base.Layer): """Depthwise separable 2D convolution. This layer performs a depthwise convolution that acts separately on @@ -1221,7 +828,6 @@ class SeparableConv2D(_SeparableConv): name=None, **kwargs): super(SeparableConv2D, self).__init__( - rank=2, filters=filters, kernel_size=kernel_size, strides=strides, @@ -1245,31 +851,6 @@ class SeparableConv2D(_SeparableConv): name=name, **kwargs) - def call(self, inputs): - # Apply the actual ops. - if self.data_format == 'channels_last': - strides = (1,) + self.strides + (1,) - else: - strides = (1, 1) + self.strides - outputs = nn.separable_conv2d( - inputs, - self.depthwise_kernel, - self.pointwise_kernel, - strides=strides, - padding=self.padding.upper(), - rate=self.dilation_rate, - data_format=utils.convert_data_format(self.data_format, ndim=4)) - - if self.use_bias: - outputs = nn.bias_add( - outputs, - self.bias, - data_format=utils.convert_data_format(self.data_format, ndim=4)) - - if self.activation is not None: - return self.activation(outputs) - return outputs - @tf_export('layers.separable_conv1d') def separable_conv1d(inputs, @@ -1511,7 +1092,7 @@ def separable_conv2d(inputs, @tf_export('layers.Conv2DTranspose') -class Conv2DTranspose(Conv2D): +class Conv2DTranspose(keras_layers.Conv2DTranspose, base.Layer): """Transposed 2D convolution layer (sometimes called 2D Deconvolution). The need for transposed convolutions generally arises @@ -1576,8 +1157,8 @@ class Conv2DTranspose(Conv2D): name=None, **kwargs): super(Conv2DTranspose, self).__init__( - filters, - kernel_size, + filters=filters, + kernel_size=kernel_size, strides=strides, padding=padding, data_format=data_format, @@ -1593,120 +1174,6 @@ class Conv2DTranspose(Conv2D): trainable=trainable, name=name, **kwargs) - self.input_spec = base.InputSpec(ndim=4) - - def build(self, input_shape): - if len(input_shape) != 4: - raise ValueError('Inputs should have rank 4. Received input shape: ' + - str(input_shape)) - if self.data_format == 'channels_first': - channel_axis = 1 - else: - channel_axis = -1 - if input_shape[channel_axis] is None: - raise ValueError('The channel dimension of the inputs ' - 'should be defined. Found `None`.') - input_dim = input_shape[channel_axis] - self.input_spec = base.InputSpec(ndim=4, axes={channel_axis: input_dim}) - kernel_shape = self.kernel_size + (self.filters, input_dim) - - self.kernel = self.add_variable(name='kernel', - shape=kernel_shape, - initializer=self.kernel_initializer, - regularizer=self.kernel_regularizer, - constraint=self.kernel_constraint, - trainable=True, - dtype=self.dtype) - if self.use_bias: - self.bias = self.add_variable(name='bias', - shape=(self.filters,), - initializer=self.bias_initializer, - regularizer=self.bias_regularizer, - constraint=self.bias_constraint, - trainable=True, - dtype=self.dtype) - else: - self.bias = None - self.built = True - - def call(self, inputs): - inputs_shape = array_ops.shape(inputs) - batch_size = inputs_shape[0] - if self.data_format == 'channels_first': - c_axis, h_axis, w_axis = 1, 2, 3 - else: - c_axis, h_axis, w_axis = 3, 1, 2 - - height, width = inputs_shape[h_axis], inputs_shape[w_axis] - kernel_h, kernel_w = self.kernel_size - stride_h, stride_w = self.strides - - # Infer the dynamic output shape: - out_height = utils.deconv_output_length(height, - kernel_h, - self.padding, - stride_h) - out_width = utils.deconv_output_length(width, - kernel_w, - self.padding, - stride_w) - if self.data_format == 'channels_first': - output_shape = (batch_size, self.filters, out_height, out_width) - strides = (1, 1, stride_h, stride_w) - else: - output_shape = (batch_size, out_height, out_width, self.filters) - strides = (1, stride_h, stride_w, 1) - - output_shape_tensor = array_ops.stack(output_shape) - outputs = nn.conv2d_transpose( - inputs, - self.kernel, - output_shape_tensor, - strides, - padding=self.padding.upper(), - data_format=utils.convert_data_format(self.data_format, ndim=4)) - - if not context.executing_eagerly(): - # Infer the static output shape: - out_shape = inputs.get_shape().as_list() - out_shape[c_axis] = self.filters - out_shape[h_axis] = utils.deconv_output_length(out_shape[h_axis], - kernel_h, - self.padding, - stride_h) - out_shape[w_axis] = utils.deconv_output_length(out_shape[w_axis], - kernel_w, - self.padding, - stride_w) - outputs.set_shape(out_shape) - - if self.use_bias: - outputs = nn.bias_add( - outputs, - self.bias, - data_format=utils.convert_data_format(self.data_format, ndim=4)) - - if self.activation is not None: - return self.activation(outputs) - return outputs - - def compute_output_shape(self, input_shape): - input_shape = tensor_shape.TensorShape(input_shape).as_list() - output_shape = list(input_shape) - if self.data_format == 'channels_first': - c_axis, h_axis, w_axis = 1, 2, 3 - else: - c_axis, h_axis, w_axis = 3, 1, 2 - - kernel_h, kernel_w = self.kernel_size - stride_h, stride_w = self.strides - - output_shape[c_axis] = self.filters - output_shape[h_axis] = utils.deconv_output_length( - output_shape[h_axis], kernel_h, self.padding, stride_h) - output_shape[w_axis] = utils.deconv_output_length( - output_shape[w_axis], kernel_w, self.padding, stride_w) - return tensor_shape.TensorShape(output_shape) @tf_export('layers.conv2d_transpose') @@ -1806,7 +1273,7 @@ def conv2d_transpose(inputs, @tf_export('layers.Conv3DTranspose') -class Conv3DTranspose(Conv3D): +class Conv3DTranspose(keras_layers.Conv3DTranspose, base.Layer): """Transposed 3D convolution layer (sometimes called 3D Deconvolution). Arguments: @@ -1885,153 +1352,6 @@ class Conv3DTranspose(Conv3D): trainable=trainable, name=name, **kwargs) - self.input_spec = base.InputSpec(ndim=5) - - def build(self, input_shape): - if len(input_shape) != 5: - raise ValueError('Inputs should have rank 5, received input shape:', - str(input_shape)) - if self.data_format == 'channels_first': - channel_axis = 1 - else: - channel_axis = -1 - if input_shape[channel_axis] is None: - raise ValueError('The channel dimension of the inputs ' - 'should be defined, found None: ' + str(input_shape)) - input_dim = input_shape[channel_axis] - kernel_shape = self.kernel_size + (self.filters, input_dim) - - self.kernel = self.add_variable( - 'kernel', - shape=kernel_shape, - initializer=self.kernel_initializer, - regularizer=self.kernel_regularizer, - constraint=self.kernel_constraint, - trainable=True, - dtype=self.dtype) - if self.use_bias: - self.bias = self.add_variable( - 'bias', - shape=(self.filters,), - initializer=self.bias_initializer, - regularizer=self.bias_regularizer, - constraint=self.bias_constraint, - trainable=True, - dtype=self.dtype) - else: - self.bias = None - self.built = True - - def call(self, inputs): - inputs_shape = array_ops.shape(inputs) - batch_size = inputs_shape[0] - if self.data_format == 'channels_first': - c_axis, d_axis, h_axis, w_axis = 1, 2, 3, 4 - else: - c_axis, d_axis, h_axis, w_axis = 4, 1, 2, 3 - - self.input_spec = base.InputSpec(ndim=5, - axes={c_axis: inputs_shape[c_axis]}) - - depth = inputs_shape[d_axis] - height = inputs_shape[h_axis] - width = inputs_shape[w_axis] - - kernel_d, kernel_h, kernel_w = self.kernel_size - stride_d, stride_h, stride_w = self.strides - - # Infer the dynamic output shape: - out_depth = utils.deconv_output_length(depth, - kernel_d, - self.padding, - stride_d) - out_height = utils.deconv_output_length(height, - kernel_h, - self.padding, - stride_h) - out_width = utils.deconv_output_length(width, - kernel_w, - self.padding, - stride_w) - if self.data_format == 'channels_first': - output_shape = (batch_size, self.filters, out_depth, out_height, - out_width) - strides = (1, 1, stride_d, stride_h, stride_w) - else: - output_shape = (batch_size, out_depth, out_height, out_width, - self.filters) - strides = (1, stride_d, stride_h, stride_w, 1) - - output_shape_tensor = array_ops.stack(output_shape) - outputs = nn.conv3d_transpose( - inputs, - self.kernel, - output_shape_tensor, - strides, - data_format=utils.convert_data_format(self.data_format, ndim=5), - padding=self.padding.upper()) - - if not context.executing_eagerly(): - # Infer the static output shape: - out_shape = inputs.get_shape().as_list() - out_shape[c_axis] = self.filters - out_shape[d_axis] = utils.deconv_output_length(out_shape[d_axis], - kernel_d, - self.padding, - stride_d) - out_shape[h_axis] = utils.deconv_output_length(out_shape[h_axis], - kernel_h, - self.padding, - stride_h) - out_shape[w_axis] = utils.deconv_output_length(out_shape[w_axis], - kernel_w, - self.padding, - stride_w) - outputs.set_shape(out_shape) - - if self.use_bias: - outputs_shape = outputs.shape.as_list() - if outputs_shape[0] is None: - outputs_shape[0] = -1 - if self.data_format == 'channels_first': - outputs_4d = array_ops.reshape(outputs, [ - outputs_shape[0], outputs_shape[1], - outputs_shape[2] * outputs_shape[3], outputs_shape[4] - ]) - else: - outputs_4d = array_ops.reshape(outputs, [ - outputs_shape[0], outputs_shape[1] * outputs_shape[2], - outputs_shape[3], outputs_shape[4] - ]) - outputs_4d = nn.bias_add( - outputs_4d, - self.bias, - data_format=utils.convert_data_format(self.data_format, ndim=4)) - outputs = array_ops.reshape(outputs_4d, outputs_shape) - - if self.activation is not None: - return self.activation(outputs) - return outputs - - def compute_output_shape(self, input_shape): - input_shape = tensor_shape.TensorShape(input_shape).as_list() - output_shape = list(input_shape) - if self.data_format == 'channels_first': - c_axis, d_axis, h_axis, w_axis = 1, 2, 3, 4 - else: - c_axis, d_axis, h_axis, w_axis = 4, 1, 2, 3 - - kernel_d, kernel_h, kernel_w = self.kernel_size - stride_d, stride_h, stride_w = self.strides - - output_shape[c_axis] = self.filters - output_shape[d_axis] = utils.deconv_output_length( - output_shape[d_axis], kernel_d, self.padding, stride_d) - output_shape[h_axis] = utils.deconv_output_length( - output_shape[h_axis], kernel_h, self.padding, stride_h) - output_shape[w_axis] = utils.deconv_output_length( - output_shape[w_axis], kernel_w, self.padding, stride_w) - return tensor_shape.TensorShape(output_shape) @tf_export('layers.conv3d_transpose') diff --git a/tensorflow/python/layers/core.py b/tensorflow/python/layers/core.py index e598d9f83a..6d8e9eac87 100644 --- a/tensorflow/python/layers/core.py +++ b/tensorflow/python/layers/core.py @@ -27,23 +27,14 @@ import six from six.moves import xrange # pylint: disable=redefined-builtin import numpy as np -from tensorflow.python.eager import context -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape +from tensorflow.python.keras._impl.keras import layers as keras_layers from tensorflow.python.layers import base -from tensorflow.python.layers import utils -from tensorflow.python.ops import array_ops from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import standard_ops from tensorflow.python.util.tf_export import tf_export @tf_export('layers.Dense') -class Dense(base.Layer): +class Dense(keras_layers.Dense, base.Layer): """Densely-connected layer class. This layer implements the operation: @@ -108,73 +99,19 @@ class Dense(base.Layer): trainable=True, name=None, **kwargs): - super(Dense, self).__init__(trainable=trainable, name=name, + super(Dense, self).__init__(units=units, + activation=activation, + use_bias=use_bias, + kernel_initializer=kernel_initializer, + bias_initializer=bias_initializer, + kernel_regularizer=kernel_regularizer, + bias_regularizer=bias_regularizer, activity_regularizer=activity_regularizer, + kernel_constraint=kernel_constraint, + bias_constraint=bias_constraint, + trainable=trainable, + name=name, **kwargs) - self.units = units - self.activation = activation - self.use_bias = use_bias - self.kernel_initializer = kernel_initializer - self.bias_initializer = bias_initializer - self.kernel_regularizer = kernel_regularizer - self.bias_regularizer = bias_regularizer - self.kernel_constraint = kernel_constraint - self.bias_constraint = bias_constraint - self.input_spec = base.InputSpec(min_ndim=2) - - def build(self, input_shape): - input_shape = tensor_shape.TensorShape(input_shape) - if input_shape[-1].value is None: - raise ValueError('The last dimension of the inputs to `Dense` ' - 'should be defined. Found `None`.') - self.input_spec = base.InputSpec(min_ndim=2, - axes={-1: input_shape[-1].value}) - self.kernel = self.add_variable('kernel', - shape=[input_shape[-1].value, self.units], - initializer=self.kernel_initializer, - regularizer=self.kernel_regularizer, - constraint=self.kernel_constraint, - dtype=self.dtype, - trainable=True) - if self.use_bias: - self.bias = self.add_variable('bias', - shape=[self.units,], - initializer=self.bias_initializer, - regularizer=self.bias_regularizer, - constraint=self.bias_constraint, - dtype=self.dtype, - trainable=True) - else: - self.bias = None - self.built = True - - def call(self, inputs): - inputs = ops.convert_to_tensor(inputs, dtype=self.dtype) - shape = inputs.get_shape().as_list() - if len(shape) > 2: - # Broadcasting is required for the inputs. - outputs = standard_ops.tensordot(inputs, self.kernel, [[len(shape) - 1], - [0]]) - # Reshape the output back to the original ndim of the input. - if not context.executing_eagerly(): - output_shape = shape[:-1] + [self.units] - outputs.set_shape(output_shape) - else: - outputs = gen_math_ops.mat_mul(inputs, self.kernel) - if self.use_bias: - outputs = nn.bias_add(outputs, self.bias) - if self.activation is not None: - return self.activation(outputs) # pylint: disable=not-callable - return outputs - - def compute_output_shape(self, input_shape): - input_shape = tensor_shape.TensorShape(input_shape) - input_shape = input_shape.with_rank_at_least(2) - if input_shape[-1].value is None: - raise ValueError( - 'The innermost dimension of input_shape must be defined, but saw: %s' - % input_shape) - return input_shape[:-1].concatenate(self.units) @tf_export('layers.dense') @@ -254,7 +191,7 @@ def dense( @tf_export('layers.Dropout') -class Dropout(base.Layer): +class Dropout(keras_layers.Dropout, base.Layer): """Applies Dropout to the input. Dropout consists in randomly setting a fraction `rate` of input units to 0 @@ -282,31 +219,14 @@ class Dropout(base.Layer): seed=None, name=None, **kwargs): - super(Dropout, self).__init__(name=name, **kwargs) - self.rate = rate - self.noise_shape = noise_shape - self.seed = seed - - def _get_noise_shape(self, inputs): - # Subclasses of `Dropout` may implement `_get_noise_shape(self, inputs)`, - # which will override `self.noise_shape`, and allows for custom noise - # shapes with dynamically sized inputs. - if self.noise_shape is None: - return self.noise_shape - return nn_ops._get_noise_shape(inputs, self.noise_shape) + super(Dropout, self).__init__(rate=rate, + noise_shape=noise_shape, + seed=seed, + name=name, + **kwargs) def call(self, inputs, training=False): - - def dropped_inputs(): - return nn.dropout(inputs, 1 - self.rate, - noise_shape=self._get_noise_shape(inputs), - seed=self.seed) - return utils.smart_cond(training, - dropped_inputs, - lambda: array_ops.identity(inputs)) - - def compute_output_shape(self, input_shape): - return input_shape + return super(Dropout, self).call(inputs, training=training) @tf_export('layers.dropout') @@ -352,7 +272,7 @@ def dropout(inputs, @tf_export('layers.Flatten') -class Flatten(base.Layer): +class Flatten(keras_layers.Flatten, base.Layer): """Flattens an input tensor while preserving the batch axis (axis 0). Examples: @@ -367,25 +287,7 @@ class Flatten(base.Layer): # now `y` has shape `(None, None)` ``` """ - - def __init__(self, **kwargs): - super(Flatten, self).__init__(**kwargs) - self.input_spec = base.InputSpec(min_ndim=2) - - def call(self, inputs): - outputs = array_ops.reshape(inputs, (array_ops.shape(inputs)[0], -1)) - if not context.executing_eagerly(): - outputs.set_shape(self.compute_output_shape(inputs.get_shape())) - return outputs - - def compute_output_shape(self, input_shape): - input_shape = tensor_shape.TensorShape(input_shape).as_list() - output_shape = [input_shape[0]] - if all(input_shape[1:]): - output_shape += [np.prod(input_shape[1:])] - else: - output_shape += [None] - return tensor_shape.TensorShape(output_shape) + pass @tf_export('layers.flatten') diff --git a/tensorflow/python/layers/normalization.py b/tensorflow/python/layers/normalization.py index 83b201e642..33284b0d69 100644 --- a/tensorflow/python/layers/normalization.py +++ b/tensorflow/python/layers/normalization.py @@ -24,26 +24,14 @@ import six from six.moves import xrange # pylint: disable=redefined-builtin import numpy as np -from tensorflow.python.eager import context -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape +from tensorflow.python.keras._impl.keras import layers as keras_layers from tensorflow.python.layers import base -from tensorflow.python.layers import utils -from tensorflow.python.ops import array_ops from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.training import distribute as distribute_lib -from tensorflow.python.training import moving_averages from tensorflow.python.util.tf_export import tf_export @tf_export('layers.BatchNormalization') -class BatchNormalization(base.Layer): +class BatchNormalization(keras_layers.BatchNormalization, base.Layer): """Batch Normalization layer from http://arxiv.org/abs/1502.03167. "Batch Normalization: Accelerating Deep Network Training by Reducing @@ -143,485 +131,31 @@ class BatchNormalization(base.Layer): name=None, **kwargs): super(BatchNormalization, self).__init__( - name=name, trainable=trainable, **kwargs) - if isinstance(axis, list): - self.axis = axis[:] - else: - self.axis = axis - self.momentum = momentum - self.epsilon = epsilon - self.center = center - self.scale = scale - self.beta_initializer = beta_initializer - self.gamma_initializer = gamma_initializer - self.moving_mean_initializer = moving_mean_initializer - self.moving_variance_initializer = moving_variance_initializer - self.beta_regularizer = beta_regularizer - self.gamma_regularizer = gamma_regularizer - self.beta_constraint = beta_constraint - self.gamma_constraint = gamma_constraint - self.renorm = renorm - self.virtual_batch_size = virtual_batch_size - self.adjustment = adjustment - if fused is None: - fused = True - - self.fused = fused - self._bessels_correction_test_only = True - - if renorm: - renorm_clipping = renorm_clipping or {} - keys = ['rmax', 'rmin', 'dmax'] - if set(renorm_clipping) - set(keys): - raise ValueError('renorm_clipping %s contains keys not in %s' % - (renorm_clipping, keys)) - self.renorm_clipping = renorm_clipping - self.renorm_momentum = renorm_momentum - - def _add_tower_local_variable(self, *args, **kwargs): - tower_context = distribute_lib.get_tower_context() - with tower_context.tower_local_var_scope('mean'): - return self.add_variable(*args, **kwargs) - - def build(self, input_shape): - input_shape = tensor_shape.TensorShape(input_shape) - if not input_shape.ndims: - raise ValueError('Input has undefined rank:', input_shape) - ndims = len(input_shape) - - # Convert axis to list and resolve negatives - if isinstance(self.axis, int): - self.axis = [self.axis] - - if not isinstance(self.axis, list): - raise TypeError('axis must be int or list, type given: %s' - % type(self.axis)) - - for idx, x in enumerate(self.axis): - if x < 0: - self.axis[idx] = ndims + x - - # Validate axes - for x in self.axis: - if x < 0 or x >= ndims: - raise ValueError('Invalid axis: %d' % x) - if len(self.axis) != len(set(self.axis)): - raise ValueError('Duplicate axis: %s' % self.axis) - - if self.virtual_batch_size is not None: - if self.virtual_batch_size <= 0: - raise ValueError('virtual_batch_size must be a positive integer that ' - 'divides the true batch size of the input Tensor') - # If using virtual batches, the first dimension must be the batch - # dimension and cannot be the batch norm axis - if 0 in self.axis: - raise ValueError('When using virtual_batch_size, the batch dimension ' - 'must be 0 and thus axis cannot include 0') - if self.adjustment is not None: - raise ValueError('When using virtual_batch_size, adjustment cannot ' - 'be specified') - - if self.fused: - # Currently fused batch norm doesn't support renorm. It also only supports - # an input tensor of rank 4 and a channel dimension on axis 1 or 3. - # TODO(yaozhang): if input is not 4D, reshape it to 4D and reshape the - # output back to its original shape accordingly. - self.fused = (not self.renorm and - ndims == 4 and - self.axis in [[1], [3]] and - self.virtual_batch_size is None and - self.adjustment is None) - # TODO(chrisying): fused batch norm is currently not supported for - # multi-axis batch norm and by extension virtual batches. In some cases, - # it might be possible to use fused batch norm but would require reshaping - # the Tensor to 4D with the axis in 1 or 3 (preferred 1) which is - # particularly tricky. A compromise might be to just support the most - # common use case (turning 5D w/ virtual batch to NCHW) - - if self.fused: - if self.axis == [1]: - self._data_format = 'NCHW' - elif self.axis == [3]: - self._data_format = 'NHWC' - else: - raise ValueError('Unsupported axis, fused batch norm only supports ' - 'axis == [1] or axis == [3]') - - # Raise parameters of fp16 batch norm to fp32 - if self.dtype == dtypes.float16 or self.dtype == dtypes.bfloat16: - param_dtype = dtypes.float32 - else: - param_dtype = self.dtype or dtypes.float32 - - axis_to_dim = {x: input_shape[x].value for x in self.axis} - for x in axis_to_dim: - if axis_to_dim[x] is None: - raise ValueError('Input has undefined `axis` dimension. Input shape: ', - input_shape) - self.input_spec = base.InputSpec(ndim=ndims, axes=axis_to_dim) - - if len(axis_to_dim) == 1 and self.virtual_batch_size is None: - # Single axis batch norm (most common/default use-case) - param_shape = (list(axis_to_dim.values())[0],) - else: - # Parameter shape is the original shape but with 1 in all non-axis dims - param_shape = [axis_to_dim[i] if i in axis_to_dim - else 1 for i in range(ndims)] - if self.virtual_batch_size is not None: - # When using virtual batches, add an extra dim at index 1 - param_shape.insert(1, 1) - for idx, x in enumerate(self.axis): - self.axis[idx] = x + 1 # Account for added dimension - - if self.scale: - self.gamma = self.add_variable( - name='gamma', - shape=param_shape, - dtype=param_dtype, - initializer=self.gamma_initializer, - regularizer=self.gamma_regularizer, - constraint=self.gamma_constraint, - trainable=True) - else: - self.gamma = None - if self.fused: - self._gamma_const = array_ops.constant( - 1.0, dtype=param_dtype, shape=param_shape) - - if self.center: - self.beta = self.add_variable( - name='beta', - shape=param_shape, - dtype=param_dtype, - initializer=self.beta_initializer, - regularizer=self.beta_regularizer, - constraint=self.beta_constraint, - trainable=True) - else: - self.beta = None - if self.fused: - self._beta_const = array_ops.constant( - 0.0, dtype=param_dtype, shape=param_shape) - - # Disable variable partitioning when creating the moving mean and variance - try: - if self._scope: - partitioner = self._scope.partitioner - self._scope.set_partitioner(None) - else: - partitioner = None - self.moving_mean = self._add_tower_local_variable( - name='moving_mean', - shape=param_shape, - dtype=param_dtype, - initializer=self.moving_mean_initializer, - trainable=False) - - self.moving_variance = self._add_tower_local_variable( - name='moving_variance', - shape=param_shape, - dtype=param_dtype, - initializer=self.moving_variance_initializer, - trainable=False) - - if self.renorm: - # Create variables to maintain the moving mean and standard deviation. - # These are used in training and thus are different from the moving - # averages above. The renorm variables are colocated with moving_mean - # and moving_variance. - # NOTE: below, the outer `with device` block causes the current device - # stack to be cleared. The nested ones use a `lambda` to set the desired - # device and ignore any devices that may be set by the custom getter. - def _renorm_variable(name, shape): - var = self._add_tower_local_variable( - name=name, - shape=shape, - dtype=param_dtype, - initializer=init_ops.zeros_initializer(), - trainable=False) - return var - - with distribute_lib.get_distribution_strategy().colocate_vars_with( - self.moving_mean): - self.renorm_mean = _renorm_variable('renorm_mean', param_shape) - self.renorm_mean_weight = _renorm_variable('renorm_mean_weight', ()) - # We initialize renorm_stddev to 0, and maintain the (0-initialized) - # renorm_stddev_weight. This allows us to (1) mix the average - # stddev with the minibatch stddev early in training, and (2) compute - # the unbiased average stddev by dividing renorm_stddev by the weight. - with distribute_lib.get_distribution_strategy().colocate_vars_with( - self.moving_variance): - self.renorm_stddev = _renorm_variable('renorm_stddev', param_shape) - self.renorm_stddev_weight = _renorm_variable('renorm_stddev_weight', - ()) - finally: - if partitioner: - self._scope.set_partitioner(partitioner) - self.built = True - - def _assign_moving_average(self, variable, value, momentum): - with ops.name_scope(None, 'AssignMovingAvg', - [variable, value, momentum]) as scope: - decay = ops.convert_to_tensor(1.0 - momentum, name='decay') - if decay.dtype != variable.dtype.base_dtype: - decay = math_ops.cast(decay, variable.dtype.base_dtype) - update_delta = (variable - value) * decay - return state_ops.assign_sub(variable, update_delta, name=scope) - - def _fused_batch_norm(self, inputs, training): - """Returns the output of fused batch norm.""" - beta = self.beta if self.center else self._beta_const - gamma = self.gamma if self.scale else self._gamma_const - - def _fused_batch_norm_training(): - return nn.fused_batch_norm( - inputs, - gamma, - beta, - epsilon=self.epsilon, - data_format=self._data_format) - - def _fused_batch_norm_inference(): - return nn.fused_batch_norm( - inputs, - gamma, - beta, - mean=self.moving_mean, - variance=self.moving_variance, - epsilon=self.epsilon, - is_training=False, - data_format=self._data_format) - - output, mean, variance = utils.smart_cond( - training, _fused_batch_norm_training, _fused_batch_norm_inference) - if not self._bessels_correction_test_only: - # Remove Bessel's correction to be consistent with non-fused batch norm. - # Note that the variance computed by fused batch norm is - # with Bessel's correction. - sample_size = math_ops.cast( - array_ops.size(inputs) / array_ops.size(variance), variance.dtype) - factor = (sample_size - math_ops.cast(1.0, variance.dtype)) / sample_size - variance *= factor - - training_value = utils.constant_value(training) - if training_value is None: - momentum = utils.smart_cond(training, lambda: self.momentum, lambda: 1.0) - else: - momentum = ops.convert_to_tensor(self.momentum) - if training_value or training_value is None: - mean_update = self._assign_moving_average(self.moving_mean, mean, - momentum) - variance_update = self._assign_moving_average(self.moving_variance, - variance, momentum) - self.add_update(mean_update, inputs=inputs) - self.add_update(variance_update, inputs=inputs) - - return output - - def _renorm_correction_and_moments(self, mean, variance, training): - """Returns the correction and update values for renorm.""" - stddev = math_ops.sqrt(variance + self.epsilon) - # Compute the average mean and standard deviation, as if they were - # initialized with this batch's moments. - mixed_renorm_mean = (self.renorm_mean + - (1. - self.renorm_mean_weight) * mean) - mixed_renorm_stddev = (self.renorm_stddev + - (1. - self.renorm_stddev_weight) * stddev) - # Compute the corrections for batch renorm. - r = stddev / mixed_renorm_stddev - d = (mean - mixed_renorm_mean) / mixed_renorm_stddev - # Ensure the corrections use pre-update moving averages. - with ops.control_dependencies([r, d]): - mean = array_ops.identity(mean) - stddev = array_ops.identity(stddev) - rmin, rmax, dmax = [self.renorm_clipping.get(key) - for key in ['rmin', 'rmax', 'dmax']] - if rmin is not None: - r = math_ops.maximum(r, rmin) - if rmax is not None: - r = math_ops.minimum(r, rmax) - if dmax is not None: - d = math_ops.maximum(d, -dmax) - d = math_ops.minimum(d, dmax) - # When not training, use r=1, d=0. - r = utils.smart_cond(training, lambda: r, lambda: array_ops.ones_like(r)) - d = utils.smart_cond(training, lambda: d, lambda: array_ops.zeros_like(d)) - - def _update_renorm_variable(var, weight, value): - """Updates a moving average and weight, returns the unbiased value.""" - value = array_ops.identity(value) - def _do_update(): - """Updates the var and weight, returns their updated ratio.""" - # Update the variables without zero debiasing. The debiasing will be - # accomplished by dividing the exponential moving average by the weight. - # For example, after a single update, the moving average would be - # (1-decay) * value. and the weight will be 1-decay, with their ratio - # giving the value. - # Make sure the weight is not updated until before r and d computation. - with ops.control_dependencies([value]): - weight_value = array_ops.constant(1., dtype=weight.dtype) - new_var = self._assign_moving_average(var, value, self.renorm_momentum) - new_weight = self._assign_moving_average(weight, weight_value, - self.renorm_momentum) - # TODO(yuefengz): the updates to var and weighted can not be batched - # together if we fetch their updated values here. Consider calculating - # new values and delaying the updates. - return new_var / new_weight - - def _fake_update(): - return array_ops.identity(var) - return utils.smart_cond(training, _do_update, _fake_update) - - # TODO(yuefengz): colocate the operations - new_mean = _update_renorm_variable(self.renorm_mean, - self.renorm_mean_weight, mean) - new_stddev = _update_renorm_variable(self.renorm_stddev, - self.renorm_stddev_weight, stddev) - # Make sqrt(moving_variance + epsilon) = new_stddev. - new_variance = math_ops.square(new_stddev) - self.epsilon - - return (r, d, new_mean, new_variance) + axis=axis, + momentum=momentum, + epsilon=epsilon, + center=center, + scale=scale, + beta_initializer=beta_initializer, + gamma_initializer=gamma_initializer, + moving_mean_initializer=moving_mean_initializer, + moving_variance_initializer=moving_variance_initializer, + beta_regularizer=beta_regularizer, + gamma_regularizer=gamma_regularizer, + beta_constraint=beta_constraint, + gamma_constraint=gamma_constraint, + renorm=renorm, + renorm_clipping=renorm_clipping, + renorm_momentum=renorm_momentum, + fused=fused, + trainable=trainable, + virtual_batch_size=virtual_batch_size, + adjustment=adjustment, + name=name, + **kwargs) def call(self, inputs, training=False): - in_eager_mode = context.executing_eagerly() - if self.virtual_batch_size is not None: - # Virtual batches (aka ghost batches) can be simulated by reshaping the - # Tensor and reusing the existing batch norm implementation - original_shape = [-1] + inputs.shape.as_list()[1:] - expanded_shape = [self.virtual_batch_size, -1] + original_shape[1:] - - # Will cause errors if virtual_batch_size does not divide the batch size - inputs = array_ops.reshape(inputs, expanded_shape) - - def undo_virtual_batching(outputs): - outputs = array_ops.reshape(outputs, original_shape) - return outputs - - if self.fused: - outputs = self._fused_batch_norm(inputs, training=training) - if self.virtual_batch_size is not None: - # Currently never reaches here since fused_batch_norm does not support - # virtual batching - return undo_virtual_batching(outputs) - return outputs - - # Compute the axes along which to reduce the mean / variance - input_shape = inputs.get_shape() - ndims = len(input_shape) - reduction_axes = [i for i in range(ndims) if i not in self.axis] - if self.virtual_batch_size is not None: - del reduction_axes[1] # Do not reduce along virtual batch dim - - # Broadcasting only necessary for single-axis batch norm where the axis is - # not the last dimension - broadcast_shape = [1] * ndims - broadcast_shape[self.axis[0]] = input_shape[self.axis[0]].value - def _broadcast(v): - if (v is not None and - len(v.get_shape()) != ndims and - reduction_axes != list(range(ndims - 1))): - return array_ops.reshape(v, broadcast_shape) - return v - - scale, offset = _broadcast(self.gamma), _broadcast(self.beta) - - def _compose_transforms(scale, offset, then_scale, then_offset): - if then_scale is not None: - scale *= then_scale - offset *= then_scale - if then_offset is not None: - offset += then_offset - return (scale, offset) - - # Determine a boolean value for `training`: could be True, False, or None. - training_value = utils.constant_value(training) - if training_value is not False: - if self.adjustment: - adj_scale, adj_bias = self.adjustment(array_ops.shape(inputs)) - # Adjust only during training. - adj_scale = utils.smart_cond(training, - lambda: adj_scale, - lambda: array_ops.ones_like(adj_scale)) - adj_bias = utils.smart_cond(training, - lambda: adj_bias, - lambda: array_ops.zeros_like(adj_bias)) - scale, offset = _compose_transforms(adj_scale, adj_bias, scale, offset) - - # Some of the computations here are not necessary when training==False - # but not a constant. However, this makes the code simpler. - keep_dims = self.virtual_batch_size is not None or len(self.axis) > 1 - mean, variance = nn.moments(inputs, reduction_axes, keep_dims=keep_dims) - - moving_mean = self.moving_mean - moving_variance = self.moving_variance - - mean = utils.smart_cond(training, - lambda: mean, - lambda: moving_mean) - variance = utils.smart_cond(training, - lambda: variance, - lambda: moving_variance) - - if self.renorm: - r, d, new_mean, new_variance = self._renorm_correction_and_moments( - mean, variance, training) - # When training, the normalized values (say, x) will be transformed as - # x * gamma + beta without renorm, and (x * r + d) * gamma + beta - # = x * (r * gamma) + (d * gamma + beta) with renorm. - r = _broadcast(array_ops.stop_gradient(r, name='renorm_r')) - d = _broadcast(array_ops.stop_gradient(d, name='renorm_d')) - scale, offset = _compose_transforms(r, d, scale, offset) - else: - new_mean, new_variance = mean, variance - - if self.virtual_batch_size is not None: - # This isn't strictly correct since in ghost batch norm, you are - # supposed to sequentially update the moving_mean and moving_variance - # with each sub-batch. However, since the moving statistics are only - # used during evaluation, it is more efficient to just update in one - # step and should not make a significant difference in the result. - new_mean = math_ops.reduce_mean(new_mean, - axis=1, keep_dims=True) - new_variance = math_ops.reduce_mean(new_variance, - axis=1, keep_dims=True) - - def _do_update(var, value): - if in_eager_mode and not self.trainable: - return - - return self._assign_moving_average(var, value, self.momentum) - - mean_update = utils.smart_cond( - training, - lambda: _do_update(self.moving_mean, new_mean), - lambda: self.moving_mean) - variance_update = utils.smart_cond( - training, - lambda: _do_update(self.moving_variance, new_variance), - lambda: self.moving_variance) - if not context.executing_eagerly(): - self.add_update(mean_update, inputs=inputs) - self.add_update(variance_update, inputs=inputs) - - else: - mean, variance = self.moving_mean, self.moving_variance - - outputs = nn.batch_normalization(inputs, - _broadcast(mean), - _broadcast(variance), - offset, - scale, - self.epsilon) - # If some components of the shape got lost due to adjustments, fix that. - outputs.set_shape(input_shape) - - if self.virtual_batch_size is not None: - return undo_virtual_batching(outputs) - - return outputs - - def compute_output_shape(self, input_shape): - return input_shape + return super(BatchNormalization, self).call(inputs, training=training) @tf_export('layers.batch_normalization') diff --git a/tensorflow/python/layers/pooling.py b/tensorflow/python/layers/pooling.py index 50503ce093..75abe56f51 100644 --- a/tensorflow/python/layers/pooling.py +++ b/tensorflow/python/layers/pooling.py @@ -13,92 +13,19 @@ # limitations under the License. # ============================================================================= -# pylint: disable=unused-import,g-bad-import-order """Contains the pooling layer classes and their functional aliases. """ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.python.eager import context -from tensorflow.python.framework import tensor_shape +from tensorflow.python.keras._impl.keras import layers as keras_layers from tensorflow.python.layers import base -from tensorflow.python.layers import utils -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import nn from tensorflow.python.util.tf_export import tf_export -class _Pooling1D(base.Layer): - """Pooling layer for arbitrary pooling functions, for 1D inputs. - - This class only exists for code reuse. It will never be an exposed API. - - Arguments: - pool_function: The pooling function to apply, e.g. `tf.nn.max_pool`. - pool_size: An integer or tuple/list of a single integer, - representing the size of the pooling window. - strides: An integer or tuple/list of a single integer, specifying the - strides of the pooling operation. - padding: A string. The padding method, either 'valid' or 'same'. - Case-insensitive. - data_format: A string, one of `channels_last` (default) or `channels_first`. - The ordering of the dimensions in the inputs. - `channels_last` corresponds to inputs with shape - `(batch, length, channels)` while `channels_first` corresponds to - inputs with shape `(batch, channels, length)`. - name: A string, the name of the layer. - """ - - def __init__(self, pool_function, pool_size, strides, - padding='valid', data_format='channels_last', - name=None, **kwargs): - super(_Pooling1D, self).__init__(name=name, **kwargs) - self.pool_function = pool_function - self.pool_size = utils.normalize_tuple(pool_size, 1, 'pool_size') - self.strides = utils.normalize_tuple(strides, 1, 'strides') - self.padding = utils.normalize_padding(padding) - self.data_format = utils.normalize_data_format(data_format) - self.input_spec = base.InputSpec(ndim=3) - - def call(self, inputs): - # There is no TF op for 1D pooling, hence we make the inputs 4D. - if self.data_format == 'channels_last': - # input is NWC, make it NHWC - inputs = array_ops.expand_dims(inputs, 1) - # pool on the W dim - pool_shape = (1, 1) + self.pool_size + (1,) - strides = (1, 1) + self.strides + (1,) - data_format = 'NHWC' - else: - # input is NCW, make it NCHW - inputs = array_ops.expand_dims(inputs, 2) - # pool on the W dim - pool_shape = (1, 1, 1) + self.pool_size - strides = (1, 1, 1) + self.strides - data_format = 'NCHW' - - outputs = self.pool_function( - inputs, - ksize=pool_shape, - strides=strides, - padding=self.padding.upper(), - data_format=data_format) - - if self.data_format == 'channels_last': - return array_ops.squeeze(outputs, 1) - else: - return array_ops.squeeze(outputs, 2) - - def compute_output_shape(self, input_shape): - input_shape = tensor_shape.TensorShape(input_shape).as_list() - length = utils.conv_output_length(input_shape[1], self.pool_size[0], - self.padding, self.strides[0]) - return tensor_shape.TensorShape([input_shape[0], length, input_shape[2]]) - - @tf_export('layers.AveragePooling1D') -class AveragePooling1D(_Pooling1D): +class AveragePooling1D(keras_layers.AveragePooling1D, base.Layer): """Average Pooling layer for 1D inputs. Arguments: @@ -119,8 +46,9 @@ class AveragePooling1D(_Pooling1D): def __init__(self, pool_size, strides, padding='valid', data_format='channels_last', name=None, **kwargs): + if strides is None: + raise ValueError('Argument `strides` must not be None.') super(AveragePooling1D, self).__init__( - nn.avg_pool, pool_size=pool_size, strides=strides, padding=padding, @@ -165,7 +93,7 @@ def average_pooling1d(inputs, pool_size, strides, @tf_export('layers.MaxPooling1D') -class MaxPooling1D(_Pooling1D): +class MaxPooling1D(keras_layers.MaxPooling1D, base.Layer): """Max Pooling layer for 1D inputs. Arguments: @@ -186,8 +114,9 @@ class MaxPooling1D(_Pooling1D): def __init__(self, pool_size, strides, padding='valid', data_format='channels_last', name=None, **kwargs): + if strides is None: + raise ValueError('Argument `strides` must not be None.') super(MaxPooling1D, self).__init__( - nn.max_pool, pool_size=pool_size, strides=strides, padding=padding, @@ -231,79 +160,8 @@ def max_pooling1d(inputs, pool_size, strides, return layer.apply(inputs) -class _Pooling2D(base.Layer): - """Pooling layer for arbitrary pooling functions, for 2D inputs (e.g. images). - - This class only exists for code reuse. It will never be an exposed API. - - Arguments: - pool_function: The pooling function to apply, e.g. `tf.nn.max_pool`. - pool_size: An integer or tuple/list of 2 integers: (pool_height, pool_width) - specifying the size of the pooling window. - Can be a single integer to specify the same value for - all spatial dimensions. - strides: An integer or tuple/list of 2 integers, - specifying the strides of the pooling operation. - Can be a single integer to specify the same value for - all spatial dimensions. - padding: A string. The padding method, either 'valid' or 'same'. - Case-insensitive. - data_format: A string, one of `channels_last` (default) or `channels_first`. - The ordering of the dimensions in the inputs. - `channels_last` corresponds to inputs with shape - `(batch, height, width, channels)` while `channels_first` corresponds to - inputs with shape `(batch, channels, height, width)`. - name: A string, the name of the layer. - """ - - def __init__(self, pool_function, pool_size, strides, - padding='valid', data_format='channels_last', - name=None, **kwargs): - super(_Pooling2D, self).__init__(name=name, **kwargs) - self.pool_function = pool_function - self.pool_size = utils.normalize_tuple(pool_size, 2, 'pool_size') - self.strides = utils.normalize_tuple(strides, 2, 'strides') - self.padding = utils.normalize_padding(padding) - self.data_format = utils.normalize_data_format(data_format) - self.input_spec = base.InputSpec(ndim=4) - - def call(self, inputs): - if self.data_format == 'channels_last': - pool_shape = (1,) + self.pool_size + (1,) - strides = (1,) + self.strides + (1,) - else: - pool_shape = (1, 1) + self.pool_size - strides = (1, 1) + self.strides - outputs = self.pool_function( - inputs, - ksize=pool_shape, - strides=strides, - padding=self.padding.upper(), - data_format=utils.convert_data_format(self.data_format, 4)) - return outputs - - def compute_output_shape(self, input_shape): - input_shape = tensor_shape.TensorShape(input_shape).as_list() - if self.data_format == 'channels_first': - rows = input_shape[2] - cols = input_shape[3] - else: - rows = input_shape[1] - cols = input_shape[2] - rows = utils.conv_output_length(rows, self.pool_size[0], self.padding, - self.strides[0]) - cols = utils.conv_output_length(cols, self.pool_size[1], self.padding, - self.strides[1]) - if self.data_format == 'channels_first': - return tensor_shape.TensorShape( - [input_shape[0], input_shape[1], rows, cols]) - else: - return tensor_shape.TensorShape( - [input_shape[0], rows, cols, input_shape[3]]) - - @tf_export('layers.AveragePooling2D') -class AveragePooling2D(_Pooling2D): +class AveragePooling2D(keras_layers.AveragePooling2D, base.Layer): """Average pooling layer for 2D inputs (e.g. images). Arguments: @@ -328,8 +186,9 @@ class AveragePooling2D(_Pooling2D): def __init__(self, pool_size, strides, padding='valid', data_format='channels_last', name=None, **kwargs): + if strides is None: + raise ValueError('Argument `strides` must not be None.') super(AveragePooling2D, self).__init__( - nn.avg_pool, pool_size=pool_size, strides=strides, padding=padding, data_format=data_format, name=name, **kwargs) @@ -373,7 +232,7 @@ def average_pooling2d(inputs, @tf_export('layers.MaxPooling2D') -class MaxPooling2D(_Pooling2D): +class MaxPooling2D(keras_layers.MaxPooling2D, base.Layer): """Max pooling layer for 2D inputs (e.g. images). Arguments: @@ -398,8 +257,9 @@ class MaxPooling2D(_Pooling2D): def __init__(self, pool_size, strides, padding='valid', data_format='channels_last', name=None, **kwargs): + if strides is None: + raise ValueError('Argument `strides` must not be None.') super(MaxPooling2D, self).__init__( - nn.max_pool, pool_size=pool_size, strides=strides, padding=padding, data_format=data_format, name=name, **kwargs) @@ -442,90 +302,8 @@ def max_pooling2d(inputs, return layer.apply(inputs) -class _Pooling3D(base.Layer): - """Pooling layer for arbitrary pooling functions, for 3D inputs. - - This class only exists for code reuse. It will never be an exposed API. - - Arguments: - pool_function: The pooling function to apply, e.g. `tf.nn.max_pool`. - pool_size: An integer or tuple/list of 3 integers: - (pool_depth, pool_height, pool_width) - specifying the size of the pooling window. - Can be a single integer to specify the same value for - all spatial dimensions. - strides: An integer or tuple/list of 3 integers, - specifying the strides of the pooling operation. - Can be a single integer to specify the same value for - all spatial dimensions. - padding: A string. The padding method, either 'valid' or 'same'. - Case-insensitive. - data_format: A string, one of `channels_last` (default) or `channels_first`. - The ordering of the dimensions in the inputs. - `channels_last` corresponds to inputs with shape - `(batch, depth, height, width, channels)` - while `channels_first` corresponds to - inputs with shape `(batch, channels, depth, height, width)`. - name: A string, the name of the layer. - """ - - def __init__(self, pool_function, pool_size, strides, - padding='valid', data_format='channels_last', - name=None, **kwargs): - super(_Pooling3D, self).__init__(name=name, **kwargs) - self.pool_function = pool_function - self.pool_size = utils.normalize_tuple(pool_size, 3, 'pool_size') - self.strides = utils.normalize_tuple(strides, 3, 'strides') - self.padding = utils.normalize_padding(padding) - self.data_format = utils.normalize_data_format(data_format) - self.input_spec = base.InputSpec(ndim=5) - - def call(self, inputs): - pool_shape = (1,) + self.pool_size + (1,) - strides = (1,) + self.strides + (1,) - - if self.data_format == 'channels_first': - # TF does not support `channels_first` with 3D pooling operations, - # so we must handle this case manually. - # TODO(fchollet): remove this when TF pooling is feature-complete. - inputs = array_ops.transpose(inputs, (0, 2, 3, 4, 1)) - - outputs = self.pool_function( - inputs, - ksize=pool_shape, - strides=strides, - padding=self.padding.upper()) - - if self.data_format == 'channels_first': - outputs = array_ops.transpose(outputs, (0, 4, 1, 2, 3)) - return outputs - - def compute_output_shape(self, input_shape): - input_shape = tensor_shape.TensorShape(input_shape).as_list() - if self.data_format == 'channels_first': - len_dim1 = input_shape[2] - len_dim2 = input_shape[3] - len_dim3 = input_shape[4] - else: - len_dim1 = input_shape[1] - len_dim2 = input_shape[2] - len_dim3 = input_shape[3] - len_dim1 = utils.conv_output_length(len_dim1, self.pool_size[0], - self.padding, self.strides[0]) - len_dim2 = utils.conv_output_length(len_dim2, self.pool_size[1], - self.padding, self.strides[1]) - len_dim3 = utils.conv_output_length(len_dim3, self.pool_size[2], - self.padding, self.strides[2]) - if self.data_format == 'channels_first': - return tensor_shape.TensorShape( - [input_shape[0], input_shape[1], len_dim1, len_dim2, len_dim3]) - else: - return tensor_shape.TensorShape( - [input_shape[0], len_dim1, len_dim2, len_dim3, input_shape[4]]) - - @tf_export('layers.AveragePooling3D') -class AveragePooling3D(_Pooling3D): +class AveragePooling3D(keras_layers.AveragePooling3D, base.Layer): """Average pooling layer for 3D inputs (e.g. volumes). Arguments: @@ -552,8 +330,9 @@ class AveragePooling3D(_Pooling3D): def __init__(self, pool_size, strides, padding='valid', data_format='channels_last', name=None, **kwargs): + if strides is None: + raise ValueError('Argument `strides` must not be None.') super(AveragePooling3D, self).__init__( - nn.avg_pool3d, pool_size=pool_size, strides=strides, padding=padding, data_format=data_format, name=name, **kwargs) @@ -599,7 +378,7 @@ def average_pooling3d(inputs, @tf_export('layers.MaxPooling3D') -class MaxPooling3D(_Pooling3D): +class MaxPooling3D(keras_layers.MaxPooling3D, base.Layer): """Max pooling layer for 3D inputs (e.g. volumes). Arguments: @@ -626,8 +405,9 @@ class MaxPooling3D(_Pooling3D): def __init__(self, pool_size, strides, padding='valid', data_format='channels_last', name=None, **kwargs): + if strides is None: + raise ValueError('Argument `strides` must not be None.') super(MaxPooling3D, self).__init__( - nn.max_pool3d, pool_size=pool_size, strides=strides, padding=padding, data_format=data_format, name=name, **kwargs) diff --git a/tensorflow/python/layers/utils_test.py b/tensorflow/python/layers/utils_test.py index c941aad7bc..7e94dda648 100644 --- a/tensorflow/python/layers/utils_test.py +++ b/tensorflow/python/layers/utils_test.py @@ -19,7 +19,6 @@ from __future__ import division from __future__ import print_function from tensorflow.python.layers import utils -from tensorflow.python.ops import array_ops from tensorflow.python.platform import test @@ -89,33 +88,5 @@ class ConvUtilsTest(test.TestCase): self.assertEqual(6, utils.deconv_output_length(4, 2, 'full', 2)) -class GraphUtilsTest(test.TestCase): - - def testGetReachableFromInputs(self): - - with self.test_session(): - pl_1 = array_ops.placeholder(shape=None, dtype='float32') - pl_2 = array_ops.placeholder(shape=None, dtype='float32') - pl_3 = array_ops.placeholder(shape=None, dtype='float32') - x_1 = pl_1 + pl_2 - x_2 = pl_2 * 2 - x_3 = pl_3 + 1 - x_4 = x_1 + x_2 - x_5 = x_3 * pl_1 - - self.assertEqual( - utils.get_reachable_from_inputs([pl_1]), - {pl_1, x_1, x_4, x_5}) - self.assertEqual( - utils.get_reachable_from_inputs([pl_1, pl_2]), - {pl_1, pl_2, x_1, x_2, x_4, x_5}) - self.assertEqual( - utils.get_reachable_from_inputs([pl_3]), - {pl_3, x_3, x_5}) - self.assertEqual( - utils.get_reachable_from_inputs([x_3]), - {x_3, x_5}) - - if __name__ == '__main__': test.main() diff --git a/tensorflow/python/ops/nn.py b/tensorflow/python/ops/nn.py index ee1a00623a..244702d13b 100644 --- a/tensorflow/python/ops/nn.py +++ b/tensorflow/python/ops/nn.py @@ -126,8 +126,6 @@ from tensorflow.python.ops.nn_impl import * from tensorflow.python.ops.nn_ops import * from tensorflow.python.ops.candidate_sampling_ops import * from tensorflow.python.ops.embedding_ops import * -from tensorflow.python.ops.rnn import * -from tensorflow.python.ops import rnn_cell # pylint: enable=wildcard-import,unused-import diff --git a/tensorflow/tools/api/golden/tensorflow.keras.-model.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.-model.pbtxt index 7be2f4f61f..7713d78b8a 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.-model.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.-model.pbtxt @@ -3,7 +3,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -74,10 +69,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "state_updates" mtype: "" @@ -128,7 +119,7 @@ tf_class { } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.-sequential.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.-sequential.pbtxt index bf361cf805..69b81f75fa 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.-sequential.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.-sequential.pbtxt @@ -4,7 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -15,10 +14,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -75,10 +70,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "state_updates" mtype: "" @@ -133,7 +124,7 @@ tf_class { } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-activation.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-activation.pbtxt index db8f626b98..96272d1b7d 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-activation.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-activation.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.Activation" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-activity-regularization.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-activity-regularization.pbtxt index 809b3a5430..8fd55c8686 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-activity-regularization.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-activity-regularization.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.ActivityRegularization" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-add.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-add.pbtxt index 68d41bb6cc..47d1532c3c 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-add.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-add.pbtxt @@ -3,7 +3,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -66,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -104,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-alpha-dropout.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-alpha-dropout.pbtxt index 970b777e51..797d422a90 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-alpha-dropout.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-alpha-dropout.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.AlphaDropout" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling1-d.pbtxt index 529c64ab29..269be1455b 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling1-d.pbtxt @@ -1,10 +1,8 @@ path: "tensorflow.keras.layers.AveragePooling1D" tf_class { is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -15,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -67,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -93,7 +83,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'pool_size\', \'strides\', \'padding\'], varargs=None, keywords=kwargs, defaults=[\'2\', \'None\', \'valid\'], " + argspec: "args=[\'self\', \'pool_size\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=kwargs, defaults=[\'2\', \'None\', \'valid\', \'None\'], " } member_method { name: "add_loss" @@ -105,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling2-d.pbtxt index 7e7c330d74..3448136215 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling2-d.pbtxt @@ -1,10 +1,8 @@ path: "tensorflow.keras.layers.AveragePooling2D" tf_class { is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -15,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -67,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -105,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling3-d.pbtxt index ada8466d74..979008d0ed 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling3-d.pbtxt @@ -1,10 +1,8 @@ path: "tensorflow.keras.layers.AveragePooling3D" tf_class { is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -15,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -67,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -105,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average.pbtxt index 2a5c1cd530..0ffdffd4cd 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average.pbtxt @@ -3,7 +3,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -66,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -104,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool1-d.pbtxt index 9a2cb29815..6b00f110ee 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool1-d.pbtxt @@ -1,10 +1,8 @@ path: "tensorflow.keras.layers.AvgPool1D" tf_class { is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -15,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -67,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -93,7 +83,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'pool_size\', \'strides\', \'padding\'], varargs=None, keywords=kwargs, defaults=[\'2\', \'None\', \'valid\'], " + argspec: "args=[\'self\', \'pool_size\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=kwargs, defaults=[\'2\', \'None\', \'valid\', \'None\'], " } member_method { name: "add_loss" @@ -105,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool2-d.pbtxt index f5e991ea42..caff5a2f1d 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool2-d.pbtxt @@ -1,10 +1,8 @@ path: "tensorflow.keras.layers.AvgPool2D" tf_class { is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -15,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -67,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -105,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool3-d.pbtxt index 31732214a6..4a72394921 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool3-d.pbtxt @@ -1,10 +1,8 @@ path: "tensorflow.keras.layers.AvgPool3D" tf_class { is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -15,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -67,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -105,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-batch-normalization.pbtxt index 422eddf10d..9804394fa5 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-batch-normalization.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-batch-normalization.pbtxt @@ -1,9 +1,7 @@ path: "tensorflow.keras.layers.BatchNormalization" tf_class { is_instance: "" - is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -66,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -92,7 +82,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'axis\', \'momentum\', \'epsilon\', \'center\', \'scale\', \'beta_initializer\', \'gamma_initializer\', \'moving_mean_initializer\', \'moving_variance_initializer\', \'beta_regularizer\', \'gamma_regularizer\', \'beta_constraint\', \'gamma_constraint\'], varargs=None, keywords=kwargs, defaults=[\'-1\', \'0.99\', \'0.001\', \'True\', \'True\', \'zeros\', \'ones\', \'zeros\', \'ones\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'axis\', \'momentum\', \'epsilon\', \'center\', \'scale\', \'beta_initializer\', \'gamma_initializer\', \'moving_mean_initializer\', \'moving_variance_initializer\', \'beta_regularizer\', \'gamma_regularizer\', \'beta_constraint\', \'gamma_constraint\', \'renorm\', \'renorm_clipping\', \'renorm_momentum\', \'fused\', \'trainable\', \'virtual_batch_size\', \'adjustment\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'-1\', \'0.99\', \'0.001\', \'True\', \'True\', \'zeros\', \'ones\', \'zeros\', \'ones\', \'None\', \'None\', \'None\', \'None\', \'False\', \'None\', \'0.99\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "add_loss" @@ -104,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-bidirectional.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-bidirectional.pbtxt index 9053a37916..5e5b04c7c6 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-bidirectional.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-bidirectional.pbtxt @@ -3,7 +3,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -18,10 +17,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -70,10 +65,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable" mtype: "" @@ -112,11 +103,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-concatenate.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-concatenate.pbtxt index 3d536d2182..b8eb4079b9 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-concatenate.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-concatenate.pbtxt @@ -3,7 +3,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -66,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -104,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt index a535f18170..3fdb101425 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt @@ -4,7 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -47,10 +46,6 @@ tf_class { name: "filters" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -139,10 +134,6 @@ tf_class { name: "recurrent_regularizer" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "states" mtype: "" @@ -193,11 +184,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv1-d.pbtxt index 801a033972..0be42471e3 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv1-d.pbtxt @@ -1,10 +1,8 @@ path: "tensorflow.keras.layers.Conv1D" tf_class { is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -15,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -67,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -93,7 +83,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'filters\', \'kernel_size\', \'strides\', \'padding\', \'dilation_rate\', \'activation\', \'use_bias\', \'kernel_initializer\', \'bias_initializer\', \'kernel_regularizer\', \'bias_regularizer\', \'activity_regularizer\', \'kernel_constraint\', \'bias_constraint\'], varargs=None, keywords=kwargs, defaults=[\'1\', \'valid\', \'1\', \'None\', \'True\', \'glorot_uniform\', \'zeros\', \'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'filters\', \'kernel_size\', \'strides\', \'padding\', \'data_format\', \'dilation_rate\', \'activation\', \'use_bias\', \'kernel_initializer\', \'bias_initializer\', \'kernel_regularizer\', \'bias_regularizer\', \'activity_regularizer\', \'kernel_constraint\', \'bias_constraint\'], varargs=None, keywords=kwargs, defaults=[\'1\', \'valid\', \'channels_last\', \'1\', \'None\', \'True\', \'glorot_uniform\', \'zeros\', \'None\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "add_loss" @@ -105,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d-transpose.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d-transpose.pbtxt index 13352e264a..39ba31a709 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d-transpose.pbtxt @@ -1,11 +1,9 @@ path: "tensorflow.keras.layers.Conv2DTranspose" tf_class { is_instance: "" - is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -16,10 +14,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -68,10 +62,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -106,11 +96,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d.pbtxt index f400e4a15c..26d9d8c476 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d.pbtxt @@ -1,10 +1,8 @@ path: "tensorflow.keras.layers.Conv2D" tf_class { is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -15,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -67,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -105,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d-transpose.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d-transpose.pbtxt index b3a9f573b8..43611017fa 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d-transpose.pbtxt @@ -1,11 +1,9 @@ path: "tensorflow.keras.layers.Conv3DTranspose" tf_class { is_instance: "" - is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -16,10 +14,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -68,10 +62,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -106,11 +96,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d.pbtxt index a9be09c0ab..fa4925ab99 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d.pbtxt @@ -1,10 +1,8 @@ path: "tensorflow.keras.layers.Conv3D" tf_class { is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -15,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -67,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -105,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution1-d.pbtxt index be1ef5eb92..c5c5d5e7c0 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution1-d.pbtxt @@ -1,10 +1,8 @@ path: "tensorflow.keras.layers.Convolution1D" tf_class { is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -15,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -67,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -93,7 +83,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'filters\', \'kernel_size\', \'strides\', \'padding\', \'dilation_rate\', \'activation\', \'use_bias\', \'kernel_initializer\', \'bias_initializer\', \'kernel_regularizer\', \'bias_regularizer\', \'activity_regularizer\', \'kernel_constraint\', \'bias_constraint\'], varargs=None, keywords=kwargs, defaults=[\'1\', \'valid\', \'1\', \'None\', \'True\', \'glorot_uniform\', \'zeros\', \'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'filters\', \'kernel_size\', \'strides\', \'padding\', \'data_format\', \'dilation_rate\', \'activation\', \'use_bias\', \'kernel_initializer\', \'bias_initializer\', \'kernel_regularizer\', \'bias_regularizer\', \'activity_regularizer\', \'kernel_constraint\', \'bias_constraint\'], varargs=None, keywords=kwargs, defaults=[\'1\', \'valid\', \'channels_last\', \'1\', \'None\', \'True\', \'glorot_uniform\', \'zeros\', \'None\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "add_loss" @@ -105,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt index 30034f7eaf..36dc2d2e9a 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt @@ -1,11 +1,9 @@ path: "tensorflow.keras.layers.Convolution2DTranspose" tf_class { is_instance: "" - is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -16,10 +14,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -68,10 +62,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -106,11 +96,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d.pbtxt index 189b38054c..23ec74370b 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d.pbtxt @@ -1,10 +1,8 @@ path: "tensorflow.keras.layers.Convolution2D" tf_class { is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -15,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -67,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -105,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt index a76d85c629..0e4089c578 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt @@ -1,11 +1,9 @@ path: "tensorflow.keras.layers.Convolution3DTranspose" tf_class { is_instance: "" - is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -16,10 +14,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -68,10 +62,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -106,11 +96,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d.pbtxt index 782195d4ad..23ddbe1a92 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d.pbtxt @@ -1,10 +1,8 @@ path: "tensorflow.keras.layers.Convolution3D" tf_class { is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -15,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -67,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -105,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping1-d.pbtxt index 2cb7a39ea5..e04ab6bea8 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping1-d.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.Cropping1D" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping2-d.pbtxt index 8080330699..655314afff 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping2-d.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.Cropping2D" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping3-d.pbtxt index 678f40bbc2..d5215f1330 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping3-d.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.Cropping3D" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-dense.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-dense.pbtxt index fac826109b..310a3c3b91 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-dense.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-dense.pbtxt @@ -1,9 +1,7 @@ path: "tensorflow.keras.layers.Dense" tf_class { is_instance: "" - is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -66,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -104,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt index b38716aa2c..2d67b5f720 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt @@ -2,10 +2,8 @@ path: "tensorflow.keras.layers.DepthwiseConv2D" tf_class { is_instance: "" is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -16,10 +14,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -68,10 +62,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -106,11 +96,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-dot.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-dot.pbtxt index 285d544af2..0e493a7f2b 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-dot.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-dot.pbtxt @@ -3,7 +3,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -66,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -104,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-dropout.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-dropout.pbtxt index b77976974c..14726b4b6c 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-dropout.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-dropout.pbtxt @@ -1,9 +1,7 @@ path: "tensorflow.keras.layers.Dropout" tf_class { is_instance: "" - is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -66,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -104,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-e-l-u.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-e-l-u.pbtxt index b07714d3f2..32a50455ed 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-e-l-u.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-e-l-u.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.ELU" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-embedding.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-embedding.pbtxt index e67d4ddfc4..2f615d8112 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-embedding.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-embedding.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.Embedding" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-flatten.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-flatten.pbtxt index b2a668e5a8..82dc878a8c 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-flatten.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-flatten.pbtxt @@ -1,9 +1,7 @@ path: "tensorflow.keras.layers.Flatten" tf_class { is_instance: "" - is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -66,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -104,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-g-r-u-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-g-r-u-cell.pbtxt index 4274b8d425..d79d02b954 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-g-r-u-cell.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-g-r-u-cell.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.GRUCell" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-g-r-u.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-g-r-u.pbtxt index 8d9f06083c..1d38ae64bb 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-g-r-u.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-g-r-u.pbtxt @@ -3,7 +3,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -34,10 +33,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "implementation" mtype: "" @@ -126,10 +121,6 @@ tf_class { name: "reset_after" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "states" mtype: "" @@ -176,11 +167,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-dropout.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-dropout.pbtxt index f4f1a5d51c..135de9cd95 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-dropout.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-dropout.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.GaussianDropout" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-noise.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-noise.pbtxt index e502df5e17..5db6e433ee 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-noise.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-noise.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.GaussianNoise" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt index 9c8d5bfcd8..bf0dba0a92 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt @@ -1,9 +1,8 @@ path: "tensorflow.keras.layers.GlobalAveragePooling1D" tf_class { is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -66,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -104,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt index 8dd65f1f24..6da9803609 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt @@ -1,9 +1,8 @@ path: "tensorflow.keras.layers.GlobalAveragePooling2D" tf_class { is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -66,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -104,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt index 5e30571cc7..345593dec8 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt @@ -1,9 +1,8 @@ path: "tensorflow.keras.layers.GlobalAveragePooling3D" tf_class { is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -66,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -104,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt index ba90fa4546..5d3be9085e 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt @@ -1,9 +1,8 @@ path: "tensorflow.keras.layers.GlobalAvgPool1D" tf_class { is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -66,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -104,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt index 8823857758..0b79a87e05 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt @@ -1,9 +1,8 @@ path: "tensorflow.keras.layers.GlobalAvgPool2D" tf_class { is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -66,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -104,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt index 500ced852b..68cdbac652 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt @@ -1,9 +1,8 @@ path: "tensorflow.keras.layers.GlobalAvgPool3D" tf_class { is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -66,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -104,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool1-d.pbtxt index cf2717ed46..d5872b444f 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool1-d.pbtxt @@ -1,9 +1,8 @@ path: "tensorflow.keras.layers.GlobalMaxPool1D" tf_class { is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -66,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -104,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool2-d.pbtxt index a86ff1a469..4b0cf9a5d3 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool2-d.pbtxt @@ -1,9 +1,8 @@ path: "tensorflow.keras.layers.GlobalMaxPool2D" tf_class { is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -66,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -104,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool3-d.pbtxt index e01cc7c1b0..4c1adb2131 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool3-d.pbtxt @@ -1,9 +1,8 @@ path: "tensorflow.keras.layers.GlobalMaxPool3D" tf_class { is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -66,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -104,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt index 259c1fb37c..815f1cf580 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt @@ -1,9 +1,8 @@ path: "tensorflow.keras.layers.GlobalMaxPooling1D" tf_class { is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -66,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -104,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt index 0c41bf97f7..e027dd6cc2 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt @@ -1,9 +1,8 @@ path: "tensorflow.keras.layers.GlobalMaxPooling2D" tf_class { is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -66,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -104,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt index bec8817aa3..c647b24a23 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt @@ -1,9 +1,8 @@ path: "tensorflow.keras.layers.GlobalMaxPooling3D" tf_class { is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -66,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -104,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-input-layer.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-input-layer.pbtxt index 17be862229..75d70734b4 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-input-layer.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-input-layer.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.InputLayer" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-input-spec.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-input-spec.pbtxt index 3aeef347ae..29edabe048 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-input-spec.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-input-spec.pbtxt @@ -1,6 +1,6 @@ path: "tensorflow.keras.layers.InputSpec" tf_class { - is_instance: "" + is_instance: "" is_instance: "" member_method { name: "__init__" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt index 6d2a8c5619..0ed383a355 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.LSTMCell" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-l-s-t-m.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-l-s-t-m.pbtxt index 490b5b618c..6d14c9c8f6 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-l-s-t-m.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-l-s-t-m.pbtxt @@ -3,7 +3,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -34,10 +33,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "implementation" mtype: "" @@ -122,10 +117,6 @@ tf_class { name: "recurrent_regularizer" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "states" mtype: "" @@ -176,11 +167,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-lambda.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-lambda.pbtxt index 21a65b838a..ddf96aba34 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-lambda.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-lambda.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.Lambda" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-layer.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-layer.pbtxt index 127b04738e..aca282d624 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-layer.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-layer.pbtxt @@ -1,7 +1,6 @@ path: "tensorflow.keras.layers.Layer" tf_class { is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -12,10 +11,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -64,10 +59,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -90,7 +81,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\'], varargs=None, keywords=kwargs, defaults=None" + argspec: "args=[\'self\', \'trainable\', \'name\', \'dtype\'], varargs=None, keywords=kwargs, defaults=[\'True\', \'None\', \'None\'], " } member_method { name: "add_loss" @@ -102,11 +93,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-leaky-re-l-u.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-leaky-re-l-u.pbtxt index 87e49f2ed5..b9c53b43c8 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-leaky-re-l-u.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-leaky-re-l-u.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.LeakyReLU" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected1-d.pbtxt index 1aa3aad324..2ee566d03b 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected1-d.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.LocallyConnected1D" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected2-d.pbtxt index 5e9dc7d477..db0d0e816a 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected2-d.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.LocallyConnected2D" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-masking.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-masking.pbtxt index 0d101e5b68..82008b89d0 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-masking.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-masking.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.Masking" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool1-d.pbtxt index c85cd49ac8..31a34a17d0 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool1-d.pbtxt @@ -1,10 +1,8 @@ path: "tensorflow.keras.layers.MaxPool1D" tf_class { is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -15,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -67,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -93,7 +83,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'pool_size\', \'strides\', \'padding\'], varargs=None, keywords=kwargs, defaults=[\'2\', \'None\', \'valid\'], " + argspec: "args=[\'self\', \'pool_size\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=kwargs, defaults=[\'2\', \'None\', \'valid\', \'None\'], " } member_method { name: "add_loss" @@ -105,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool2-d.pbtxt index 4f59e330c9..70d24ac75c 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool2-d.pbtxt @@ -1,10 +1,8 @@ path: "tensorflow.keras.layers.MaxPool2D" tf_class { is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -15,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -67,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -105,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool3-d.pbtxt index c0ea0eb050..55b16564b3 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool3-d.pbtxt @@ -1,10 +1,8 @@ path: "tensorflow.keras.layers.MaxPool3D" tf_class { is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -15,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -67,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -105,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling1-d.pbtxt index ca37ae5131..a230b74c38 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling1-d.pbtxt @@ -1,10 +1,8 @@ path: "tensorflow.keras.layers.MaxPooling1D" tf_class { is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -15,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -67,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -93,7 +83,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'pool_size\', \'strides\', \'padding\'], varargs=None, keywords=kwargs, defaults=[\'2\', \'None\', \'valid\'], " + argspec: "args=[\'self\', \'pool_size\', \'strides\', \'padding\', \'data_format\'], varargs=None, keywords=kwargs, defaults=[\'2\', \'None\', \'valid\', \'None\'], " } member_method { name: "add_loss" @@ -105,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling2-d.pbtxt index 3ede237834..d98f7c39f5 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling2-d.pbtxt @@ -1,10 +1,8 @@ path: "tensorflow.keras.layers.MaxPooling2D" tf_class { is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -15,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -67,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -105,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling3-d.pbtxt index d87e25a7ba..b2e96a4203 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling3-d.pbtxt @@ -1,10 +1,8 @@ path: "tensorflow.keras.layers.MaxPooling3D" tf_class { is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -15,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -67,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -105,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-maximum.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-maximum.pbtxt index e4df7b48ae..0c45bbdf17 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-maximum.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-maximum.pbtxt @@ -3,7 +3,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -66,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -104,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-multiply.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-multiply.pbtxt index 6bf7c77743..6423d83418 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-multiply.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-multiply.pbtxt @@ -3,7 +3,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -66,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -104,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-p-re-l-u.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-p-re-l-u.pbtxt index c14be132b7..6e17081375 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-p-re-l-u.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-p-re-l-u.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.PReLU" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-permute.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-permute.pbtxt index 72ffbceae0..d01d371da5 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-permute.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-permute.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.Permute" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-r-n-n.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-r-n-n.pbtxt index d3e780c8b2..d3f5508640 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-r-n-n.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-r-n-n.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.RNN" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "states" mtype: "" @@ -107,11 +98,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-repeat-vector.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-repeat-vector.pbtxt index a27980a9d1..44e1007f54 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-repeat-vector.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-repeat-vector.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.RepeatVector" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-reshape.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-reshape.pbtxt index 67f991276c..8fc3ec3331 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-reshape.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-reshape.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.Reshape" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-conv1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-conv1-d.pbtxt index fccea5e8af..457d277495 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-conv1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-conv1-d.pbtxt @@ -1,11 +1,9 @@ path: "tensorflow.keras.layers.SeparableConv1D" tf_class { is_instance: "" - is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -16,10 +14,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -68,10 +62,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -106,11 +96,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-conv2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-conv2-d.pbtxt index d20663bdb0..54eda8ee21 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-conv2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-conv2-d.pbtxt @@ -1,11 +1,9 @@ path: "tensorflow.keras.layers.SeparableConv2D" tf_class { is_instance: "" - is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -16,10 +14,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -68,10 +62,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -106,11 +96,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-convolution1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-convolution1-d.pbtxt index 889fa0a1b5..7111965546 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-convolution1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-convolution1-d.pbtxt @@ -1,11 +1,9 @@ path: "tensorflow.keras.layers.SeparableConvolution1D" tf_class { is_instance: "" - is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -16,10 +14,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -68,10 +62,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -106,11 +96,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-convolution2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-convolution2-d.pbtxt index c850f3fedc..815e34a48d 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-convolution2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-convolution2-d.pbtxt @@ -1,11 +1,9 @@ path: "tensorflow.keras.layers.SeparableConvolution2D" tf_class { is_instance: "" - is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -16,10 +14,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -68,10 +62,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -106,11 +96,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt index 526d88ccba..6614760e5e 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.SimpleRNNCell" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-simple-r-n-n.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-simple-r-n-n.pbtxt index 7fddae3447..bfcfd71ecd 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-simple-r-n-n.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-simple-r-n-n.pbtxt @@ -3,7 +3,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -34,10 +33,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -114,10 +109,6 @@ tf_class { name: "recurrent_regularizer" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "states" mtype: "" @@ -164,11 +155,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-softmax.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-softmax.pbtxt index 5b9b62fc97..9c4618c4e9 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-softmax.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-softmax.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.Softmax" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt index 769da30999..9a0a19d2d5 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt @@ -2,9 +2,7 @@ path: "tensorflow.keras.layers.SpatialDropout1D" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -15,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -67,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -105,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt index fca2e42a15..446f7122a6 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt @@ -2,9 +2,7 @@ path: "tensorflow.keras.layers.SpatialDropout2D" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -15,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -67,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -105,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt index 36e8de09a9..52a0485b5c 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt @@ -2,9 +2,7 @@ path: "tensorflow.keras.layers.SpatialDropout3D" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -15,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -67,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -105,11 +95,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt index a96f16fae9..c82e7a192d 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.StackedRNNCells" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "state_size" mtype: "" @@ -107,11 +98,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt index e1cbd0e150..9ccf251a18 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.ThresholdedReLU" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-time-distributed.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-time-distributed.pbtxt index f0d35728fb..e080a07799 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-time-distributed.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-time-distributed.pbtxt @@ -3,7 +3,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -66,10 +61,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable" mtype: "" @@ -108,11 +99,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling1-d.pbtxt index 74efaea6dd..5fadca0b83 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling1-d.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.UpSampling1D" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling2-d.pbtxt index dc5bd5fd53..2d395bf7e8 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling2-d.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.UpSampling2D" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling3-d.pbtxt index e01ccfb74a..18d58ec3b2 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling3-d.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.UpSampling3D" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-wrapper.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-wrapper.pbtxt index 7e6f90f762..6223cb2f3c 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-wrapper.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-wrapper.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.Wrapper" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable" mtype: "" @@ -107,11 +98,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding1-d.pbtxt index 4d0d402dad..e71bba6a7f 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding1-d.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.ZeroPadding1D" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding2-d.pbtxt index b353a529bc..aba6d8cb1f 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding2-d.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.ZeroPadding2D" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding3-d.pbtxt index 9fe1256e61..ce545ecc95 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding3-d.pbtxt @@ -2,7 +2,6 @@ path: "tensorflow.keras.layers.ZeroPadding3D" tf_class { is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -13,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -65,10 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -103,11 +94,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.models.-model.pbtxt index 8ccf15f9ab..3ac285681f 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.models.-model.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.models.-model.pbtxt @@ -3,7 +3,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -14,10 +13,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -74,10 +69,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "state_updates" mtype: "" @@ -128,7 +119,7 @@ tf_class { } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt index be12b0bd2e..51ba0c5043 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt @@ -4,7 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - is_instance: "" is_instance: "" is_instance: "" member { @@ -15,10 +14,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -75,10 +70,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "scope_name" - mtype: "" - } member { name: "state_updates" mtype: "" @@ -133,7 +124,7 @@ tf_class { } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'getter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling1-d.pbtxt index 1c4f550d7f..38fd78a5a8 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling1-d.pbtxt @@ -1,8 +1,10 @@ path: "tensorflow.layers.AveragePooling1D" tf_class { is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -25,6 +27,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -53,6 +59,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -95,7 +105,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -109,6 +123,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -117,10 +135,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -133,6 +163,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -141,4 +175,12 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } } diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling2-d.pbtxt index d2db095269..86a524cc91 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling2-d.pbtxt @@ -1,8 +1,10 @@ path: "tensorflow.layers.AveragePooling2D" tf_class { is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -25,6 +27,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -53,6 +59,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -95,7 +105,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -109,6 +123,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -117,10 +135,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -133,6 +163,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -141,4 +175,12 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } } diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling3-d.pbtxt index 34d9a9df28..8a811fe456 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling3-d.pbtxt @@ -1,8 +1,10 @@ path: "tensorflow.layers.AveragePooling3D" tf_class { is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -25,6 +27,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -53,6 +59,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -95,7 +105,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -109,6 +123,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -117,10 +135,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -133,6 +163,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -141,4 +175,12 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } } diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt index 21ad0efecf..3923e706be 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt @@ -1,7 +1,9 @@ path: "tensorflow.layers.BatchNormalization" tf_class { is_instance: "" + is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -24,6 +26,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -52,6 +58,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -94,7 +104,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -108,6 +122,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\', \'training\'], varargs=None, keywords=None, defaults=[\'False\'], " } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -116,10 +134,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -132,6 +162,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -140,4 +174,12 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } } diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-conv1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-conv1-d.pbtxt index ed38747c76..7a0a8a2a51 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-conv1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-conv1-d.pbtxt @@ -1,8 +1,10 @@ path: "tensorflow.layers.Conv1D" tf_class { is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -25,6 +27,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -53,6 +59,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -95,7 +105,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -109,6 +123,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -117,10 +135,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -133,6 +163,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -141,4 +175,12 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } } diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d-transpose.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d-transpose.pbtxt index ff453c6059..7ed3a65251 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d-transpose.pbtxt @@ -1,9 +1,11 @@ path: "tensorflow.layers.Conv2DTranspose" tf_class { is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -26,6 +28,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -54,6 +60,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -96,7 +106,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -110,6 +124,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -118,10 +136,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -134,6 +164,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -142,4 +176,12 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } } diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d.pbtxt index 5583bd22dc..23831aa74f 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d.pbtxt @@ -1,8 +1,10 @@ path: "tensorflow.layers.Conv2D" tf_class { is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -25,6 +27,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -53,6 +59,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -95,7 +105,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -109,6 +123,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -117,10 +135,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -133,6 +163,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -141,4 +175,12 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } } diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d-transpose.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d-transpose.pbtxt index 63f0c32a7c..9d41a6b099 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d-transpose.pbtxt @@ -1,9 +1,11 @@ path: "tensorflow.layers.Conv3DTranspose" tf_class { is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -26,6 +28,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -54,6 +60,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -96,7 +106,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -110,6 +124,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -118,10 +136,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -134,6 +164,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -142,4 +176,12 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } } diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d.pbtxt index b77726252c..865fe08e63 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d.pbtxt @@ -1,8 +1,10 @@ path: "tensorflow.layers.Conv3D" tf_class { is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -25,6 +27,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -53,6 +59,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -95,7 +105,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -109,6 +123,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -117,10 +135,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -133,6 +163,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -141,4 +175,12 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } } diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-dense.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-dense.pbtxt index 92db9f6dcd..ee164aae20 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-dense.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-dense.pbtxt @@ -1,7 +1,9 @@ path: "tensorflow.layers.Dense" tf_class { is_instance: "" + is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -24,6 +26,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -52,6 +58,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -94,7 +104,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -108,6 +122,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -116,10 +134,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -132,6 +162,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -140,4 +174,12 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } } diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-dropout.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-dropout.pbtxt index 80fa846a24..8167dc79cd 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-dropout.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-dropout.pbtxt @@ -1,7 +1,9 @@ path: "tensorflow.layers.Dropout" tf_class { is_instance: "" + is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -24,6 +26,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -52,6 +58,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -94,7 +104,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -108,6 +122,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\', \'training\'], varargs=None, keywords=None, defaults=[\'False\'], " } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -116,10 +134,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -132,6 +162,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -140,4 +174,12 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } } diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-flatten.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-flatten.pbtxt index f63213b3dd..efa4419692 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-flatten.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-flatten.pbtxt @@ -1,7 +1,9 @@ path: "tensorflow.layers.Flatten" tf_class { is_instance: "" + is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -24,6 +26,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -52,6 +58,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -94,7 +104,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -108,6 +122,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -116,10 +134,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -132,6 +162,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -140,4 +174,12 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } } diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-input-spec.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-input-spec.pbtxt index 7c1d05cd2b..2ff89f0a6f 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-input-spec.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-input-spec.pbtxt @@ -1,6 +1,6 @@ path: "tensorflow.layers.InputSpec" tf_class { - is_instance: "" + is_instance: "" is_instance: "" member_method { name: "__init__" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-layer.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-layer.pbtxt index 4e45b2d513..b3a6dfdffa 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-layer.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-layer.pbtxt @@ -1,6 +1,7 @@ path: "tensorflow.layers.Layer" tf_class { is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -23,6 +24,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -51,6 +56,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -81,7 +90,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'trainable\', \'name\', \'dtype\', \'activity_regularizer\'], varargs=None, keywords=kwargs, defaults=[\'True\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'trainable\', \'name\', \'dtype\'], varargs=None, keywords=kwargs, defaults=[\'True\', \'None\', \'None\'], " } member_method { name: "add_loss" @@ -93,7 +102,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -107,6 +120,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -115,10 +132,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -131,6 +160,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -139,4 +172,12 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } } diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling1-d.pbtxt index 19ec33fce7..cef396489d 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling1-d.pbtxt @@ -1,8 +1,10 @@ path: "tensorflow.layers.MaxPooling1D" tf_class { is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -25,6 +27,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -53,6 +59,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -95,7 +105,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -109,6 +123,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -117,10 +135,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -133,6 +163,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -141,4 +175,12 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } } diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling2-d.pbtxt index 76180c333a..565f0c7a79 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling2-d.pbtxt @@ -1,8 +1,10 @@ path: "tensorflow.layers.MaxPooling2D" tf_class { is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -25,6 +27,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -53,6 +59,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -95,7 +105,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -109,6 +123,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -117,10 +135,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -133,6 +163,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -141,4 +175,12 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } } diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling3-d.pbtxt index ded75c8ff0..595ce2eead 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling3-d.pbtxt @@ -1,8 +1,10 @@ path: "tensorflow.layers.MaxPooling3D" tf_class { is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -25,6 +27,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -53,6 +59,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -95,7 +105,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -109,6 +123,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -117,10 +135,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -133,6 +163,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -141,4 +175,12 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } } diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-separable-conv1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-separable-conv1-d.pbtxt index 3dbfa5453f..ccca96f722 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-separable-conv1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-separable-conv1-d.pbtxt @@ -1,9 +1,11 @@ path: "tensorflow.layers.SeparableConv1D" tf_class { is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -26,6 +28,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -54,6 +60,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -96,7 +106,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -110,6 +124,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -118,10 +136,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -134,6 +164,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -142,4 +176,12 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } } diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-separable-conv2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-separable-conv2-d.pbtxt index ab171df1d1..1c99c96182 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-separable-conv2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-separable-conv2-d.pbtxt @@ -1,9 +1,11 @@ path: "tensorflow.layers.SeparableConv2D" tf_class { is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -26,6 +28,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -54,6 +60,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -96,7 +106,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -110,6 +124,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -118,10 +136,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -134,6 +164,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -142,4 +176,12 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } } diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt index 9c71a24d05..f909cd8756 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt @@ -4,6 +4,7 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -26,6 +27,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -54,6 +59,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -104,7 +113,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -118,6 +131,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\', \'state\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -126,10 +143,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -142,6 +171,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -150,6 +183,14 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "zero_state" argspec: "args=[\'self\', \'batch_size\', \'dtype\'], varargs=None, keywords=None, defaults=None" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt index 9e19f96b74..173d2eae63 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt @@ -4,6 +4,7 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -26,6 +27,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -54,6 +59,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -104,7 +113,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -118,6 +131,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\', \'state\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -126,10 +143,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -142,6 +171,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -150,6 +183,14 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "zero_state" argspec: "args=[\'self\', \'batch_size\', \'dtype\'], varargs=None, keywords=None, defaults=None" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt index 7540aa6286..3c3e382297 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt @@ -3,6 +3,7 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -25,6 +26,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -53,6 +58,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -103,7 +112,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -117,6 +130,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -125,10 +142,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -141,6 +170,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -149,6 +182,14 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "zero_state" argspec: "args=[\'self\', \'batch_size\', \'dtype\'], varargs=None, keywords=None, defaults=None" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt index fc1ff38669..db16660f11 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt @@ -3,6 +3,7 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -25,6 +26,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -53,6 +58,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -107,7 +116,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -121,6 +134,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -129,10 +146,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -145,6 +174,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -153,6 +186,14 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "zero_state" argspec: "args=[\'self\', \'batch_size\', \'dtype\'], varargs=None, keywords=None, defaults=None" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt index 751122cfff..d7f658aaee 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt @@ -4,6 +4,7 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -26,6 +27,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -54,6 +59,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -104,7 +113,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -118,6 +131,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\', \'state\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -126,10 +143,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -142,6 +171,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -150,6 +183,14 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "zero_state" argspec: "args=[\'self\', \'batch_size\', \'dtype\'], varargs=None, keywords=None, defaults=None" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt index 4b6313f395..b9ab487c77 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt @@ -4,6 +4,7 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -26,6 +27,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -54,6 +59,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -104,7 +113,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -118,6 +131,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\', \'state\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -126,10 +143,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -142,6 +171,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -150,6 +183,14 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "zero_state" argspec: "args=[\'self\', \'batch_size\', \'dtype\'], varargs=None, keywords=None, defaults=None" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt index 00e8c71140..b9e3d93475 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt @@ -3,6 +3,7 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -25,6 +26,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -53,6 +58,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -103,7 +112,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -117,6 +130,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\', \'state\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -125,10 +142,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -141,6 +170,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -149,6 +182,14 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "zero_state" argspec: "args=[\'self\', \'batch_size\', \'dtype\'], varargs=None, keywords=None, defaults=None" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt index 3852f90dd6..75b5898c59 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt @@ -2,6 +2,7 @@ path: "tensorflow.nn.rnn_cell.RNNCell" tf_class { is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -24,6 +25,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -52,6 +57,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -90,7 +99,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'trainable\', \'name\', \'dtype\', \'activity_regularizer\'], varargs=None, keywords=kwargs, defaults=[\'True\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'trainable\', \'name\', \'dtype\'], varargs=None, keywords=kwargs, defaults=[\'True\', \'None\', \'None\'], " } member_method { name: "add_loss" @@ -102,7 +111,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -116,6 +129,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -124,10 +141,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -140,6 +169,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -148,6 +181,14 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "zero_state" argspec: "args=[\'self\', \'batch_size\', \'dtype\'], varargs=None, keywords=None, defaults=None" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt index 8f3f0f7506..fee0dc63b9 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt @@ -3,6 +3,7 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member { @@ -25,6 +26,10 @@ tf_class { name: "input" mtype: "" } + member { + name: "input_mask" + mtype: "" + } member { name: "input_shape" mtype: "" @@ -53,6 +58,10 @@ tf_class { name: "output" mtype: "" } + member { + name: "output_mask" + mtype: "" + } member { name: "output_shape" mtype: "" @@ -103,7 +112,11 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "apply" @@ -117,6 +130,10 @@ tf_class { name: "call" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_output_shape" argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" @@ -125,10 +142,22 @@ tf_class { name: "count_params" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_input_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -141,6 +170,10 @@ tf_class { name: "get_output_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_output_shape_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -149,6 +182,14 @@ tf_class { name: "get_updates_for" argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "zero_state" argspec: "args=[\'self\', \'batch_size\', \'dtype\'], varargs=None, keywords=None, defaults=None" -- GitLab From 0932d4af60cd8c9ce322a8e16c8f51d300eb4402 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 13:57:00 -0700 Subject: [PATCH 105/791] Handle duplicate features by coalescing them together into a single feature. PiperOrigin-RevId: 192341065 --- .../python/sdca_estimator_test.py | 53 ++++++++++++++++--- .../linear_optimizer/python/sdca_optimizer.py | 53 ++++++++++++------- 2 files changed, 80 insertions(+), 26 deletions(-) diff --git a/tensorflow/contrib/linear_optimizer/python/sdca_estimator_test.py b/tensorflow/contrib/linear_optimizer/python/sdca_estimator_test.py index 79a5928a21..bed3d5139f 100644 --- a/tensorflow/contrib/linear_optimizer/python/sdca_estimator_test.py +++ b/tensorflow/contrib/linear_optimizer/python/sdca_estimator_test.py @@ -30,6 +30,13 @@ from tensorflow.python.platform import test class SDCALogisticClassifierTest(test.TestCase): + def _single_threaded_test_session(self): + # TODO(andreasst): figure out why SDCALinearRegressor needs a single + # threaded session to pass in tsan mode but SDCALogisticClassifier does not. + config = config_pb2.ConfigProto( + inter_op_parallelism_threads=1, intra_op_parallelism_threads=1) + return self.test_session(config=config) + def testRealValuedFeatures(self): """Tests SDCALogisticClassifier works with real valued features.""" @@ -41,7 +48,7 @@ class SDCALogisticClassifierTest(test.TestCase): 'weights': constant_op.constant([[1.0], [1.0]]) }, constant_op.constant([[0], [1]]) - with self.test_session(): + with self._single_threaded_test_session(): maintenance_cost = feature_column_lib.real_valued_column( 'maintenance_cost') sq_footage = feature_column_lib.real_valued_column('sq_footage') @@ -66,7 +73,7 @@ class SDCALogisticClassifierTest(test.TestCase): constant_op.constant([[500.0, 800.0], [200.0, 600.0]]) }, constant_op.constant([[0], [1]]) - with self.test_session(): + with self._single_threaded_test_session(): dense_feature = feature_column_lib.real_valued_column( 'dense_feature', dimension=2) classifier = sdca_estimator.SDCALogisticClassifier( @@ -86,7 +93,7 @@ class SDCALogisticClassifierTest(test.TestCase): 'weights': constant_op.constant([[1.0], [1.0], [1.0]]) }, constant_op.constant([[1], [0], [1]]) - with self.test_session(): + with self._single_threaded_test_session(): price_bucket = feature_column_lib.bucketized_column( feature_column_lib.real_valued_column('price'), boundaries=[500.0, 700.0]) @@ -120,7 +127,7 @@ class SDCALogisticClassifierTest(test.TestCase): constant_op.constant([[1.0], [1.0], [1.0]]) }, constant_op.constant([[1], [0], [1]]) - with self.test_session(): + with self._single_threaded_test_session(): price = feature_column_lib.real_valued_column('price') country = feature_column_lib.sparse_column_with_hash_bucket( 'country', hash_bucket_size=5) @@ -151,7 +158,7 @@ class SDCALogisticClassifierTest(test.TestCase): dense_shape=[3, 5]) }, constant_op.constant([[1], [0], [1]]) - with self.test_session(): + with self._single_threaded_test_session(): country = feature_column_lib.sparse_column_with_hash_bucket( 'country', hash_bucket_size=5) country_weighted_by_price = feature_column_lib.weighted_sparse_column( @@ -163,6 +170,38 @@ class SDCALogisticClassifierTest(test.TestCase): metrics = classifier.evaluate(input_fn=input_fn, steps=1) self.assertGreater(metrics['accuracy'], 0.9) + def testSparseFeaturesWithDuplicates(self): + """Tests SDCALogisticClassifier with duplicated sparse features.""" + + def input_fn(): + return { + 'example_id': + constant_op.constant(['1', '2']), + 'age': + sparse_tensor.SparseTensor( + values=['20-29'] * 5 + ['31-40'] * 5, + indices=[[0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [1, 0], + [1, 0], [1, 0], [1, 0], [1, 0]], + dense_shape=[2, 1]), + 'gender': + sparse_tensor.SparseTensor( + values=['m'] * 5 + ['f'] * 5, + indices=[[0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [1, 0], + [1, 0], [1, 0], [1, 0], [1, 0]], + dense_shape=[2, 1]), + }, constant_op.constant([[1], [0]]) + + with self._single_threaded_test_session(): + age = feature_column_lib.sparse_column_with_hash_bucket( + 'age', hash_bucket_size=10) + gender = feature_column_lib.sparse_column_with_hash_bucket( + 'gender', hash_bucket_size=10) + classifier = sdca_estimator.SDCALogisticClassifier( + example_id_column='example_id', feature_columns=[age, gender]) + classifier.fit(input_fn=input_fn, steps=50) + metrics = classifier.evaluate(input_fn=input_fn, steps=1) + self.assertLess(metrics['loss'], 0.060) + def testCrossedFeatures(self): """Tests SDCALogisticClassifier with crossed features.""" @@ -182,7 +221,7 @@ class SDCALogisticClassifierTest(test.TestCase): dense_shape=[3, 1]) }, constant_op.constant([[0], [0], [1]]) - with self.test_session(): + with self._single_threaded_test_session(): language = feature_column_lib.sparse_column_with_hash_bucket( 'language', hash_bucket_size=5) country = feature_column_lib.sparse_column_with_hash_bucket( @@ -215,7 +254,7 @@ class SDCALogisticClassifierTest(test.TestCase): constant_op.constant([[3.0], [1.0], [1.0]]) }, constant_op.constant([[1], [0], [1]]) - with self.test_session(): + with self._single_threaded_test_session(): price = feature_column_lib.real_valued_column('price') sq_footage_bucket = feature_column_lib.bucketized_column( feature_column_lib.real_valued_column('sq_footage'), diff --git a/tensorflow/contrib/linear_optimizer/python/sdca_optimizer.py b/tensorflow/contrib/linear_optimizer/python/sdca_optimizer.py index dffdddacfb..5d4572bf6c 100644 --- a/tensorflow/contrib/linear_optimizer/python/sdca_optimizer.py +++ b/tensorflow/contrib/linear_optimizer/python/sdca_optimizer.py @@ -19,6 +19,7 @@ from __future__ import print_function from tensorflow.contrib import layers from tensorflow.contrib.linear_optimizer.python.ops import sdca_ops from tensorflow.contrib.linear_optimizer.python.ops.sparse_feature_column import SparseFeatureColumn +from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops @@ -181,28 +182,42 @@ class SDCAOptimizer(object): elif isinstance( column, ( + layers.feature_column._WeightedSparseColumn, # pylint: disable=protected-access layers.feature_column._CrossedColumn, # pylint: disable=protected-access layers.feature_column._SparseColumn)): # pylint: disable=protected-access - sparse_features.append( - SparseFeatureColumn( - array_ops.reshape( - array_ops.split( - value=transformed_tensor.indices, - num_or_size_splits=2, - axis=1)[0], [-1]), - array_ops.reshape(transformed_tensor.values, [-1]), None)) - sparse_feature_weights.append(columns_to_variables[column][0]) - elif isinstance(column, layers.feature_column._WeightedSparseColumn): # pylint: disable=protected-access - id_tensor = column.id_tensor(transformed_tensor) - weight_tensor = column.weight_tensor(transformed_tensor) + + if isinstance(column, layers.feature_column._WeightedSparseColumn): # pylint: disable=protected-access + id_tensor = column.id_tensor(transformed_tensor) + weight_tensor = array_ops.reshape( + column.weight_tensor(transformed_tensor).values, [-1]) + else: + id_tensor = transformed_tensor + weight_tensor = array_ops.ones( + [array_ops.shape(id_tensor.indices)[0]], dtypes.float32) + + example_ids = array_ops.reshape(id_tensor.indices[:, 0], [-1]) + + flat_ids = array_ops.reshape(id_tensor.values, [-1]) + projection_length = math_ops.reduce_max(flat_ids) + 1 + # project ids based on example ids so that we can dedup ids that + # occur multiple times for a single example. + projected_ids = projection_length * example_ids + flat_ids + + # Remove any redudant ids. + ids, idx = array_ops.unique(projected_ids) + # Keep only one example id per duplicated ids. + example_ids_filtered = math_ops.unsorted_segment_min( + example_ids, idx, + array_ops.shape(ids)[0]) + + # reproject ids back feature id space. + reproject_ids = (ids - projection_length * example_ids_filtered) + + weights = array_ops.reshape( + math_ops.unsorted_segment_sum(weight_tensor, idx, + array_ops.shape(ids)[0]), [-1]) sparse_feature_with_values.append( - SparseFeatureColumn( - array_ops.reshape( - array_ops.split( - value=id_tensor.indices, num_or_size_splits=2, axis=1) - [0], [-1]), - array_ops.reshape(id_tensor.values, [-1]), - array_ops.reshape(weight_tensor.values, [-1]))) + SparseFeatureColumn(example_ids_filtered, reproject_ids, weights)) sparse_feature_with_values_weights.append( columns_to_variables[column][0]) else: -- GitLab From 4995231f9e383b4edc222f63f546b9fa8577fb69 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 13:59:49 -0700 Subject: [PATCH 106/791] test previously untested eval codepaths. PiperOrigin-RevId: 192341561 --- tensorflow/contrib/gan/BUILD | 1 + .../eval/python/classifier_metrics_test.py | 33 ++++++++++++++----- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/tensorflow/contrib/gan/BUILD b/tensorflow/contrib/gan/BUILD index 461066bbb4..b305f37791 100644 --- a/tensorflow/contrib/gan/BUILD +++ b/tensorflow/contrib/gan/BUILD @@ -364,6 +364,7 @@ py_test( "//tensorflow/python:framework_ops", "//tensorflow/python:variables", "//third_party/py/numpy", + "@absl_py//absl/testing:parameterized", ], ) diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py index 663e49bdca..4fb8d58bc9 100644 --- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py +++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py @@ -22,6 +22,7 @@ import os import tarfile import tempfile +from absl.testing import parameterized import numpy as np from scipy import linalg as scp_linalg @@ -182,13 +183,20 @@ def _run_with_mock(function, *args, **kwargs): return function(*args, **kwargs) -class ClassifierMetricsTest(test.TestCase): +class ClassifierMetricsTest(test.TestCase, parameterized.TestCase): - def test_run_inception_graph(self): + @parameterized.named_parameters( + ('GraphDef', False), + ('DefaultGraphDefFn', True)) + def test_run_inception_graph(self, use_default_graph_def): """Test `run_inception` graph construction.""" batch_size = 7 img = array_ops.ones([batch_size, 299, 299, 3]) - logits = _run_with_mock(classifier_metrics.run_inception, img) + + if use_default_graph_def: + logits = _run_with_mock(classifier_metrics.run_inception, img) + else: + logits = classifier_metrics.run_inception(img, _get_dummy_graphdef()) self.assertTrue(isinstance(logits, ops.Tensor)) logits.shape.assert_is_compatible_with([batch_size, 1001]) @@ -196,14 +204,23 @@ class ClassifierMetricsTest(test.TestCase): # Check that none of the model variables are trainable. self.assertListEqual([], variables.trainable_variables()) - def test_run_inception_graph_pool_output(self): + @parameterized.named_parameters( + ('GraphDef', False), + ('DefaultGraphDefFn', True)) + def test_run_inception_graph_pool_output(self, use_default_graph_def): """Test `run_inception` graph construction with pool output.""" batch_size = 3 img = array_ops.ones([batch_size, 299, 299, 3]) - pool = _run_with_mock( - classifier_metrics.run_inception, - img, - output_tensor=classifier_metrics.INCEPTION_FINAL_POOL) + + if use_default_graph_def: + pool = _run_with_mock( + classifier_metrics.run_inception, + img, + output_tensor=classifier_metrics.INCEPTION_FINAL_POOL) + else: + pool = classifier_metrics.run_inception( + img, _get_dummy_graphdef(), + output_tensor=classifier_metrics.INCEPTION_FINAL_POOL) self.assertTrue(isinstance(pool, ops.Tensor)) pool.shape.assert_is_compatible_with([batch_size, 2048]) -- GitLab From 9fe03a590c12b6b52cd561551c31ea2420fa39c7 Mon Sep 17 00:00:00 2001 From: Suharsh Sivakumar Date: Tue, 10 Apr 2018 14:02:02 -0700 Subject: [PATCH 107/791] Pad support for quantized zero. PiperOrigin-RevId: 192342172 --- .../internal/optimized/optimized_ops.h | 28 ++-- .../internal/reference/reference_ops.h | 13 +- tensorflow/contrib/lite/kernels/pad.cc | 27 ++-- tensorflow/contrib/lite/kernels/pad_test.cc | 129 +++++++++++++++--- 4 files changed, 158 insertions(+), 39 deletions(-) diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h index 5acf1eaede..e329e02273 100644 --- a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h +++ b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h @@ -5240,7 +5240,7 @@ template inline void Pad(const T* input_data, const Dims<4>& input_dims, const std::vector& left_paddings, const std::vector& right_paddings, T* output_data, - const Dims<4>& output_dims) { + const Dims<4>& output_dims, const int32_t pad_value) { gemmlowp::ScopedProfilingLabel label("Pad"); const int output_batch = ArraySize(output_dims, 3); const int output_height = ArraySize(output_dims, 2); @@ -5260,27 +5260,27 @@ inline void Pad(const T* input_data, const Dims<4>& input_dims, const int input_depth = ArraySize(input_dims, 0); if (left_b_padding != 0) { - memset(output_data, 0, + memset(output_data, pad_value, left_b_padding * output_height * output_width * output_depth * sizeof(T)); } for (int out_b = left_b_padding; out_b < output_batch - right_b_padding; ++out_b) { if (left_h_padding != 0) { - memset(output_data + Offset(output_dims, 0, 0, 0, out_b), 0, + memset(output_data + Offset(output_dims, 0, 0, 0, out_b), pad_value, left_h_padding * output_width * output_depth * sizeof(T)); } for (int out_h = left_h_padding; out_h < output_height - right_h_padding; ++out_h) { if (left_w_padding != 0) { - memset(output_data + Offset(output_dims, 0, 0, out_h, out_b), 0, + memset(output_data + Offset(output_dims, 0, 0, out_h, out_b), pad_value, left_w_padding * output_depth * sizeof(T)); } for (int out_w = left_w_padding; out_w < output_width - right_w_padding; ++out_w) { if (left_d_padding != 0) { - memset(output_data + Offset(output_dims, 0, out_w, out_h, out_b), 0, - left_d_padding * sizeof(T)); + memset(output_data + Offset(output_dims, 0, out_w, out_h, out_b), + pad_value, left_d_padding * sizeof(T)); } T* out = output_data + @@ -5294,20 +5294,21 @@ inline void Pad(const T* input_data, const Dims<4>& input_dims, memset( output_data + Offset(output_dims, output_depth - right_d_padding, out_w, out_h, out_b), - 0, right_d_padding * sizeof(T)); + pad_value, right_d_padding * sizeof(T)); } } if (right_w_padding != 0) { memset( output_data + Offset(output_dims, 0, output_width - right_w_padding, out_h, out_b), - 0, right_w_padding * output_depth * sizeof(T)); + pad_value, right_w_padding * output_depth * sizeof(T)); } } if (right_h_padding != 0) { memset(output_data + Offset(output_dims, 0, 0, output_height - right_h_padding, out_b), - 0, right_h_padding * output_width * output_depth * sizeof(T)); + pad_value, + right_h_padding * output_width * output_depth * sizeof(T)); } } if (right_b_padding != 0) { @@ -5319,6 +5320,15 @@ inline void Pad(const T* input_data, const Dims<4>& input_dims, } } +template +inline void Pad(const T* input_data, const Dims<4>& input_dims, + const std::vector& left_paddings, + const std::vector& right_paddings, T* output_data, + const Dims<4>& output_dims) { + Pad(input_data, input_dims, left_paddings, right_paddings, output_data, + output_dims, 0); +} + template inline void StridedSlice(const T* input_data, const Dims<4>& input_dims, int begin_mask, int end_mask, diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h index 4bbec52bf7..250a308f2a 100644 --- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h +++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h @@ -2860,7 +2860,7 @@ template inline void Pad(const T* input_data, const Dims<4>& input_dims, const std::vector& left_paddings, const std::vector& right_paddings, T* output_data, - const Dims<4>& output_dims) { + const Dims<4>& output_dims, const int32_t pad_value) { const int output_batch = ArraySize(output_dims, 3); const int output_height = ArraySize(output_dims, 2); const int output_width = ArraySize(output_dims, 1); @@ -2890,7 +2890,7 @@ inline void Pad(const T* input_data, const Dims<4>& input_dims, out_w >= output_width - right_w_padding || out_d < left_d_padding || out_d >= output_depth - right_d_padding) { - *out_ptr++ = 0; + *out_ptr++ = static_cast(pad_value); } else { *out_ptr++ = *in_ptr++; } @@ -2900,6 +2900,15 @@ inline void Pad(const T* input_data, const Dims<4>& input_dims, } } +template +inline void Pad(const T* input_data, const Dims<4>& input_dims, + const std::vector& left_paddings, + const std::vector& right_paddings, T* output_data, + const Dims<4>& output_dims) { + Pad(input_data, input_dims, left_paddings, right_paddings, output_data, + output_dims, 0); +} + inline bool LoopCondition(int index, int stop, int stride) { return stride > 0 ? index < stop : index > stop; } diff --git a/tensorflow/contrib/lite/kernels/pad.cc b/tensorflow/contrib/lite/kernels/pad.cc index c29da3862e..4f9449a225 100644 --- a/tensorflow/contrib/lite/kernels/pad.cc +++ b/tensorflow/contrib/lite/kernels/pad.cc @@ -119,39 +119,46 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { after_padding.push_back(paddings_data[idx * 2 + 1]); } -#define TF_LITE_PAD(type, scalar) \ +#define TF_LITE_PAD(type, scalar, pad_value) \ type::Pad(GetTensorData(op_context.input), \ GetTensorDims(op_context.input), before_padding, after_padding, \ GetTensorData(op_context.output), \ - GetTensorDims(op_context.output)) + GetTensorDims(op_context.output), pad_value) switch (op_context.input->type) { case kTfLiteFloat32: if (kernel_type == kReference) { - TF_LITE_PAD(reference_ops, float); + TF_LITE_PAD(reference_ops, float, 0); } else if (kernel_type == kGenericOptimized) { - TF_LITE_PAD(optimized_ops, float); + TF_LITE_PAD(optimized_ops, float, 0); } break; case kTfLiteUInt8: + // Quantized Pad requires that 0 is represented in the quantized range. + TF_LITE_ENSURE(context, op_context.output->params.zero_point >= + std::numeric_limits::min()); + TF_LITE_ENSURE(context, op_context.output->params.zero_point <= + std::numeric_limits::max()); if (kernel_type == kReference) { - TF_LITE_PAD(reference_ops, uint8_t); + TF_LITE_PAD(reference_ops, uint8_t, + op_context.output->params.zero_point); } else if (kernel_type == kGenericOptimized) { - TF_LITE_PAD(optimized_ops, uint8_t); + TF_LITE_PAD(optimized_ops, uint8_t, + op_context.output->params.zero_point); } break; case kTfLiteInt32: if (kernel_type == kReference) { - TF_LITE_PAD(reference_ops, int32_t); + TF_LITE_PAD(reference_ops, int32_t, 0); } else if (kernel_type == kGenericOptimized) { - TF_LITE_PAD(optimized_ops, int32_t); + TF_LITE_PAD(optimized_ops, int32_t, 0); } break; case kTfLiteInt64: if (kernel_type == kReference) { - TF_LITE_PAD(reference_ops, int64_t); + TF_LITE_PAD(reference_ops, int64_t, 0); } else if (kernel_type == kGenericOptimized) { - TF_LITE_PAD(optimized_ops, int64_t); + TF_LITE_PAD(optimized_ops, int64_t, 0); } break; default: diff --git a/tensorflow/contrib/lite/kernels/pad_test.cc b/tensorflow/contrib/lite/kernels/pad_test.cc index 28834ad071..c06237e572 100644 --- a/tensorflow/contrib/lite/kernels/pad_test.cc +++ b/tensorflow/contrib/lite/kernels/pad_test.cc @@ -22,6 +22,7 @@ namespace tflite { namespace { using ::testing::ElementsAreArray; +using ::testing::Matcher; class PadOpModel : public SingleOpModel { public: @@ -29,6 +30,10 @@ class PadOpModel : public SingleOpModel { PopulateTensor(input_, data); } + void SetQuantizedInput(std::initializer_list data) { + QuantizeAndPopulate(input_, data); + } + void SetPaddings(std::initializer_list paddings) { PopulateTensor(paddings_, paddings); } @@ -36,6 +41,11 @@ class PadOpModel : public SingleOpModel { std::vector GetOutput() { return ExtractVector(output_); } std::vector GetOutputShape() { return GetTensorShape(output_); } + std::vector GetDequantizedOutput() { + return Dequantize(ExtractVector(output_), + GetScale(output_), GetZeroPoint(output_)); + } + protected: int input_; int output_; @@ -50,16 +60,17 @@ class PadOpModel : public SingleOpModel { // m.Invoke(); class PadOpConstModel : public PadOpModel { public: - PadOpConstModel(std::initializer_list input_shape, + PadOpConstModel(const TensorData& input, std::initializer_list paddings_shape, - std::initializer_list paddings) { - input_ = AddInput(TensorType_FLOAT32); + std::initializer_list paddings, + const TensorData& output) { + input_ = AddInput(input); paddings_ = AddConstInput(TensorType_INT32, paddings, paddings_shape); - output_ = AddOutput(TensorType_FLOAT32); + output_ = AddOutput(output); SetBuiltinOp(BuiltinOperator_PAD, BuiltinOptions_PadOptions, CreatePadOptions(builder_).Union()); - BuildInterpreter({input_shape}); + BuildInterpreter({input.shape}); } }; @@ -72,40 +83,45 @@ class PadOpConstModel : public PadOpModel { // m.Invoke(); class PadOpDynamicModel : public PadOpModel { public: - PadOpDynamicModel(std::initializer_list input_shape, - std::initializer_list paddings_shape) { - input_ = AddInput(TensorType_FLOAT32); + PadOpDynamicModel(const TensorData& input, + std::initializer_list paddings_shape, + const TensorData& output) { + input_ = AddInput(input); paddings_ = AddInput(TensorType_INT32); - output_ = AddOutput(TensorType_FLOAT32); + output_ = AddOutput(output); SetBuiltinOp(BuiltinOperator_PAD, BuiltinOptions_PadOptions, CreatePadOptions(builder_).Union()); - BuildInterpreter({input_shape, paddings_shape}); + BuildInterpreter({input.shape, paddings_shape}); } }; TEST(PadOpTest, TooManyDimensions) { EXPECT_DEATH( - PadOpConstModel({1, 2, 3, 4, 5, 6, 7, 8, 9}, {9, 2}, - {1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9}), + PadOpConstModel({TensorType_FLOAT32, {1, 2, 3, 4, 5, 6, 7, 8, 9}}, {9, 2}, + {1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9}, + {TensorType_FLOAT32}), "dims != 4"); } TEST(PadOpTest, UnequalDimensions) { - EXPECT_DEATH(PadOpConstModel({1, 1, 2, 1}, {3, 2}, {1, 1, 2, 2, 3, 3}), + EXPECT_DEATH(PadOpConstModel({TensorType_FLOAT32, {1, 1, 2, 1}}, {3, 2}, + {1, 1, 2, 2, 3, 3}, {TensorType_FLOAT32}), "3 != 4"); } TEST(PadOpTest, InvalidPadValue) { EXPECT_DEATH( - PadOpConstModel({1, 1, 2, 1}, {4, 2}, {0, 0, 1, -1, 2, -1, 0, 0}), + PadOpConstModel({TensorType_FLOAT32, {1, 1, 2, 1}}, {4, 2}, + {0, 0, 1, -1, 2, -1, 0, 0}, {TensorType_FLOAT32}), "Pad value has to be greater than equal to 0."); } TEST(PadOpTest, SimpleConstTest) { // Padding is represented as four 2-D lists representing above padding and // below padding (i.e. {{0, 0}, {1, 1}, {1, 1}, {0, 0}}). - PadOpConstModel m({1, 2, 2, 1}, {4, 2}, {0, 0, 1, 1, 1, 1, 0, 0}); + PadOpConstModel m({TensorType_FLOAT32, {1, 2, 2, 1}}, {4, 2}, + {0, 0, 1, 1, 1, 1, 0, 0}, {TensorType_FLOAT32}); m.SetInput({1, 2, 3, 4}); m.Invoke(); EXPECT_THAT(m.GetOutput(), ElementsAreArray({0, 0, 0, 0, 0, 1, 2, 0, 0, 3, 4, @@ -114,7 +130,8 @@ TEST(PadOpTest, SimpleConstTest) { } TEST(PadOpTest, SimpleDynamicTest) { - PadOpDynamicModel m({1, 2, 2, 1}, {4, 2}); + PadOpDynamicModel m({TensorType_FLOAT32, {1, 2, 2, 1}}, {4, 2}, + {TensorType_FLOAT32}); m.SetInput({1, 2, 3, 4}); m.SetPaddings({0, 0, 1, 1, 1, 1, 0, 0}); m.Invoke(); @@ -124,7 +141,8 @@ TEST(PadOpTest, SimpleDynamicTest) { } TEST(PadOpTest, AdvancedConstTest) { - PadOpConstModel m({1, 2, 3, 1}, {4, 2}, {0, 0, 0, 2, 1, 3, 0, 0}); + PadOpConstModel m({TensorType_FLOAT32, {1, 2, 3, 1}}, {4, 2}, + {0, 0, 0, 2, 1, 3, 0, 0}, {TensorType_FLOAT32}); m.SetInput({1, 2, 3, 4, 5, 6}); m.Invoke(); EXPECT_THAT(m.GetOutput(), @@ -134,7 +152,8 @@ TEST(PadOpTest, AdvancedConstTest) { } TEST(PadOpTest, AdvancedDynamicTest) { - PadOpDynamicModel m({1, 2, 3, 1}, {4, 2}); + PadOpDynamicModel m({TensorType_FLOAT32, {1, 2, 3, 1}}, {4, 2}, + {TensorType_FLOAT32}); m.SetInput({1, 2, 3, 4, 5, 6}); m.SetPaddings({0, 0, 0, 2, 1, 3, 0, 0}); m.Invoke(); @@ -144,6 +163,80 @@ TEST(PadOpTest, AdvancedDynamicTest) { EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 4, 7, 1})); } +class QuantizedPadOpTest : public ::testing::Test { + protected: + std::vector> DequantizedArrayNear( + const std::vector& values, const float min, const float max) { + const float quantization_tolerance = (max - min) / 255.0; + return ArrayFloatNear(values, quantization_tolerance); + } +}; + +TEST_F(QuantizedPadOpTest, ZeroNotInQuantizationRange) { + // The test_util and actual quantization code currently ensure that the range + // must include zero, but if that ever changes, this test will catch it. + EXPECT_DEATH(PadOpConstModel m({TensorType_UINT8, {1, 2, 2, 1}, 1.0, 2.0}, + {4, 2}, {0, 0, 1, 1, 1, 1, 0, 0}, + {TensorType_UINT8, {}, 1.0, 2.0}), + ".*Check failed: f_min <= 0.*"); +} + +TEST_F(QuantizedPadOpTest, SimpleConstTest) { + // Padding is represented as four 2-D lists representing above padding and + // below padding (i.e. {{0, 0}, {1, 1}, {1, 1}, {0, 0}}). + PadOpConstModel m({TensorType_UINT8, {1, 2, 2, 1}, -1.0, 1.0}, {4, 2}, + {0, 0, 1, 1, 1, 1, 0, 0}, + {TensorType_UINT8, {}, -1.0, 1.0}); + m.SetQuantizedInput({-0.8, 0.2, 0.9, 0.7}); + m.Invoke(); + EXPECT_THAT(m.GetDequantizedOutput(), + ElementsAreArray(DequantizedArrayNear( + {0, 0, 0, 0, 0, -0.8, 0.2, 0, 0, 0.9, 0.7, 0, 0, 0, 0, 0}, + -1.0, 1.0))); + EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 4, 4, 1})); +} + +TEST_F(QuantizedPadOpTest, SimpleDynamicTest) { + PadOpDynamicModel m({TensorType_UINT8, {1, 2, 2, 1}, -1.0, 1.0}, {4, 2}, + {TensorType_UINT8, {}, -1.0, 1.0}); + m.SetQuantizedInput({-0.8, 0.2, 0.9, 0.7}); + m.SetPaddings({0, 0, 1, 1, 1, 1, 0, 0}); + m.Invoke(); + EXPECT_THAT(m.GetDequantizedOutput(), + ElementsAreArray(DequantizedArrayNear( + {0, 0, 0, 0, 0, -0.8, 0.2, 0, 0, 0.9, 0.7, 0, 0, 0, 0, 0}, + -1.0, 1.0))); + EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 4, 4, 1})); +} + +TEST_F(QuantizedPadOpTest, AdvancedConstTest) { + PadOpConstModel m({TensorType_UINT8, {1, 2, 3, 1}, -1.0, 1.0}, {4, 2}, + {0, 0, 0, 2, 1, 3, 0, 0}, + {TensorType_UINT8, {}, -1.0, 1.0}); + m.SetQuantizedInput({-0.8, 0.2, 0.9, 0.7, 0.1, -0.3}); + m.Invoke(); + EXPECT_THAT(m.GetDequantizedOutput(), + ElementsAreArray(DequantizedArrayNear( + {0, -0.8, 0.2, 0.9, 0, 0, 0, 0, 0.7, 0.1, -0.3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + -1.0, 1.0))); + EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 4, 7, 1})); +} + +TEST_F(QuantizedPadOpTest, AdvancedDynamicTest) { + PadOpDynamicModel m({TensorType_UINT8, {1, 2, 3, 1}, -1.0, 1.0}, {4, 2}, + {TensorType_UINT8, {}, -1.0, 1.0}); + m.SetQuantizedInput({-0.8, 0.2, 0.9, 0.7, 0.1, -0.3}); + m.SetPaddings({0, 0, 0, 2, 1, 3, 0, 0}); + m.Invoke(); + EXPECT_THAT(m.GetDequantizedOutput(), + ElementsAreArray(DequantizedArrayNear( + {0, -0.8, 0.2, 0.9, 0, 0, 0, 0, 0.7, 0.1, -0.3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + -1.0, 1.0))); + EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 4, 7, 1})); +} + } // namespace } // namespace tflite -- GitLab From 06efb16fb0b9ef7c7ce3d4bc0d5c677b3cbd5a6f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 14:04:29 -0700 Subject: [PATCH 108/791] [XLA] Redesign: implement and test Rev, BitcastConvertType, Map, and ReducePrecision. PiperOrigin-RevId: 192342686 --- .../xla/client/xla_client/xla_builder.cc | 57 +++++++- tensorflow/compiler/xla/tests/BUILD | 8 +- .../xla/tests/bitcast_convert_test.cc | 20 +-- tensorflow/compiler/xla/tests/map_test.cc | 137 +++++++++--------- .../xla/tests/reduce_precision_test.cc | 27 ++-- tensorflow/compiler/xla/tests/reverse_test.cc | 4 +- 6 files changed, 153 insertions(+), 100 deletions(-) diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc index 74d48635eb..7481b357ff 100644 --- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc +++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc @@ -1056,7 +1056,17 @@ XlaOp XlaBuilder::Transpose(const XlaOp& operand, XlaOp XlaBuilder::Rev(const XlaOp& operand, tensorflow::gtl::ArraySlice dimensions) { - return UnimplementedOp(); + return NoteErrorOrReturn([&]() -> StatusOr { + HloInstructionProto instr; + TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); + TF_ASSIGN_OR_RETURN( + *instr.mutable_shape(), + ShapeInference::InferReverseShape(operand_shape, dimensions)); + for (int64 dim : dimensions) { + instr.add_dimensions(dim); + } + return AddInstruction(std::move(instr), HloOpcode::kReverse, {operand}); + }); } XlaOp XlaBuilder::Sort(const XlaOp& operand) { @@ -1087,7 +1097,15 @@ XlaOp XlaBuilder::ConvertElementType(const XlaOp& operand, XlaOp XlaBuilder::BitcastConvertType(const XlaOp& operand, PrimitiveType new_element_type) { - return UnimplementedOp(); + return NoteErrorOrReturn([&]() -> StatusOr { + HloInstructionProto instr; + TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); + TF_ASSIGN_OR_RETURN( + *instr.mutable_shape(), + ShapeInference::InferConvertShape(operand_shape, new_element_type)); + return AddInstruction(std::move(instr), HloOpcode::kBitcastConvert, + {operand}); + }); } XlaOp XlaBuilder::SquareF32(const XlaOp& operand) { @@ -1113,7 +1131,28 @@ XlaOp XlaBuilder::Map(tensorflow::gtl::ArraySlice operands, const XlaComputation& computation, tensorflow::gtl::ArraySlice dimensions, tensorflow::gtl::ArraySlice static_operands) { - return UnimplementedOp(); + return NoteErrorOrReturn([&]() -> StatusOr { + if (!static_operands.empty()) { + return Unimplemented("static_operands is not supported in Map"); + } + + HloInstructionProto instr; + + std::vector operand_shape_ptrs; + TF_ASSIGN_OR_RETURN(const auto& operand_shapes, GetOperandShapes(operands)); + c_transform(operand_shapes, std::back_inserter(operand_shape_ptrs), + [](const Shape& shape) { return &shape; }); + TF_ASSIGN_OR_RETURN(const ProgramShape& called_program_shape, + computation.GetProgramShape()); + TF_ASSIGN_OR_RETURN( + *instr.mutable_shape(), + ShapeInference::InferMapShape(operand_shape_ptrs, called_program_shape, + dimensions)); + + AddCalledComputation(computation, &instr); + + return AddInstruction(std::move(instr), HloOpcode::kMap, operands); + }); } XlaOp XlaBuilder::RngOp(RandomDistribution distribution, @@ -1283,7 +1322,17 @@ XlaOp XlaBuilder::SelectAndScatterWithGeneralPadding( XlaOp XlaBuilder::ReducePrecision(const XlaOp& operand, const int exponent_bits, const int mantissa_bits) { - return UnimplementedOp(); + return NoteErrorOrReturn([&]() -> StatusOr { + HloInstructionProto instr; + TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); + TF_ASSIGN_OR_RETURN(*instr.mutable_shape(), + ShapeInference::InferReducePrecisionShape( + operand_shape, exponent_bits, mantissa_bits)); + instr.set_exponent_bits(exponent_bits); + instr.set_mantissa_bits(mantissa_bits); + return AddInstruction(std::move(instr), HloOpcode::kReducePrecision, + {operand}); + }); } void XlaBuilder::Send(const XlaOp& operand, const ChannelHandle& handle) { diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD index 699b077d80..19fb4886db 100644 --- a/tensorflow/compiler/xla/tests/BUILD +++ b/tensorflow/compiler/xla/tests/BUILD @@ -415,6 +415,8 @@ xla_test( "//tensorflow/compiler/xla/client:global_data", "//tensorflow/compiler/xla/client:local_client", "//tensorflow/compiler/xla/client/lib:arithmetic", + "//tensorflow/compiler/xla/client/xla_client:xla_builder", + "//tensorflow/compiler/xla/client/xla_client:xla_computation", "//tensorflow/compiler/xla/tests:client_library_test_base", "//tensorflow/compiler/xla/tests:literal_test_util", "//tensorflow/compiler/xla/tests:test_utils", @@ -641,9 +643,9 @@ xla_test( "//tensorflow/compiler/xla:test", "//tensorflow/compiler/xla:types", "//tensorflow/compiler/xla:xla_data_proto", - "//tensorflow/compiler/xla/client:computation_builder", "//tensorflow/compiler/xla/client:global_data", "//tensorflow/compiler/xla/client:local_client", + "//tensorflow/compiler/xla/client/xla_client:xla_builder", "//tensorflow/compiler/xla/service:reduce_precision_insertion", "//tensorflow/compiler/xla/tests:client_library_test_base", "//tensorflow/compiler/xla/tests:literal_test_util", @@ -1397,8 +1399,8 @@ xla_test( deps = [ "//tensorflow/compiler/xla:array2d", "//tensorflow/compiler/xla:array4d", - "//tensorflow/compiler/xla/client:computation_builder", "//tensorflow/compiler/xla/client:local_client", + "//tensorflow/compiler/xla/client/xla_client:xla_builder", "//tensorflow/compiler/xla/tests:client_library_test_base", "//tensorflow/compiler/xla/tests:literal_test_util", "//tensorflow/compiler/xla/tests:xla_internal_test_main", @@ -1486,8 +1488,8 @@ xla_test( deps = [ "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:xla_data_proto", - "//tensorflow/compiler/xla/client:computation_builder", "//tensorflow/compiler/xla/client:local_client", + "//tensorflow/compiler/xla/client/xla_client:xla_builder", "//tensorflow/compiler/xla/tests:client_library_test_base", "//tensorflow/compiler/xla/tests:literal_test_util", "//tensorflow/compiler/xla/tests:xla_internal_test_main", diff --git a/tensorflow/compiler/xla/tests/bitcast_convert_test.cc b/tensorflow/compiler/xla/tests/bitcast_convert_test.cc index 0d94d65c10..777ac167a3 100644 --- a/tensorflow/compiler/xla/tests/bitcast_convert_test.cc +++ b/tensorflow/compiler/xla/tests/bitcast_convert_test.cc @@ -18,8 +18,8 @@ limitations under the License. #include #include -#include "tensorflow/compiler/xla/client/computation_builder.h" #include "tensorflow/compiler/xla/client/local_client.h" +#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h" #include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/tests/client_library_test_base.h" #include "tensorflow/compiler/xla/tests/literal_test_util.h" @@ -42,7 +42,7 @@ class BitcastConvertTest : public ClientLibraryTestBase { }; TEST_F(BitcastConvertTest, ConvertR1S32ToR1S32) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto a = builder.ConstantR1({42, 64}); builder.BitcastConvertType(a, S32); @@ -51,7 +51,7 @@ TEST_F(BitcastConvertTest, ConvertR1S32ToR1S32) { } TEST_F(BitcastConvertTest, ConvertR1F32ToR1F32) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto a = builder.ConstantR1({42.0f, 64.0f}); builder.BitcastConvertType(a, F32); @@ -60,7 +60,7 @@ TEST_F(BitcastConvertTest, ConvertR1F32ToR1F32) { } TEST_F(BitcastConvertTest, BitcastR1S32ToR1F32) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto a = builder.ConstantR1({0, static_cast(0x80000000), 0x3F800000, static_cast(0xBF800000), 0x3F000000, @@ -72,7 +72,7 @@ TEST_F(BitcastConvertTest, BitcastR1S32ToR1F32) { } XLA_TEST_F(BitcastConvertTest, ConvertR1S0S32ToR1S0F32) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto a = builder.ConstantR1({}); builder.BitcastConvertType(a, F32); @@ -81,7 +81,7 @@ XLA_TEST_F(BitcastConvertTest, ConvertR1S0S32ToR1S0F32) { } TEST_F(BitcastConvertTest, ConvertR1F32ToR1S32) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto a = builder.ConstantR1({42.6, 64.4}); builder.BitcastConvertType(a, S32); @@ -90,7 +90,7 @@ TEST_F(BitcastConvertTest, ConvertR1F32ToR1S32) { } TEST_F(BitcastConvertTest, ConvertS32Extremes) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto a = builder.ConstantR1( {std::numeric_limits::min(), std::numeric_limits::max()}); builder.BitcastConvertType(a, F32); @@ -100,7 +100,7 @@ TEST_F(BitcastConvertTest, ConvertS32Extremes) { } TEST_F(BitcastConvertTest, ConvertMapToS32) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto b = builder.CreateSubBuilder("convert"); auto param = b->Parameter(0, ShapeUtil::MakeShape(F32, {}), "in"); b->BitcastConvertType(param, S32); @@ -112,7 +112,7 @@ TEST_F(BitcastConvertTest, ConvertMapToS32) { } TEST_F(BitcastConvertTest, ConvertMapToF32) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto b = builder.CreateSubBuilder("convert"); auto param = b->Parameter(0, ShapeUtil::MakeShape(S32, {}), "in"); b->BitcastConvertType(param, F32); @@ -129,7 +129,7 @@ TEST_F(BitcastConvertTest, ConvertMapToF32) { // input -> convert -> reshape // the new convert should have the same element type as the old convert. TEST_F(BitcastConvertTest, ConvertReshape) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto input = builder.ConstantR1({0x42280000}); auto reshape = builder.Reshape(input, /*dimensions=*/{0}, /*new_sizes=*/{}); builder.BitcastConvertType(reshape, F32); diff --git a/tensorflow/compiler/xla/tests/map_test.cc b/tensorflow/compiler/xla/tests/map_test.cc index 0cd812fd1b..efe6cc6787 100644 --- a/tensorflow/compiler/xla/tests/map_test.cc +++ b/tensorflow/compiler/xla/tests/map_test.cc @@ -21,6 +21,8 @@ limitations under the License. #include "tensorflow/compiler/xla/client/global_data.h" #include "tensorflow/compiler/xla/client/lib/arithmetic.h" #include "tensorflow/compiler/xla/client/local_client.h" +#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h" +#include "tensorflow/compiler/xla/client/xla_client/xla_computation.h" #include "tensorflow/compiler/xla/literal_util.h" #include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/statusor.h" @@ -50,18 +52,18 @@ class MapTest : public ClientLibraryTestBase { // x {R0F32} ----> (add) // / // 1.0f ---------/ - Computation CreateAdderToOne() { - ComputationBuilder mapped_builder(client_, TestName()); + XlaComputation CreateAdderToOne() { + XlaBuilder mapped_builder(TestName()); auto x = mapped_builder.Parameter(0, ShapeUtil::MakeShape(F32, {}), "x"); auto one = mapped_builder.ConstantR0(1.0); - auto adder_to_one = mapped_builder.Add(x, one); + mapped_builder.Add(x, one); auto computation_status = mapped_builder.Build(); TF_CHECK_OK(computation_status.status()); return computation_status.ConsumeValueOrDie(); } - Computation CreateMax() { - ComputationBuilder b(client_, TestName()); + XlaComputation CreateMax() { + XlaBuilder b(TestName()); auto lhs = b.Parameter(0, ShapeUtil::MakeShape(F32, {}), "x"); auto rhs = b.Parameter(1, ShapeUtil::MakeShape(F32, {}), "y"); b.Max(lhs, rhs); @@ -73,8 +75,8 @@ class MapTest : public ClientLibraryTestBase { // Creates a computation that accepts an F32 and returns T(1) (ignoring the // argument). template - Computation CreateScalarOne() { - ComputationBuilder mapped_builder(client_, "scalar_one"); + XlaComputation CreateScalarOne() { + XlaBuilder mapped_builder("scalar_one"); (void)mapped_builder.Parameter(0, ShapeUtil::MakeShape(F32, {}), "x"); mapped_builder.ConstantR0(1); auto computation_status = mapped_builder.Build(); @@ -87,11 +89,11 @@ class MapTest : public ClientLibraryTestBase { // x {R0F32} ----> (mul) // / // 2.0f ---------/ - Computation CreateMulByTwo() { - ComputationBuilder mapped_builder(client_, TestName()); + XlaComputation CreateMulByTwo() { + XlaBuilder mapped_builder(TestName()); auto x = mapped_builder.Parameter(0, ShapeUtil::MakeShape(F32, {}), "x"); auto two = mapped_builder.ConstantR0(2.0); - auto mul_by_two = mapped_builder.Mul(x, two); + mapped_builder.Mul(x, two); auto computation_status = mapped_builder.Build(); TF_CHECK_OK(computation_status.status()); return computation_status.ConsumeValueOrDie(); @@ -105,12 +107,12 @@ class MapTest : public ClientLibraryTestBase { // x {R0F32} ----> (add) ----> (mul) // / // 1.0f ---------/ - Computation CreateAdderToOneTimesItself() { - ComputationBuilder mapped_builder(client_, TestName()); + XlaComputation CreateAdderToOneTimesItself() { + XlaBuilder mapped_builder(TestName()); auto x = mapped_builder.Parameter(0, ShapeUtil::MakeShape(F32, {}), "x"); auto one = mapped_builder.ConstantR0(1.0); auto adder_to_one = mapped_builder.Add(x, one); - auto result = mapped_builder.Mul(x, adder_to_one); + mapped_builder.Mul(x, adder_to_one); auto computation_status = mapped_builder.Build(); TF_CHECK_OK(computation_status.status()); return computation_status.ConsumeValueOrDie(); @@ -122,12 +124,13 @@ class MapTest : public ClientLibraryTestBase { // x {R0F32} -----------> (map) ----> (add) // / / // embedded_computation --/ n --/ - Computation CreateMapPlusN(const Computation& embedded_computation, float n) { - ComputationBuilder builder(client_, TestName()); + XlaComputation CreateMapPlusN(const XlaComputation& embedded_computation, + float n) { + XlaBuilder builder(TestName()); auto x = builder.Parameter(0, ShapeUtil::MakeShape(F32, {}), "x"); auto map = builder.Map({x}, embedded_computation, {}); auto constant_n = builder.ConstantR0(n); - auto add = builder.Add(map, constant_n); + builder.Add(map, constant_n); auto computation_status = builder.Build(); TF_CHECK_OK(computation_status.status()); return computation_status.ConsumeValueOrDie(); @@ -135,11 +138,11 @@ class MapTest : public ClientLibraryTestBase { // Creates a binary function with signature (F32, F32) -> Pred // defined by (x, y) -> x > y. - Computation CreateGt() { - ComputationBuilder b(client_, "Gt"); + XlaComputation CreateGt() { + XlaBuilder b("Gt"); auto x = b.Parameter(0, ShapeUtil::MakeShape(F32, {}), "x"); auto y = b.Parameter(1, ShapeUtil::MakeShape(F32, {}), "y"); - auto gt = b.Gt(x, y); + b.Gt(x, y); auto computation_status = b.Build(); TF_CHECK_OK(computation_status.status()); return computation_status.ConsumeValueOrDie(); @@ -152,13 +155,13 @@ class MapTest : public ClientLibraryTestBase { // y {R0F32} ----> (add) ---> (add) // / // z {R0F32} ---------------/ - Computation CreateTernaryAdder() { - ComputationBuilder mapped_builder(client_, "TernaryAdder"); + XlaComputation CreateTernaryAdder() { + XlaBuilder mapped_builder("TernaryAdder"); auto x = mapped_builder.Parameter(0, ShapeUtil::MakeShape(F32, {}), "x"); auto y = mapped_builder.Parameter(1, ShapeUtil::MakeShape(F32, {}), "y"); auto z = mapped_builder.Parameter(2, ShapeUtil::MakeShape(F32, {}), "z"); auto xy = mapped_builder.Add(x, y); - auto xyz = mapped_builder.Add(xy, z); + mapped_builder.Add(xy, z); auto computation_status = mapped_builder.Build(); TF_CHECK_OK(computation_status.status()); return computation_status.ConsumeValueOrDie(); @@ -167,13 +170,13 @@ class MapTest : public ClientLibraryTestBase { TEST_F(MapTest, MapEachElemPlusOneR0) { // Applies lambda (x) (+ x 1)) to an input scalar. - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); std::unique_ptr param0_literal = Literal::CreateR0(42.0); std::unique_ptr param0_data = client_->TransferToServer(*param0_literal).ConsumeValueOrDie(); auto param = builder.Parameter(0, param0_literal->shape(), "param0"); - auto map = builder.Map({param}, CreateAdderToOne(), {}); + builder.Map({param}, CreateAdderToOne(), {}); ComputeAndCompareR0(&builder, 43.0, {param0_data.get()}, ErrorSpec(0.01f)); @@ -181,13 +184,13 @@ TEST_F(MapTest, MapEachElemPlusOneR0) { XLA_TEST_F(MapTest, MapEachElemPlusOneR1S0) { // Maps (lambda (x) (+ x 1)) onto an input R1F32 vector of length 0. - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); std::unique_ptr param0_literal = Literal::CreateR1({}); std::unique_ptr param0_data = client_->TransferToServer(*param0_literal).ConsumeValueOrDie(); auto param = builder.Parameter(0, param0_literal->shape(), "param0"); - auto map = builder.Map({param}, CreateAdderToOne(), {0}); + builder.Map({param}, CreateAdderToOne(), {0}); ComputeAndCompareR1(&builder, {}, {param0_data.get()}, ErrorSpec(0.01f)); @@ -195,55 +198,55 @@ XLA_TEST_F(MapTest, MapEachElemPlusOneR1S0) { TEST_F(MapTest, MapEachElemPlusOneR1S4) { // Maps (lambda (x) (+ x 1)) onto an input R1F32 vector of length 4. - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); std::unique_ptr param0_literal = Literal::CreateR1({2.2f, 3.3f, 4.4f, 5.5f}); std::unique_ptr param0_data = client_->TransferToServer(*param0_literal).ConsumeValueOrDie(); auto param = builder.Parameter(0, param0_literal->shape(), "param0"); - auto map = builder.Map({param}, CreateAdderToOne(), {0}); + builder.Map({param}, CreateAdderToOne(), {0}); ComputeAndCompareR1(&builder, {3.2f, 4.3f, 5.4f, 6.5f}, {param0_data.get()}, ErrorSpec(0.01f)); } TEST_F(MapTest, MapEachF32ElementToS32Constant) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); std::unique_ptr param0_literal = Literal::CreateR1({2.2f, 3.3f, 4.4f, 5.5f}); std::unique_ptr param0_data = client_->TransferToServer(*param0_literal).ConsumeValueOrDie(); auto param = builder.Parameter(0, param0_literal->shape(), "param0"); - auto map = builder.Map({param}, CreateScalarOne(), {0}); + builder.Map({param}, CreateScalarOne(), {0}); ComputeAndCompareR1(&builder, {1, 1, 1, 1}, {param0_data.get()}); } TEST_F(MapTest, MapEachF32ElementToU32Constant) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); std::unique_ptr param0_literal = Literal::CreateR1({2.2f, 3.3f, 4.4f, 5.5f}); std::unique_ptr param0_data = client_->TransferToServer(*param0_literal).ConsumeValueOrDie(); auto param = builder.Parameter(0, param0_literal->shape(), "param0"); - auto map = builder.Map({param}, CreateScalarOne(), {0}); + builder.Map({param}, CreateScalarOne(), {0}); ComputeAndCompareR1(&builder, {1, 1, 1, 1}, {param0_data.get()}); } TEST_F(MapTest, MapEachElemLongerChainR1) { // Maps (lambda (x) (* (+ x 1) x)) onto an input R1F32 vector. - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); std::unique_ptr param0_literal = Literal::CreateR1({2.6f, -5.1f, 0.1f, 0.2f, 999.0f, 255.5f}); std::unique_ptr param0_data = client_->TransferToServer(*param0_literal).ConsumeValueOrDie(); auto param = builder.Parameter(0, param0_literal->shape(), "param0"); - auto map = builder.Map({param}, CreateAdderToOneTimesItself(), {0}); + builder.Map({param}, CreateAdderToOneTimesItself(), {0}); ComputeAndCompareR1( &builder, {9.36f, 20.91f, 0.11f, 0.24f, 999000.0f, 65535.75f}, @@ -253,14 +256,14 @@ TEST_F(MapTest, MapEachElemLongerChainR1) { XLA_TEST_F(MapTest, MapMultipleMapsR1S0) { // Maps (lambda (x) (+ x 1)) onto an input R1F32 vector of length 0, and then // maps (lambda (x) (* x 2)) on the result. - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); std::unique_ptr param0_literal = Literal::CreateR1({}); std::unique_ptr param0_data = client_->TransferToServer(*param0_literal).ConsumeValueOrDie(); auto param = builder.Parameter(0, param0_literal->shape(), "param0"); auto map1 = builder.Map({param}, CreateAdderToOne(), {0}); - auto map2 = builder.Map({map1}, CreateMulByTwo(), {0}); + builder.Map({map1}, CreateMulByTwo(), {0}); ComputeAndCompareR1(&builder, {}, {param0_data.get()}, ErrorSpec(0.01f)); @@ -269,7 +272,7 @@ XLA_TEST_F(MapTest, MapMultipleMapsR1S0) { TEST_F(MapTest, MapMultipleMapsR1S4) { // Maps (lambda (x) (+ x 1)) onto an input R1F32 vector of length 4, and then // maps (lambda (x) (* x 2)) on the result. - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); std::unique_ptr param0_literal = Literal::CreateR1({2.2f, 3.3f, 4.4f, 5.5f}); std::unique_ptr param0_data = @@ -277,7 +280,7 @@ TEST_F(MapTest, MapMultipleMapsR1S4) { auto param = builder.Parameter(0, param0_literal->shape(), "param0"); auto map1 = builder.Map({param}, CreateAdderToOne(), {0}); - auto map2 = builder.Map({map1}, CreateMulByTwo(), {0}); + builder.Map({map1}, CreateMulByTwo(), {0}); ComputeAndCompareR1(&builder, {6.4f, 8.6f, 10.8f, 13.0f}, {param0_data.get()}, ErrorSpec(0.01f)); @@ -285,14 +288,14 @@ TEST_F(MapTest, MapMultipleMapsR1S4) { TEST_F(MapTest, MapEachElemPlusOneR2) { // Maps (lambda (x) (+ x 1)) onto an input R2F32 vector. - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); std::unique_ptr param0_literal = Literal::CreateR2( {{13.25f, 14.0f}, {-7.1f, -7.2f}, {-8.8f, 8.8f}}); std::unique_ptr param0_data = client_->TransferToServer(*param0_literal).ConsumeValueOrDie(); auto param = builder.Parameter(0, param0_literal->shape(), "param0"); - auto map = builder.Map({param}, CreateAdderToOne(), {0, 1}); + builder.Map({param}, CreateAdderToOne(), {0, 1}); Array2D expected_array( {{14.25f, 15.0f}, {-6.1f, -6.2f}, {-7.8f, 9.8f}}); @@ -317,18 +320,18 @@ XLA_TEST_F(MapTest, ComplexNestedMaps) { auto embed2 = CreateMapPlusN(embed1, 2.0); auto embed3 = CreateMapPlusN(embed1, 4.0); - ComputationBuilder embed4_builder(client_, "embed4"); + XlaBuilder embed4_builder("embed4"); auto embed4_param = embed4_builder.Parameter(0, scalar_shape, "x"); auto embed4_map_lhs = embed4_builder.Map({embed4_param}, embed2, {}); auto embed4_map_rhs = embed4_builder.Map({embed4_param}, embed3, {}); - auto embed4_add = embed4_builder.Add(embed4_map_lhs, embed4_map_rhs); + embed4_builder.Add(embed4_map_lhs, embed4_map_rhs); auto embed4_status = embed4_builder.Build(); ASSERT_IS_OK(embed4_status.status()); auto embed4 = embed4_status.ConsumeValueOrDie(); auto embed5 = CreateMapPlusN(embed2, 6.0); - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto constant_42 = builder.ConstantR0(42.0); auto constant_7 = builder.ConstantR0(7.0); auto map_42 = builder.Map({constant_42}, embed5, {}); @@ -359,7 +362,8 @@ TEST_F(MapTest, VersionedEmbeddedComputation) { // Add another Add(1) operation to the existing embedded computation. This // requires using the stub interface because the ComputationBuilder does not - // allow modification to the Computation objects after they have been built. + // allow modification to the XlaComputation objects after they have been + // built. BinaryOpRequest request; request.set_binop(BINOP_ADD); *request.mutable_lhs() = adder_to_one; @@ -381,7 +385,7 @@ TEST_F(MapTest, VersionedEmbeddedComputation) { TEST_F(MapTest, MapBinaryAdder) { // Maps (lambda (x y) (+ x y)) onto two R1F32 vectors. - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); std::unique_ptr param0_literal = Literal::CreateR1({2.2f, 3.3f, 4.4f, 5.5f}); std::unique_ptr param0_data = @@ -393,8 +397,7 @@ TEST_F(MapTest, MapBinaryAdder) { auto param0 = builder.Parameter(0, param0_literal->shape(), "param0"); auto param1 = builder.Parameter(1, param1_literal->shape(), "param1"); - auto map = builder.Map({param0, param1}, - CreateScalarAddComputation(F32, &builder), {0}); + builder.Map({param0, param1}, CreateScalarAddComputation(F32, &builder), {0}); ComputeAndCompareR1(&builder, {7.3f, 7.7, 4.3f, 0}, {param0_data.get(), param1_data.get()}, @@ -404,7 +407,7 @@ TEST_F(MapTest, MapBinaryAdder) { // Adds two rank-2 arrays with different layouts. This test exercises a path // for Map that used to fail in shape inference (b/28989438). XLA_TEST_F(MapTest, AddWithMixedLayouts) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); std::unique_ptr param0_literal = Literal::CreateR2WithLayout( {{1, 2}, {3, 4}}, LayoutUtil::MakeLayout({1, 0})); std::unique_ptr param0_data = @@ -417,8 +420,8 @@ XLA_TEST_F(MapTest, AddWithMixedLayouts) { auto param0 = builder.Parameter(0, param0_literal->shape(), "param0"); auto param1 = builder.Parameter(1, param1_literal->shape(), "param1"); - auto map = builder.Map({param0, param1}, - CreateScalarAddComputation(S32, &builder), {0, 1}); + builder.Map({param0, param1}, CreateScalarAddComputation(S32, &builder), + {0, 1}); Array2D expected(2, 2); expected(0, 0) = 11; @@ -430,7 +433,7 @@ XLA_TEST_F(MapTest, AddWithMixedLayouts) { } XLA_TEST_F(MapTest, AddR3_3x0x2) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); std::unique_ptr param0_literal = Literal::CreateR3FromArray3D(Array3D(3, 0, 2)); std::unique_ptr param0_data = @@ -443,8 +446,8 @@ XLA_TEST_F(MapTest, AddR3_3x0x2) { auto param0 = builder.Parameter(0, param0_literal->shape(), "param0"); auto param1 = builder.Parameter(1, param1_literal->shape(), "param1"); - auto map = builder.Map({param0, param1}, - CreateScalarAddComputation(S32, &builder), {0, 1, 2}); + builder.Map({param0, param1}, CreateScalarAddComputation(S32, &builder), + {0, 1, 2}); ComputeAndCompareR3(&builder, Array3D(3, 0, 2), {param0_data.get(), param1_data.get()}); @@ -452,7 +455,7 @@ XLA_TEST_F(MapTest, AddR3_3x0x2) { TEST_F(MapTest, MapTernaryAdder) { // Maps (lambda (x y z) (+ x y z)) onto three R1F32 vectors. - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); std::unique_ptr param0_literal = Literal::CreateR1({2.2f, 3.3f, 4.4f, 5.5f}); std::unique_ptr param0_data = @@ -469,7 +472,7 @@ TEST_F(MapTest, MapTernaryAdder) { auto param0 = builder.Parameter(0, param0_literal->shape(), "param0"); auto param1 = builder.Parameter(1, param1_literal->shape(), "param1"); auto param2 = builder.Parameter(2, param2_literal->shape(), "param2"); - auto map = builder.Map({param0, param1, param2}, CreateTernaryAdder(), {0}); + builder.Map({param0, param1, param2}, CreateTernaryAdder(), {0}); ComputeAndCompareR1( &builder, {-2.7f, -92.3f, -895.7f, -400.0f}, @@ -479,24 +482,24 @@ TEST_F(MapTest, MapTernaryAdder) { TEST_F(MapTest, MapGt) { // Maps (x,y) -> x > y onto two R1F32 vectors. - ComputationBuilder b(client_, TestName()); + XlaBuilder b(TestName()); auto gt = CreateGt(); b.Map({b.ConstantR1({1, 20}), b.ConstantR1({10, 2})}, gt, {0}); ComputeAndCompareR1(&b, {false, true}, {}); } TEST_F(MapTest, NestedBinaryMap) { - Computation max_with_square; + XlaComputation max_with_square; { // max_with_square(x) = do max(x, x^2) via a map. - ComputationBuilder b(client_, "max_with_square"); + XlaBuilder b("max_with_square"); auto x = b.Parameter(0, ShapeUtil::MakeShape(F32, {}), "x"); b.Map({x, b.Mul(x, x)}, CreateMax(), {}); auto computation_status = b.Build(); ASSERT_IS_OK(computation_status.status()); max_with_square = computation_status.ConsumeValueOrDie(); } - ComputationBuilder b(client_, TestName()); + XlaBuilder b(TestName()); auto input = b.ConstantR1({0.1f, 0.5f, -0.5f, 1.0f, 2.0f}); b.Map({input}, max_with_square, {0}); ComputeAndCompareR1(&b, {0.1f, 0.5f, 0.25f, 1.0f, 4.0f}, {}); @@ -505,13 +508,13 @@ TEST_F(MapTest, NestedBinaryMap) { TEST_F(MapTest, MapOperantionWithBuildError) { // Maps (lambda (x y) (+ x y)) onto two R1F32 vectors but uses an unsupported // type combination (F32 + U16) to test that the error is reported to the - // outermost ComputationBuilder. - ComputationBuilder builder(client_, TestName()); + // outermost XlaBuilder. + XlaBuilder builder(TestName()); auto sub_builder = builder.CreateSubBuilder("ErrorAdd"); auto x = sub_builder->Parameter(0, ShapeUtil::MakeShape(F32, {}), "x"); auto y = sub_builder->Parameter(1, ShapeUtil::MakeShape(U16, {}), "y"); - auto adder = sub_builder->Add(x, y); + sub_builder->Add(x, y); auto error_add = sub_builder->BuildAndNoteError(); std::unique_ptr param0_literal = @@ -525,9 +528,9 @@ TEST_F(MapTest, MapOperantionWithBuildError) { auto param0 = builder.Parameter(0, param0_literal->shape(), "param0"); auto param1 = builder.Parameter(1, param1_literal->shape(), "param1"); - auto map = builder.Map({param0, param1}, error_add, {0}); + builder.Map({param0, param1}, error_add, {0}); - StatusOr computation_status = builder.Build(); + StatusOr computation_status = builder.Build(); ASSERT_TRUE(!computation_status.ok()); EXPECT_THAT( computation_status.status().ToString(), @@ -545,7 +548,7 @@ using MapTestWithFullOpt = ClientLibraryTestBase; // to have issues with such patterns and maybe invalidate the pointer to entry // computation. TEST_F(MapTestWithFullOpt, MapScalarPower) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto sub_builder = builder.CreateSubBuilder("power"); auto x = sub_builder->Parameter(0, ShapeUtil::MakeShape(F32, {}), "x"); @@ -572,7 +575,7 @@ TEST_F(MapTestWithFullOpt, MapScalarPower) { // Regression test for b/35786417, where the inliner would not notice the change // of parameter order inside the map. TEST_F(MapTestWithFullOpt, MapSubtractOppositeOrder) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto sub_builder = builder.CreateSubBuilder("power"); auto x = sub_builder->Parameter(0, ShapeUtil::MakeShape(F32, {}), "x"); @@ -598,7 +601,7 @@ TEST_F(MapTestWithFullOpt, MapSubtractOppositeOrder) { // Regression test for b/35786417, where the inliner would CHECK-fail due to the // mul inside the map having more parameters than the map does. TEST_F(MapTestWithFullOpt, MapSquare) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto sub_builder = builder.CreateSubBuilder("power"); auto x = sub_builder->Parameter(0, ShapeUtil::MakeShape(F32, {}), "x"); diff --git a/tensorflow/compiler/xla/tests/reduce_precision_test.cc b/tensorflow/compiler/xla/tests/reduce_precision_test.cc index dc7ce3253c..b311785449 100644 --- a/tensorflow/compiler/xla/tests/reduce_precision_test.cc +++ b/tensorflow/compiler/xla/tests/reduce_precision_test.cc @@ -20,9 +20,9 @@ limitations under the License. #include #include "tensorflow/compiler/xla/array2d.h" -#include "tensorflow/compiler/xla/client/computation_builder.h" #include "tensorflow/compiler/xla/client/global_data.h" #include "tensorflow/compiler/xla/client/local_client.h" +#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h" #include "tensorflow/compiler/xla/layout_util.h" #include "tensorflow/compiler/xla/literal_util.h" #include "tensorflow/compiler/xla/service/reduce_precision_insertion.h" @@ -228,15 +228,14 @@ XLA_TEST_P(ReducePrecisionAccuracyTest, ReducePrecisionF32) { // This is required for proper handling of NaN values. SetFastMathDisabled(true); - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); std::unique_ptr a_literal = Literal::CreateR1({input_values}); std::unique_ptr a_data = client_->TransferToServer(*a_literal).ConsumeValueOrDie(); auto a = builder.Parameter(0, a_literal->shape(), "a"); - auto reduce_precision = - builder.ReducePrecision(a, exponent_bits, mantissa_bits); + builder.ReducePrecision(a, exponent_bits, mantissa_bits); ComputeAndCompareR1(&builder, expected_values, {a_data.get()}); } @@ -252,7 +251,7 @@ class ReducePrecisionInsertionTest : public ClientLibraryTestBase {}; // The interpreter has no fusion pass, so skip this test. XLA_TEST_F(ReducePrecisionInsertionTest, DISABLED_ON_INTERPRETER(ReducePrecisionBeforeFusion)) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); std::unique_ptr a_literal = Literal::CreateR1({1.00001}); std::unique_ptr a_data = @@ -265,7 +264,7 @@ XLA_TEST_F(ReducePrecisionInsertionTest, // Near 1.0, Log(x) approximates x - 1; this lets us confirm that the // reduce-precision operation showed up in the correct place in the // graph. - auto log = builder.Log(abs); + builder.Log(abs); // Insert precision-reduction after the Abs(x) operation, rounding that // result to exactly 1.0f. @@ -281,7 +280,7 @@ XLA_TEST_F(ReducePrecisionInsertionTest, // The interpreter has no fusion pass, so skip this test. XLA_TEST_F(ReducePrecisionInsertionTest, DISABLED_ON_INTERPRETER(ReducePrecisionSkippedAfterFusion)) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); std::unique_ptr a_literal = Literal::CreateR1({1.00001}); std::unique_ptr a_data = @@ -290,7 +289,7 @@ XLA_TEST_F(ReducePrecisionInsertionTest, // These two operations should be fused by any reasonable backend. auto abs = builder.Abs(a); - auto neg = builder.Neg(abs); + builder.Neg(abs); // Add a pass after operation fusion, suffixing kAbs operations. This // should not see into the fusion nodes and thus should not affect the @@ -307,7 +306,7 @@ XLA_TEST_F(ReducePrecisionInsertionTest, // The interpreter has no fusion pass, so skip this test. XLA_TEST_F(ReducePrecisionInsertionTest, DISABLED_ON_INTERPRETER(ReducePrecisionAddedAfterFusion)) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); std::unique_ptr a_literal = Literal::CreateR1({1.00001}); std::unique_ptr a_data = @@ -316,7 +315,7 @@ XLA_TEST_F(ReducePrecisionInsertionTest, // These two operations should be fused by any reasonable backend. auto abs = builder.Abs(a); - auto neg = builder.Neg(abs); + builder.Neg(abs); // Add a pass after operation fusion, suffixing kFusion operations. auto reduce_precision_pass = execution_options_.mutable_debug_options() @@ -331,7 +330,7 @@ XLA_TEST_F(ReducePrecisionInsertionTest, // The interpreter has no fusion pass, so skip this test. XLA_TEST_F(ReducePrecisionInsertionTest, DISABLED_ON_INTERPRETER(ReducePrecisionSkippedFusionContains)) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); std::unique_ptr a_literal = Literal::CreateR1({1.00001}); std::unique_ptr a_data = @@ -340,7 +339,7 @@ XLA_TEST_F(ReducePrecisionInsertionTest, // These two operations should be fused by any reasonable backend. auto abs = builder.Abs(a); - auto neg = builder.Neg(abs); + builder.Neg(abs); // Add a pass suffixing fusion nodes containing kCos operations. This // should have no effect. @@ -356,7 +355,7 @@ XLA_TEST_F(ReducePrecisionInsertionTest, // The interpreter has no fusion pass, so skip this test. XLA_TEST_F(ReducePrecisionInsertionTest, DISABLED_ON_INTERPRETER(ReducePrecisionAddedFusionContains)) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); std::unique_ptr a_literal = Literal::CreateR1({1.00001}); std::unique_ptr a_data = @@ -365,7 +364,7 @@ XLA_TEST_F(ReducePrecisionInsertionTest, // These two operations should be fused by any reasonable backend. auto abs = builder.Abs(a); - auto neg = builder.Neg(abs); + builder.Neg(abs); // Add a pass suffixing fusion nodes containing kAbs operations. This // should see the kAbs operation within the above fusion node. diff --git a/tensorflow/compiler/xla/tests/reverse_test.cc b/tensorflow/compiler/xla/tests/reverse_test.cc index 8fc841f140..6959c95502 100644 --- a/tensorflow/compiler/xla/tests/reverse_test.cc +++ b/tensorflow/compiler/xla/tests/reverse_test.cc @@ -17,8 +17,8 @@ limitations under the License. #include "tensorflow/compiler/xla/array2d.h" #include "tensorflow/compiler/xla/array4d.h" -#include "tensorflow/compiler/xla/client/computation_builder.h" #include "tensorflow/compiler/xla/client/local_client.h" +#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h" #include "tensorflow/compiler/xla/tests/client_library_test_base.h" #include "tensorflow/compiler/xla/tests/literal_test_util.h" #include "tensorflow/compiler/xla/tests/test_macros.h" @@ -85,7 +85,7 @@ TEST_P(FloatReverseTest, Reverses) { auto r1_literal = Literal::CreateR1(input_vector); auto input_literal = r1_literal->Reshape(spec.input_dims).ConsumeValueOrDie(); - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto a = AddParam(*input_literal, &builder); builder.Rev(a, spec.reversal); -- GitLab From 0b80e3dca1bf051f973212d45315c44c9c6a125d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 14:16:36 -0700 Subject: [PATCH 109/791] Add missing import for RNNClassifier PiperOrigin-RevId: 192344760 --- tensorflow/contrib/estimator/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/contrib/estimator/__init__.py b/tensorflow/contrib/estimator/__init__.py index 9a87fa915d..be20d1b777 100644 --- a/tensorflow/contrib/estimator/__init__.py +++ b/tensorflow/contrib/estimator/__init__.py @@ -28,6 +28,7 @@ from tensorflow.contrib.estimator.python.estimator.linear import * from tensorflow.contrib.estimator.python.estimator.logit_fns import * from tensorflow.contrib.estimator.python.estimator.multi_head import * from tensorflow.contrib.estimator.python.estimator.replicate_model_fn import * +from tensorflow.contrib.estimator.python.estimator.rnn import * from tensorflow.python.util.all_util import remove_undocumented # pylint: enable=unused-import,line-too-long,wildcard-import -- GitLab From b0af2c890049a37b86f9724074570d80bb0dc14d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 14:22:33 -0700 Subject: [PATCH 110/791] Bug fix for statistical_testing: - Max/Min computations should be done over the sample dimension. - Change dominate check to be greater_equal instead of greater (for matching dimensions). PiperOrigin-RevId: 192345809 --- .../kernel_tests/statistical_testing_test.py | 22 +++++----- .../python/ops/statistical_testing.py | 43 +++++++++++-------- 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py b/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py index 0400c80c29..c4fb669ebb 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py @@ -21,7 +21,6 @@ from __future__ import print_function import numpy as np from tensorflow.contrib.distributions.python.ops import statistical_testing as st -from tensorflow.python.framework import errors from tensorflow.python.platform import test @@ -129,13 +128,13 @@ class StatisticalTestingTest(test.TestCase): # Test that the test assertion confirms that the mean of the # standard uniform distribution is not 0.4. - with self.assertRaises(errors.InvalidArgumentError): + with self.assertRaisesOpError("Mean confidence interval too high"): sess.run(st.assert_true_mean_equal_by_dkwm( samples, 0., 1., 0.4, false_fail_rate=1e-6)) # Test that the test assertion confirms that the mean of the # standard uniform distribution is not 0.6. - with self.assertRaises(errors.InvalidArgumentError): + with self.assertRaisesOpError("Mean confidence interval too low"): sess.run(st.assert_true_mean_equal_by_dkwm( samples, 0., 1., 0.6, false_fail_rate=1e-6)) @@ -172,7 +171,7 @@ class StatisticalTestingTest(test.TestCase): # Test that the test assertion confirms that the mean of the # standard uniform distribution is different from the mean of beta(2, 1). beta_high_samples = rng.beta(2, 1, size=num_samples).astype(np.float32) - with self.assertRaises(errors.InvalidArgumentError): + with self.assertRaisesOpError("samples1 has a smaller mean"): sess.run(st.assert_true_mean_equal_by_dkwm_two_sample( samples1, 0., 1., beta_high_samples, 0., 1., @@ -190,7 +189,7 @@ class StatisticalTestingTest(test.TestCase): # Test that the test assertion confirms that the mean of the # standard uniform distribution is different from the mean of beta(1, 2). beta_low_samples = rng.beta(1, 2, size=num_samples).astype(np.float32) - with self.assertRaises(errors.InvalidArgumentError): + with self.assertRaisesOpError("samples2 has a smaller mean"): sess.run(st.assert_true_mean_equal_by_dkwm_two_sample( samples1, 0., 1., beta_low_samples, 0., 1., @@ -198,21 +197,22 @@ class StatisticalTestingTest(test.TestCase): def test_dkwm_argument_validity_checking(self): rng = np.random.RandomState(seed=0) - samples = rng.uniform(size=5000).astype(np.float32) + samples = rng.uniform( + low=[0., 1.], high=[1., 2.], size=(2500, 1, 2)).astype(np.float32) # Test that the test library complains if the given samples fall # outside the purported bounds. with self.test_session() as sess: - with self.assertRaises(errors.InvalidArgumentError): + with self.assertRaisesOpError("maximum value exceeds expectations"): sess.run(st.true_mean_confidence_interval_by_dkwm( - samples, 0., 0.5, error_rate=0.5)) - with self.assertRaises(errors.InvalidArgumentError): + samples, [[0., 1.]], [[0.5, 1.5]], error_rate=0.5)) + with self.assertRaisesOpError("minimum value falls below expectations"): sess.run(st.true_mean_confidence_interval_by_dkwm( - samples, 0.5, 1., error_rate=0.5)) + samples, [[0.5, 1.5]], [[1., 2.]], error_rate=0.5)) # But doesn't complain if they don't. op = st.true_mean_confidence_interval_by_dkwm( - samples, 0., 1., error_rate=0.5) + samples, [[0., 1.]], [[1., 2.]], error_rate=0.5) _ = sess.run(op) diff --git a/tensorflow/contrib/distributions/python/ops/statistical_testing.py b/tensorflow/contrib/distributions/python/ops/statistical_testing.py index 5c52015e5f..9b9fff0afa 100644 --- a/tensorflow/contrib/distributions/python/ops/statistical_testing.py +++ b/tensorflow/contrib/distributions/python/ops/statistical_testing.py @@ -234,7 +234,7 @@ def _maximum_mean(samples, envelope, high, name=None): envelope = ops.convert_to_tensor(envelope, name="envelope") high = ops.convert_to_tensor(high, name="high") - xmax = math_ops.reduce_max(samples, axis=[-1]) + xmax = math_ops.reduce_max(samples, axis=[0]) msg = "Given sample maximum value exceeds expectations" check_op = check_ops.assert_less_equal(xmax, high, message=msg) with ops.control_dependencies([check_op]): @@ -279,7 +279,7 @@ def _minimum_mean(samples, envelope, low, name=None): envelope = ops.convert_to_tensor(envelope, name="envelope") low = ops.convert_to_tensor(low, name="low") - xmin = math_ops.reduce_min(samples, axis=[-1]) + xmin = math_ops.reduce_min(samples, axis=[0]) msg = "Given sample minimum value falls below expectations" check_op = check_ops.assert_greater_equal(xmin, low, message=msg) with ops.control_dependencies([check_op]): @@ -319,8 +319,8 @@ def _dkwm_cdf_envelope(n, error_rate, name=None): return math_ops.sqrt(-gen_math_ops.log(error_rate / 2.) / (2. * n)) -def _check_shape_dominates(tensor, tensors): - """Check that broadcasting `tensor` against `tensors` does not expand it. +def _check_shape_dominates(samples, parameters): + """Check that broadcasting `samples` against `parameters` does not expand it. Why? Because I want to be very sure that the samples tensor is not accidentally enlarged by broadcasting against tensors that are @@ -328,24 +328,27 @@ def _check_shape_dominates(tensor, tensors): sample counts end up inflated. Args: - tensor: A Tensor whose shape is to be protected against broadcasting. - tensors: A list of Tensors to check + samples: A Tensor whose shape is to be protected against broadcasting. + parameters: A list of Tensors who are parameters for the statistical test. Returns: - tensor: `tf.identity(tensor)` with control dependencies attached; - be sure to use that downstream. + samples: Return original `samples` with control dependencies attached + to ensure no broadcasting. """ def check(t): - target = array_ops.shape(tensor)[1:] - result = array_ops.broadcast_dynamic_shape(target, array_ops.shape(t)) + samples_batch_shape = array_ops.shape(samples)[1:] + broadcasted_batch_shape = array_ops.broadcast_dynamic_shape( + samples_batch_shape, array_ops.shape(t)) # This rank check ensures that I don't get a wrong answer from the # _shapes_ broadcasting against each other. - gt = check_ops.assert_greater(array_ops.rank(target), array_ops.rank(t)) - eq = check_ops.assert_equal(target, result) - return gt, eq - checks = list(itertools.chain(*[check(t) for t in tensors])) + samples_batch_ndims = array_ops.size(samples_batch_shape) + ge = check_ops.assert_greater_equal( + samples_batch_ndims, array_ops.rank(t)) + eq = check_ops.assert_equal(samples_batch_shape, broadcasted_batch_shape) + return ge, eq + checks = list(itertools.chain(*[check(t) for t in parameters])) with ops.control_dependencies(checks): - return array_ops.identity(array_ops.identity(tensor)) + return array_ops.identity(samples) def true_mean_confidence_interval_by_dkwm( @@ -684,9 +687,13 @@ def assert_true_mean_equal_by_dkwm_two_sample( # I want to assert # not (max_mean_1 < min_mean_2 or min_mean_1 > max_mean_2), # but I think I only have and-combination of asserts, so use DeMorgan. - clause1_op = check_ops.assert_greater_equal(max_mean_1, min_mean_2) - with ops.control_dependencies([clause1_op]): - return check_ops.assert_less_equal(min_mean_1, max_mean_2) + check_confidence_intervals_can_intersect = check_ops.assert_greater_equal( + max_mean_1, min_mean_2, message="Confidence intervals do not " + "intersect: samples1 has a smaller mean than samples2") + with ops.control_dependencies([check_confidence_intervals_can_intersect]): + return check_ops.assert_less_equal( + min_mean_1, max_mean_2, message="Confidence intervals do not " + "intersect: samples2 has a smaller mean than samples1") def min_discrepancy_of_true_means_detectable_by_dkwm_two_sample( -- GitLab From 706d8d34c4db4d8568e195d2cfdd54d812ff0b12 Mon Sep 17 00:00:00 2001 From: Yu-Cheng Ling Date: Tue, 10 Apr 2018 14:24:51 -0700 Subject: [PATCH 111/791] ParseOpData returns kTfLiteError when error happens. PiperOrigin-RevId: 192346224 --- tensorflow/contrib/lite/model.cc | 86 ++++++++++++++++---------------- 1 file changed, 44 insertions(+), 42 deletions(-) diff --git a/tensorflow/contrib/lite/model.cc b/tensorflow/contrib/lite/model.cc index 13e5532909..87af953061 100644 --- a/tensorflow/contrib/lite/model.cc +++ b/tensorflow/contrib/lite/model.cc @@ -261,13 +261,11 @@ T* MallocPOD() { // Parse the appropriate data out of the op. // // This handles builtin data explicitly as there are flatbuffer schemas. -// -// Returns memory that must be feed. -// -// TODO(nupurgarg): Pass in void ** and return TfLiteStatus to ensure program -// crashes if error reporter is called. -void* ParseOpData(const Operator* op, BuiltinOperator op_type, - ErrorReporter* error_reporter) { +// If it returns kTfLiteOk, it passes the data out with `builtin_data`, which +// need to be released by calling `free`.` +// If it returns kTfLiteError, `builtin_data` will be `nullptr`. +TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, + ErrorReporter* error_reporter, void** builtin_data) { auto parse_padding = [](Padding padding) { switch (padding) { case Padding_SAME: @@ -316,7 +314,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, } }; - void* builtin_data = nullptr; + *builtin_data = nullptr; switch (op_type) { case BuiltinOperator_CALL: // TODO(aselle): Implement call in BuiltinOptions, but nullptrs are @@ -333,7 +331,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, params->activation = parse_activation(conv_params->fused_activation_function()); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_TANH: @@ -358,10 +356,11 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, ConvertTensorType(schema_params->out_data_type(), ¶ms->out_data_type, error_reporter); if (in_status != kTfLiteOk || out_status != kTfLiteOk) { - break; + free(params); + return kTfLiteError; } } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_LSH_PROJECTION: { @@ -370,7 +369,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, if (auto* lshParams = op->builtin_options_as_LSHProjectionOptions()) { params->type = parseLSHProjectionType(lshParams->type()); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_AVERAGE_POOL_2D: @@ -386,7 +385,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, params->activation = parse_activation(pool_params->fused_activation_function()); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_DEPTHWISE_CONV_2D: { @@ -400,7 +399,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, params->activation = parse_activation(conv_params->fused_activation_function()); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_SVDF: { @@ -410,7 +409,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, params->activation = parse_activation(svdf_params->fused_activation_function()); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_BIDIRECTIONAL_SEQUENCE_RNN: @@ -422,7 +421,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, parse_activation(sequence_rnn_params->fused_activation_function()); params->time_major = sequence_rnn_params->time_major(); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_RNN: { @@ -431,7 +430,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, params->activation = parse_activation(rnn_params->fused_activation_function()); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_EMBEDDING_LOOKUP: @@ -444,7 +443,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, op->builtin_options_as_EmbeddingLookupSparseOptions()) { params->combiner = parseCombinerType(embedding_params->combiner()); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_FULLY_CONNECTED: { @@ -455,7 +454,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, params->activation = parse_activation( fully_connected_params->fused_activation_function()); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_HASHTABLE_LOOKUP: @@ -466,7 +465,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, if (auto* softmax_params = op->builtin_options_as_SoftmaxOptions()) { params->beta = softmax_params->beta(); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_CONCATENATION: { @@ -478,7 +477,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, parse_activation(concatenation_params->fused_activation_function()); params->axis = concatenation_params->axis(); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_MUL: { @@ -487,7 +486,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, params->activation = parse_activation(schema_params->fused_activation_function()); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_ADD: { @@ -496,7 +495,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, params->activation = parse_activation(schema_params->fused_activation_function()); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_DIV: { @@ -505,7 +504,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, params->activation = parse_activation(schema_params->fused_activation_function()); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_SUB: { @@ -514,7 +513,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, params->activation = parse_activation(schema_params->fused_activation_function()); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_L2_NORMALIZATION: { @@ -523,7 +522,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, params->activation = parse_activation(schema_params->fused_activation_function()); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_LOCAL_RESPONSE_NORMALIZATION: { @@ -535,7 +534,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, params->alpha = schema_params->alpha(); params->beta = schema_params->beta(); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_BIDIRECTIONAL_SEQUENCE_LSTM: @@ -548,7 +547,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, params->cell_clip = lstm_params->cell_clip(); params->proj_clip = lstm_params->proj_clip(); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_RESIZE_BILINEAR: { @@ -557,7 +556,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, op->builtin_options_as_ResizeBilinearOptions()) { params->align_corners = schema_params->align_corners(); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_PAD: { @@ -571,7 +570,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, params->shape, error_reporter); params->num_dimensions = new_shape->Length(); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_SKIP_GRAM: { @@ -581,7 +580,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, params->max_skip_size = skip_gram_params->max_skip_size(); params->include_all_ngrams = skip_gram_params->include_all_ngrams(); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_SPACE_TO_DEPTH: { @@ -589,7 +588,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, if (auto* schema_params = op->builtin_options_as_SpaceToDepthOptions()) { params->block_size = schema_params->block_size(); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_GATHER: { @@ -599,7 +598,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, params->axis = gather_params->axis(); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_SPACE_TO_BATCH_ND: { @@ -616,7 +615,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, if (auto* schema_params = op->builtin_options_as_MeanOptions()) { params->keep_dims = schema_params->keep_dims(); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_SPLIT: { @@ -624,7 +623,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, if (auto* schema_params = op->builtin_options_as_SplitOptions()) { params->num_splits = schema_params->num_splits(); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_SQUEEZE: { @@ -635,7 +634,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, params->squeeze_dims, error_reporter); params->num_squeeze_dims = squeeze_dims->Length(); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_STRIDED_SLICE: { @@ -647,7 +646,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, params->new_axis_mask = schema_params->new_axis_mask(); params->shrink_axis_mask = schema_params->shrink_axis_mask(); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_MAXIMUM: @@ -660,16 +659,16 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type, ConvertTensorType(schema_params->output_type(), ¶ms->output_type, error_reporter); } - builtin_data = reinterpret_cast(params); + *builtin_data = reinterpret_cast(params); break; } case BuiltinOperator_DELEGATE: { // TODO(ycling): Revisit when supporting saving delegated models. error_reporter->Report("DELEGATE op shouldn't exist in model."); - break; + return kTfLiteError; } } - return builtin_data; + return kTfLiteOk; } } // namespace @@ -709,10 +708,13 @@ TfLiteStatus InterpreterBuilder::ParseNodes( reinterpret_cast(op->custom_options()->data()), op->custom_options()->size(), nullptr, reg); } else { + void* builtin_data = nullptr; + TF_LITE_ENSURE_STATUS( + ParseOpData(op, op_type, error_reporter_, &builtin_data)); interpreter->AddNodeWithParameters( FlatBufferIntArrayToVector(op->inputs()), - FlatBufferIntArrayToVector(op->outputs()), nullptr, 0, - ParseOpData(op, op_type, error_reporter_), reg); + FlatBufferIntArrayToVector(op->outputs()), nullptr, 0, builtin_data, + reg); } } -- GitLab From 02afb3d56e9270a9808693741b08c4fba997c3a2 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 14:51:54 -0700 Subject: [PATCH 112/791] Run EvaluateNodes for ModelPrunerTest_StopGradientPruning. Also updated the test fixture to inherit from GrapplerTest. PiperOrigin-RevId: 192350828 --- tensorflow/core/grappler/optimizers/BUILD | 2 ++ .../core/grappler/optimizers/model_pruner_test.cc | 11 ++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD index e4bc030885..a4545bb8f8 100644 --- a/tensorflow/core/grappler/optimizers/BUILD +++ b/tensorflow/core/grappler/optimizers/BUILD @@ -357,9 +357,11 @@ tf_cuda_cc_test( "//tensorflow/core:protos_all_cc", "//tensorflow/core:test", "//tensorflow/core:test_main", + "//tensorflow/core:testlib", "//tensorflow/core/grappler:grappler_item", "//tensorflow/core/grappler:utils", "//tensorflow/core/grappler/inputs:trivial_test_graph_input_yielder", + "//tensorflow/core/grappler/utils:grappler_test", ], ) diff --git a/tensorflow/core/grappler/optimizers/model_pruner_test.cc b/tensorflow/core/grappler/optimizers/model_pruner_test.cc index 8480a74572..2b12eadec9 100644 --- a/tensorflow/core/grappler/optimizers/model_pruner_test.cc +++ b/tensorflow/core/grappler/optimizers/model_pruner_test.cc @@ -16,9 +16,11 @@ limitations under the License. #include "tensorflow/core/grappler/optimizers/model_pruner.h" #include "tensorflow/cc/ops/standard_ops.h" #include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/framework/tensor_testutil.h" #include "tensorflow/core/grappler/grappler_item.h" #include "tensorflow/core/grappler/inputs/trivial_test_graph_input_yielder.h" #include "tensorflow/core/grappler/utils.h" +#include "tensorflow/core/grappler/utils/grappler_test.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/platform/test.h" @@ -26,7 +28,7 @@ namespace tensorflow { namespace grappler { namespace { -class ModelPrunerTest : public ::testing::Test {}; +class ModelPrunerTest : public GrapplerTest {}; TEST_F(ModelPrunerTest, NoPruning) { // This trivial graph is so basic there's nothing to prune. @@ -86,6 +88,13 @@ TEST_F(ModelPrunerTest, StopGradientPruning) { EXPECT_EQ(NodeName(b.name()), new_e.input(0)); EXPECT_EQ(1, new_d.input_size()); EXPECT_EQ(NodeName(b.name()), new_d.input(0)); + + std::vector fetch = {"e"}; + auto expected_tensors = EvaluateNodes(item.graph, fetch); + auto actual_tensors = EvaluateNodes(output, fetch); + EXPECT_EQ(1, expected_tensors.size()); + EXPECT_EQ(1, actual_tensors.size()); + test::ExpectTensorEqual(expected_tensors[0], actual_tensors[0]); } TEST_F(ModelPrunerTest, IdentityPruning) { -- GitLab From 16997696d2dec1d74bc6341d10bad17b8c830bdd Mon Sep 17 00:00:00 2001 From: Amit Patankar Date: Tue, 10 Apr 2018 14:59:23 -0700 Subject: [PATCH 113/791] Forcing the symlink creation. --- tensorflow/tools/docker/Dockerfile | 2 +- tensorflow/tools/docker/Dockerfile.devel | 2 +- tensorflow/tools/docker/Dockerfile.devel-gpu | 2 +- tensorflow/tools/docker/Dockerfile.gpu | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/tools/docker/Dockerfile b/tensorflow/tools/docker/Dockerfile index 024cb40eb4..78cb4d250e 100644 --- a/tensorflow/tools/docker/Dockerfile +++ b/tensorflow/tools/docker/Dockerfile @@ -47,7 +47,7 @@ RUN pip --no-cache-dir install \ http://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.0.0-cp27-none-linux_x86_64.whl # --- ~ DO NOT EDIT OR DELETE BETWEEN THE LINES --- # -# RUN ln -s /usr/bin/python3 /usr/bin/python# +# RUN ln -s -f /usr/bin/python3 /usr/bin/python# # Set up our notebook config. COPY jupyter_notebook_config.py /root/.jupyter/ diff --git a/tensorflow/tools/docker/Dockerfile.devel b/tensorflow/tools/docker/Dockerfile.devel index c4f6b24e5c..b3dbe475d2 100644 --- a/tensorflow/tools/docker/Dockerfile.devel +++ b/tensorflow/tools/docker/Dockerfile.devel @@ -38,7 +38,7 @@ RUN pip --no-cache-dir install \ && \ python -m ipykernel.kernelspec -# RUN ln -s /usr/bin/python3 /usr/bin/python# +# RUN ln -s -f /usr/bin/python3 /usr/bin/python# # Set up our notebook config. COPY jupyter_notebook_config.py /root/.jupyter/ diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu index 5aea47e582..bfb96da58d 100644 --- a/tensorflow/tools/docker/Dockerfile.devel-gpu +++ b/tensorflow/tools/docker/Dockerfile.devel-gpu @@ -47,7 +47,7 @@ RUN pip --no-cache-dir install \ && \ python -m ipykernel.kernelspec -# RUN ln -s /usr/bin/python3 /usr/bin/python# +# RUN ln -s -f /usr/bin/python3 /usr/bin/python# # Set up our notebook config. COPY jupyter_notebook_config.py /root/.jupyter/ diff --git a/tensorflow/tools/docker/Dockerfile.gpu b/tensorflow/tools/docker/Dockerfile.gpu index 625321e123..9e1708662e 100644 --- a/tensorflow/tools/docker/Dockerfile.gpu +++ b/tensorflow/tools/docker/Dockerfile.gpu @@ -54,7 +54,7 @@ RUN pip --no-cache-dir install \ http://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-0.0.0-cp27-none-linux_x86_64.whl # --- ~ DO NOT EDIT OR DELETE BETWEEN THE LINES --- # -# RUN ln -s /usr/bin/python3 /usr/bin/python# +# RUN ln -s -f /usr/bin/python3 /usr/bin/python# # Set up our notebook config. COPY jupyter_notebook_config.py /root/.jupyter/ -- GitLab From 99e198185d3a4a8bb089102b71b9fc3920427887 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 15:01:49 -0700 Subject: [PATCH 114/791] Add quantized LogSoftmax. PiperOrigin-RevId: 192352432 --- .../internal/optimized/optimized_ops.h | 91 ++++++++++++++++++- .../kernels/internal/quantization_util.cc | 16 ++++ .../lite/kernels/internal/quantization_util.h | 7 +- .../internal/reference/reference_ops.h | 86 ++++++++++++++++++ .../toco/graph_transformations/quantize.cc | 16 ++++ tensorflow/contrib/lite/toco/model.h | 11 ++- 6 files changed, 224 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h index e329e02273..22c0504ad2 100644 --- a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h +++ b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h @@ -4135,6 +4135,7 @@ inline void Softmax(const uint8* input_data, const Dims<4>& input_dims, // optimized yet. inline void LogSoftmax(const float* input_data, const Dims<4>& input_dims, float* output_data, const Dims<4>& output_dims) { + gemmlowp::ScopedProfilingLabel label("LogSoftmax"); const int batches = MatchingArraySize(input_dims, 3, output_dims, 3); const int height = MatchingArraySize(input_dims, 2, output_dims, 2); const int width = MatchingArraySize(input_dims, 1, output_dims, 1); @@ -4168,6 +4169,94 @@ inline void LogSoftmax(const float* input_data, const Dims<4>& input_dims, } } +// Currently just a copy of the reference code. +inline void LogSoftmax(const uint8* input_data, const Dims<4>& input_dims, + int32 input_multiplier, int32 input_left_shift, + int32 reverse_scaling_divisor, + int32 reverse_scaling_right_shift, int diff_min, + uint8* output_data, const Dims<4>& output_dims) { + gemmlowp::ScopedProfilingLabel label("LogSoftmax/Uint8"); + // The representation chosen for the input to the exp() function is Q5.26. + // We need to leave extra space since values that we skip might be as large as + // -32 before multiplying by input_beta_multiplier, and therefore as large as + // -16 afterwards. Note that exp(-8) is definitely not insignificant to + // accumulation, but exp(-16) definitely is. + static constexpr int kScaledDiffIntegerBits = 5; + static constexpr int kAccumulationIntegerBits = 12; + static constexpr int kOutputIntegerBits = 4; + using FixedPointScaledDiff = + gemmlowp::FixedPoint; + using FixedPointAccum = gemmlowp::FixedPoint; + using FixedPoint0 = gemmlowp::FixedPoint; + + const int outer_size = MatchingFlatSizeSkipDim(input_dims, 0, output_dims); + const int depth = MatchingArraySize(input_dims, 0, output_dims, 0); + + for (int i = 0; i < outer_size; ++i) { + uint8 max_in_row = 0; + for (int c = 0; c < depth; ++c) { + max_in_row = std::max(max_in_row, input_data[i * depth + c]); + } + + FixedPointAccum sum_of_exps = FixedPointAccum::Zero(); + for (int c = 0; c < depth; ++c) { + int32 input_diff = + static_cast(input_data[i * depth + c]) - max_in_row; + if (input_diff >= diff_min) { + const int32 input_diff_rescaled = + MultiplyByQuantizedMultiplierGreaterThanOne( + input_diff, input_multiplier, input_left_shift); + const FixedPointScaledDiff scaled_diff_f8 = + FixedPointScaledDiff::FromRaw(input_diff_rescaled); + sum_of_exps = sum_of_exps + gemmlowp::Rescale( + exp_on_negative_values(scaled_diff_f8)); + } + } + + // TODO(b/77858996): Implement fixed-point log(). + // Not a fully-quantized implementation: floating-point log(). + const float float_log_sum_of_exps = + std::log(static_cast(sum_of_exps.raw()) / + (1 << (31 - kAccumulationIntegerBits))); + const int32 fixed_log_sum_of_exps = static_cast(TfLiteRound( + float_log_sum_of_exps * (1 << (31 - kScaledDiffIntegerBits)))); + + // rescaled_diff_min is smallest representable in + // Q(kScaledDiffIntegerBits).(31-kScaledDiffIntegerBits) plus the + // log-sub-exps that will be subtracted in the loop. + // + // The thresholds diff_min, etc are negative. + const int rescaled_diff_min = + fixed_log_sum_of_exps + std::numeric_limits::lowest(); + const int adjusted_diff_min = + std::max(diff_min - 1, // Note use of > below instead of >= above. + MultiplyByQuantizedMultiplierSmallerThanOne( + rescaled_diff_min, reverse_scaling_divisor, + reverse_scaling_right_shift)); + + for (int c = 0; c < depth; ++c) { + int32 input_diff = + static_cast(input_data[i * depth + c]) - max_in_row; + if (input_diff > adjusted_diff_min) { + const int32 input_diff_rescaled = + MultiplyByQuantizedMultiplierGreaterThanOne( + input_diff, input_multiplier, input_left_shift); + int32 unsat_output = + gemmlowp::RoundingDivideByPOT( + (input_diff_rescaled - fixed_log_sum_of_exps), + 31 - kScaledDiffIntegerBits - kOutputIntegerBits) + + 255; + + output_data[i * depth + c] = static_cast( + std::max(std::min(unsat_output, static_cast(255)), 0)); + } else { + // Set output to smallest value. + output_data[i * depth + c] = 0; + } + } + } +} + inline void Logistic(const float* input_data, const Dims<4>& input_dims, float* output_data, const Dims<4>& output_dims) { gemmlowp::ScopedProfilingLabel label("Logistic"); @@ -4181,7 +4270,7 @@ inline void Logistic(const uint8* input_data, const Dims<4>& input_dims, int32 input_zero_point, int32 input_range_radius, int32 input_multiplier, int input_left_shift, uint8* output_data, const Dims<4>& output_dims) { - gemmlowp::ScopedProfilingLabel label("Logistic"); + gemmlowp::ScopedProfilingLabel label("Logistic/Uint8"); /* batches */ MatchingArraySize(input_dims, 3, output_dims, 3); /* height */ MatchingArraySize(input_dims, 2, output_dims, 2); /* width */ MatchingArraySize(input_dims, 1, output_dims, 1); diff --git a/tensorflow/contrib/lite/kernels/internal/quantization_util.cc b/tensorflow/contrib/lite/kernels/internal/quantization_util.cc index 18be6777a5..dd86313726 100644 --- a/tensorflow/contrib/lite/kernels/internal/quantization_util.cc +++ b/tensorflow/contrib/lite/kernels/internal/quantization_util.cc @@ -78,6 +78,22 @@ void PreprocessSoftmaxScaling(double beta, double input_scale, quantized_multiplier, left_shift); } +void PreprocessLogSoftmaxScaling(double beta, double input_scale, + int input_integer_bits, + int32_t* quantized_multiplier, int* left_shift, + int32_t* reverse_scaling_divisor, + int* reverse_scaling_right_shift) { + PreprocessSoftmaxScaling(beta, input_scale, input_integer_bits, + quantized_multiplier, left_shift); + + // Also calculate what amounts to the inverse scaling factor for the input. + const double real_reverse_scaling_divisor = + (1 << (31 - *left_shift)) / static_cast(*quantized_multiplier); + tflite::QuantizeMultiplierSmallerThanOne(real_reverse_scaling_divisor, + reverse_scaling_divisor, + reverse_scaling_right_shift); +} + int CalculateInputRadius(int input_integer_bits, int input_left_shift) { const double max_input_rescaled = 1.0 * ((1 << input_integer_bits) - 1) * (1ll << (31 - input_integer_bits)) / diff --git a/tensorflow/contrib/lite/kernels/internal/quantization_util.h b/tensorflow/contrib/lite/kernels/internal/quantization_util.h index 9a04b76e56..1f6f5d3b15 100644 --- a/tensorflow/contrib/lite/kernels/internal/quantization_util.h +++ b/tensorflow/contrib/lite/kernels/internal/quantization_util.h @@ -196,7 +196,12 @@ void QuantizeMultiplier(double double_multiplier, int32_t* quantized_multiplier, void PreprocessSoftmaxScaling(double beta, double input_scale, int input_integer_bits, int32_t* quantized_multiplier, int* left_shift); - +// Like PreprocessSoftmaxScaling, but inverse scaling factors also calculated. +void PreprocessLogSoftmaxScaling(double beta, double input_scale, + int input_integer_bits, + int32_t* quantized_multiplier, int* left_shift, + int32_t* reverse_scaling_divisor, + int* reverse_scaling_right_shift); // Calculate the largest input that will result in a within-bounds intermediate // result within MultiplyByQuantizedMultiplierGreaterThanOne. In other words, // it must not overflow before we reduce the value by multiplication by the diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h index 250a308f2a..93b4eb5504 100644 --- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h +++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h @@ -2447,6 +2447,92 @@ inline void LogSoftmax(const float* input_data, const Dims<4>& input_dims, } } +inline void LogSoftmax(const uint8* input_data, const Dims<4>& input_dims, + int32 input_multiplier, int32 input_left_shift, + int32 reverse_scaling_divisor, + int32 reverse_scaling_right_shift, int diff_min, + uint8* output_data, const Dims<4>& output_dims) { + // The representation chosen for the input to the exp() function is Q5.26. + // We need to leave extra space since values that we skip might be as large as + // -32 before multiplying by input_beta_multiplier, and therefore as large as + // -16 afterwards. Note that exp(-8) is definitely not insignificant to + // accumulation, but exp(-16) definitely is. + static constexpr int kScaledDiffIntegerBits = 5; + static constexpr int kAccumulationIntegerBits = 12; + static constexpr int kOutputIntegerBits = 4; + using FixedPointScaledDiff = + gemmlowp::FixedPoint; + using FixedPointAccum = gemmlowp::FixedPoint; + using FixedPoint0 = gemmlowp::FixedPoint; + + const int outer_size = MatchingFlatSizeSkipDim(input_dims, 0, output_dims); + const int depth = MatchingArraySize(input_dims, 0, output_dims, 0); + + for (int i = 0; i < outer_size; ++i) { + uint8 max_in_row = 0; + for (int c = 0; c < depth; ++c) { + max_in_row = std::max(max_in_row, input_data[i * depth + c]); + } + + FixedPointAccum sum_of_exps = FixedPointAccum::Zero(); + for (int c = 0; c < depth; ++c) { + int32 input_diff = + static_cast(input_data[i * depth + c]) - max_in_row; + if (input_diff >= diff_min) { + const int32 input_diff_rescaled = + MultiplyByQuantizedMultiplierGreaterThanOne( + input_diff, input_multiplier, input_left_shift); + const FixedPointScaledDiff scaled_diff_f8 = + FixedPointScaledDiff::FromRaw(input_diff_rescaled); + sum_of_exps = sum_of_exps + gemmlowp::Rescale( + exp_on_negative_values(scaled_diff_f8)); + } + } + + // TODO(b/77858996): Implement fixed-point log(). + // Not a fully-quantized implementation: floating-point log(). + const float float_log_sum_of_exps = + std::log(static_cast(sum_of_exps.raw()) / + (1 << (31 - kAccumulationIntegerBits))); + const int32 fixed_log_sum_of_exps = static_cast(TfLiteRound( + float_log_sum_of_exps * (1 << (31 - kScaledDiffIntegerBits)))); + + // rescaled_diff_min is smallest representable in + // Q(kScaledDiffIntegerBits).(31-kScaledDiffIntegerBits) plus the + // log-sub-exps that will be subtracted in the loop. + // + // The thresholds diff_min, etc are negative. + const int rescaled_diff_min = + fixed_log_sum_of_exps + std::numeric_limits::lowest(); + const int adjusted_diff_min = + std::max(diff_min - 1, // Note use of > below instead of >= above. + MultiplyByQuantizedMultiplierSmallerThanOne( + rescaled_diff_min, reverse_scaling_divisor, + reverse_scaling_right_shift)); + + for (int c = 0; c < depth; ++c) { + int32 input_diff = + static_cast(input_data[i * depth + c]) - max_in_row; + if (input_diff > adjusted_diff_min) { + const int32 input_diff_rescaled = + MultiplyByQuantizedMultiplierGreaterThanOne( + input_diff, input_multiplier, input_left_shift); + int32 unsat_output = + gemmlowp::RoundingDivideByPOT( + (input_diff_rescaled - fixed_log_sum_of_exps), + 31 - kScaledDiffIntegerBits - kOutputIntegerBits) + + 255; + + output_data[i * depth + c] = static_cast( + std::max(std::min(unsat_output, static_cast(255)), 0)); + } else { + // Set output to smallest value. + output_data[i * depth + c] = 0; + } + } + } +} + inline void Logistic(const float* input_data, const Dims<4>& input_dims, float* output_data, const Dims<4>& output_dims) { const int flat_size = MatchingFlatSize(output_dims, input_dims); diff --git a/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc b/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc index 5b1268f9a9..f50830ae60 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc +++ b/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc @@ -44,6 +44,7 @@ bool SupportsQuantization(const Operator& op) { type == OperatorType::kTensorFlowMinimum || type == OperatorType::kTensorFlowMaximum || type == OperatorType::kLogistic || type == OperatorType::kSoftmax || + type == OperatorType::kLogSoftmax || type == OperatorType::kTensorFlowSplit || type == OperatorType::kSub || type == OperatorType::kSqueeze || type == OperatorType::kPad || type == OperatorType::kTensorFlowReshape || @@ -394,6 +395,19 @@ bool ChooseHardcodedQuantizationForOperatorOutput( *quantization_params)); return true; } + if (op.type == OperatorType::kLogSoftmax) { + // LogSoftmax has range: [LogSoftmaxOperator::kOutputRangeMin, 0]. + *quantized_data_type = GetQuantizedDataType(array, *quantized_data_type); + const QuantizationPoints qp = GetQuantizationPoints(*quantized_data_type); + quantization_params->zero_point = qp.max_value; + quantization_params->scale = + -LogSoftmaxOperator::kOutputRangeMin / (qp.max_value + 1); + // While not strictly necessary, it is easier to interpret output data and + // quantization if the scale is similar to others (such as power of 2). + CHECK(IsExactlyRepresentable(LogSoftmaxOperator::kOutputRangeMin / 2, + *quantized_data_type, *quantization_params)); + return true; + } if (op.type == OperatorType::kTanh) { // Tanh has the range: [-1, 1]. *quantized_data_type = GetQuantizedDataType(array, *quantized_data_type); @@ -661,6 +675,8 @@ bool Quantize::Run(Model* model, std::size_t op_index) { // Fix up the min/max information on the output array to match the chosen // quantization parameters. + CHECK(output_array.minmax) + << "Output array named " << output << " lacks minmax"; auto& output_minmax = output_array.GetMinMax(); FixMinMaxPostQuantization(quantized_data_type, quantization_params, &output_minmax); diff --git a/tensorflow/contrib/lite/toco/model.h b/tensorflow/contrib/lite/toco/model.h index 56ef9fe2a8..54c3a59506 100644 --- a/tensorflow/contrib/lite/toco/model.h +++ b/tensorflow/contrib/lite/toco/model.h @@ -1329,6 +1329,15 @@ struct SoftmaxOperator : Operator { // TensorFlow equivalent: LogSoftmax struct LogSoftmaxOperator : Operator { LogSoftmaxOperator() : Operator(OperatorType::kLogSoftmax) {} + + // LogSoftmax can in principal have very large negative output, depending on + // the input size. However, input x_i that is less than x_max-10 is + // accumulated as exp(x_i-x_max), which is truncated to zero. + // + // Since we effectively disregard smallish inputs in the normalizing factor, + // we also drop them in the output (set to minimum output), and in doing so + // make better use of the quantization range / resolution. + static constexpr float kOutputRangeMin = -16.0; }; // Cast operator. @@ -1522,7 +1531,7 @@ class Shape { int dims(int i) const { // Always check for out-of-bounds accesses, even in optimized builds where // standard assertions are disabled. Out-of-bounds access here is a common - // occurence. + // occurrence. CHECK_GE(i, 0); CHECK_GT(dims_.size(), i); return dims_[i]; -- GitLab From 0172f3b5b86ccdf32366259a31266a988a9445d5 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 15:23:05 -0700 Subject: [PATCH 115/791] Allow negative feature values in computation for `sum` combiner. PiperOrigin-RevId: 192355950 --- .../layers/python/layers/embedding_ops.py | 15 ++++- .../python/feature_column/feature_column.py | 15 ++++- .../feature_column/feature_column_test.py | 57 ++++++++++++++----- 3 files changed, 70 insertions(+), 17 deletions(-) diff --git a/tensorflow/contrib/layers/python/layers/embedding_ops.py b/tensorflow/contrib/layers/python/layers/embedding_ops.py index ffa208540d..49c3faf3b7 100644 --- a/tensorflow/contrib/layers/python/layers/embedding_ops.py +++ b/tensorflow/contrib/layers/python/layers/embedding_ops.py @@ -140,6 +140,9 @@ def safe_embedding_lookup_sparse(embedding_weights, # Prune invalid ids and weights. sparse_ids, sparse_weights = _prune_invalid_ids(sparse_ids, sparse_weights) + if combiner != "sum": + sparse_ids, sparse_weights = _prune_invalid_weights( + sparse_ids, sparse_weights) # Fill in dummy values for empty features, if necessary. sparse_ids, is_row_empty = sparse_ops.sparse_fill_empty_rows(sparse_ids, @@ -188,13 +191,23 @@ def _prune_invalid_ids(sparse_ids, sparse_weights): is_id_valid = math_ops.greater_equal(sparse_ids.values, 0) if sparse_weights is not None: is_id_valid = math_ops.logical_and( - is_id_valid, math_ops.greater(sparse_weights.values, 0)) + is_id_valid, + array_ops.ones_like(sparse_weights.values, dtype=dtypes.bool)) sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_id_valid) if sparse_weights is not None: sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_id_valid) return sparse_ids, sparse_weights +def _prune_invalid_weights(sparse_ids, sparse_weights): + """Prune invalid weights (< 0) from the input ids and weights.""" + if sparse_weights is not None: + is_weights_valid = math_ops.greater(sparse_weights.values, 0) + sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_weights_valid) + sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_weights_valid) + return sparse_ids, sparse_weights + + def scattered_embedding_lookup(params, values, dimension, diff --git a/tensorflow/python/feature_column/feature_column.py b/tensorflow/python/feature_column/feature_column.py index 7a104fa4ac..f9201a4794 100644 --- a/tensorflow/python/feature_column/feature_column.py +++ b/tensorflow/python/feature_column/feature_column.py @@ -3148,6 +3148,9 @@ def _safe_embedding_lookup_sparse(embedding_weights, # Prune invalid ids and weights. sparse_ids, sparse_weights = _prune_invalid_ids(sparse_ids, sparse_weights) + if combiner != 'sum': + sparse_ids, sparse_weights = _prune_invalid_weights( + sparse_ids, sparse_weights) # Fill in dummy values for empty features, if necessary. sparse_ids, is_row_empty = sparse_ops.sparse_fill_empty_rows(sparse_ids, @@ -3196,13 +3199,23 @@ def _prune_invalid_ids(sparse_ids, sparse_weights): is_id_valid = math_ops.greater_equal(sparse_ids.values, 0) if sparse_weights is not None: is_id_valid = math_ops.logical_and( - is_id_valid, math_ops.greater(sparse_weights.values, 0)) + is_id_valid, + array_ops.ones_like(sparse_weights.values, dtype=dtypes.bool)) sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_id_valid) if sparse_weights is not None: sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_id_valid) return sparse_ids, sparse_weights +def _prune_invalid_weights(sparse_ids, sparse_weights): + """Prune invalid weights (< 0) from the input ids and weights.""" + if sparse_weights is not None: + is_weights_valid = math_ops.greater(sparse_weights.values, 0) + sparse_ids = sparse_ops.sparse_retain(sparse_ids, is_weights_valid) + sparse_weights = sparse_ops.sparse_retain(sparse_weights, is_weights_valid) + return sparse_ids, sparse_weights + + class _IndicatorColumn(_DenseColumn, _SequenceDenseColumn, collections.namedtuple('_IndicatorColumn', ['categorical_column'])): diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py index 07588af37e..62718db0e5 100644 --- a/tensorflow/python/feature_column/feature_column_test.py +++ b/tensorflow/python/feature_column/feature_column_test.py @@ -1511,6 +1511,28 @@ class LinearModelTest(test.TestCase): sess.run(bias.assign([5.])) self.assertAllClose([[1005.], [5010.]], predictions.eval()) + def test_sparse_combiner_with_negative_weights(self): + wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) + wire_cast_weights = fc.weighted_categorical_column(wire_cast, 'weights') + + with ops.Graph().as_default(): + wire_tensor = sparse_tensor.SparseTensor( + values=['omar', 'stringer', 'marlo'], # hashed to = [2, 0, 3] + indices=[[0, 0], [1, 0], [1, 1]], + dense_shape=[2, 2]) + features = { + 'wire_cast': wire_tensor, + 'weights': constant_op.constant([[1., 1., -1.0]]) + } + predictions = fc.linear_model( + features, [wire_cast_weights], sparse_combiner='sum') + bias = get_linear_model_bias() + wire_cast_var = get_linear_model_column_var(wire_cast) + with _initialized_session() as sess: + sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]])) + sess.run(bias.assign([5.])) + self.assertAllClose([[1005.], [-9985.]], predictions.eval()) + def test_dense_multi_dimension_multi_output(self): price = fc.numeric_column('price', shape=2) with ops.Graph().as_default(): @@ -6164,14 +6186,16 @@ class WeightedCategoricalColumnTest(test.TestCase): key='ids', num_buckets=3), weight_feature_key='values') with ops.Graph().as_default(): - predictions = get_keras_linear_model_predictions({ - 'ids': - sparse_tensor.SparseTensorValue( - indices=((0, 0), (1, 0), (1, 1)), - values=(0, 2, 1), - dense_shape=(2, 2)), - 'values': ((.5,), (1.,)) - }, (column,)) + predictions = get_keras_linear_model_predictions( + { + 'ids': + sparse_tensor.SparseTensorValue( + indices=((0, 0), (1, 0), (1, 1)), + values=(0, 2, 1), + dense_shape=(2, 2)), + 'values': ((.5,), (1.,)) + }, (column,), + sparse_combiner='mean') with _initialized_session(): with self.assertRaisesRegexp(errors.OpError, 'Incompatible shapes'): predictions.eval() @@ -6255,13 +6279,16 @@ class WeightedCategoricalColumnTest(test.TestCase): key='ids', num_buckets=3), weight_feature_key='values') with ops.Graph().as_default(): - predictions = fc.linear_model({ - 'ids': sparse_tensor.SparseTensorValue( - indices=((0, 0), (1, 0), (1, 1)), - values=(0, 2, 1), - dense_shape=(2, 2)), - 'values': ((.5,), (1.,)) - }, (column,)) + predictions = fc.linear_model( + { + 'ids': + sparse_tensor.SparseTensorValue( + indices=((0, 0), (1, 0), (1, 1)), + values=(0, 2, 1), + dense_shape=(2, 2)), + 'values': ((.5,), (1.,)) + }, (column,), + sparse_combiner='mean') with _initialized_session(): with self.assertRaisesRegexp(errors.OpError, 'Incompatible shapes'): predictions.eval() -- GitLab From 9eaab27bc41b6865bc945dcbb6b75c2427826ef3 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 15:39:37 -0700 Subject: [PATCH 116/791] [XLA] Redesign: implement and test Conv. PiperOrigin-RevId: 192359226 --- .../xla/client/xla_client/xla_builder.cc | 170 +++++++++++++++++- .../xla/client/xla_client/xla_builder.h | 14 ++ tensorflow/compiler/xla/tests/BUILD | 2 +- .../compiler/xla/tests/convolution_test.cc | 61 ++++--- 4 files changed, 210 insertions(+), 37 deletions(-) diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc index 7481b357ff..9e4b9ccd25 100644 --- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc +++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc @@ -790,24 +790,101 @@ XlaOp XlaBuilder::DotGeneral(const XlaOp& lhs, const XlaOp& rhs, }); } +Status XlaBuilder::VerifyConvolution( + const Shape& lhs_shape, const Shape& rhs_shape, + const ConvolutionDimensionNumbers& dimension_numbers) const { + if (ShapeUtil::Rank(lhs_shape) != ShapeUtil::Rank(rhs_shape)) { + return InvalidArgument( + "Convolution arguments must have same number of " + "dimensions. Got: %s and %s", + ShapeUtil::HumanString(lhs_shape).c_str(), + ShapeUtil::HumanString(rhs_shape).c_str()); + } + int num_dims = ShapeUtil::Rank(lhs_shape); + if (num_dims < 2) { + return InvalidArgument( + "Convolution expects argument arrays with >= 3 dimensions. " + "Got: %s and %s", + ShapeUtil::HumanString(lhs_shape).c_str(), + ShapeUtil::HumanString(rhs_shape).c_str()); + } + int num_spatial_dims = num_dims - 2; + + const auto check_spatial_dimensions = + [&](const char* const field_name, + const tensorflow::protobuf::RepeatedField& + numbers) { + if (numbers.size() != num_spatial_dims) { + return InvalidArgument("Expected %d elements for %s, but got %d.", + num_spatial_dims, field_name, numbers.size()); + } + for (int i = 0; i < numbers.size(); ++i) { + if (numbers.Get(i) < 0 || numbers.Get(i) >= num_dims) { + return InvalidArgument("Convolution %s[%d] is out of bounds: %lld", + field_name, i, numbers.Get(i)); + } + } + return Status::OK(); + }; + TF_RETURN_IF_ERROR( + check_spatial_dimensions("input_spatial_dimensions", + dimension_numbers.input_spatial_dimensions())); + TF_RETURN_IF_ERROR( + check_spatial_dimensions("kernel_spatial_dimensions", + dimension_numbers.kernel_spatial_dimensions())); + return check_spatial_dimensions( + "output_spatial_dimensions", + dimension_numbers.output_spatial_dimensions()); +} + XlaOp XlaBuilder::Conv(const XlaOp& lhs, const XlaOp& rhs, tensorflow::gtl::ArraySlice window_strides, Padding padding) { - return UnimplementedOp(); + return ConvWithGeneralDimensions( + lhs, rhs, window_strides, padding, + CreateDefaultConvDimensionNumbers(window_strides.size())); } XlaOp XlaBuilder::ConvWithGeneralPadding( const XlaOp& lhs, const XlaOp& rhs, tensorflow::gtl::ArraySlice window_strides, tensorflow::gtl::ArraySlice> padding) { - return UnimplementedOp(); + return ConvGeneral(lhs, rhs, window_strides, padding, + CreateDefaultConvDimensionNumbers(window_strides.size())); } XlaOp XlaBuilder::ConvWithGeneralDimensions( const XlaOp& lhs, const XlaOp& rhs, tensorflow::gtl::ArraySlice window_strides, Padding padding, const ConvolutionDimensionNumbers& dimension_numbers) { - return UnimplementedOp(); + return NoteErrorOrReturn([&]() -> StatusOr { + TF_ASSIGN_OR_RETURN(const Shape& lhs_shape, GetShape(lhs)); + TF_ASSIGN_OR_RETURN(const Shape& rhs_shape, GetShape(rhs)); + + TF_RETURN_IF_ERROR( + VerifyConvolution(lhs_shape, rhs_shape, dimension_numbers)); + + std::vector base_area_dimensions( + dimension_numbers.input_spatial_dimensions_size()); + for (std::vector::size_type i = 0; i < base_area_dimensions.size(); + ++i) { + base_area_dimensions[i] = + lhs_shape.dimensions(dimension_numbers.input_spatial_dimensions(i)); + } + + std::vector window_dimensions( + dimension_numbers.kernel_spatial_dimensions_size()); + for (std::vector::size_type i = 0; i < window_dimensions.size(); + ++i) { + window_dimensions[i] = + rhs_shape.dimensions(dimension_numbers.kernel_spatial_dimensions(i)); + } + + return ConvGeneral(lhs, rhs, window_strides, + MakePadding(base_area_dimensions, window_dimensions, + window_strides, padding), + dimension_numbers); + }); } XlaOp XlaBuilder::ConvGeneral( @@ -815,7 +892,8 @@ XlaOp XlaBuilder::ConvGeneral( tensorflow::gtl::ArraySlice window_strides, tensorflow::gtl::ArraySlice> padding, const ConvolutionDimensionNumbers& dimension_numbers) { - return UnimplementedOp(); + return ConvGeneralDilated(lhs, rhs, window_strides, padding, {}, {}, + dimension_numbers); } XlaOp XlaBuilder::ConvGeneralDilated( @@ -825,7 +903,89 @@ XlaOp XlaBuilder::ConvGeneralDilated( tensorflow::gtl::ArraySlice lhs_dilation, tensorflow::gtl::ArraySlice rhs_dilation, const ConvolutionDimensionNumbers& dimension_numbers) { - return UnimplementedOp(); + return NoteErrorOrReturn([&]() -> StatusOr { + HloInstructionProto instr; + TF_ASSIGN_OR_RETURN(const Shape& lhs_shape, GetShape(lhs)); + TF_ASSIGN_OR_RETURN(const Shape& rhs_shape, GetShape(rhs)); + TF_RETURN_IF_ERROR( + VerifyConvolution(lhs_shape, rhs_shape, dimension_numbers)); + + std::vector window_dimensions( + dimension_numbers.kernel_spatial_dimensions_size()); + for (std::vector::size_type i = 0; i < window_dimensions.size(); + ++i) { + window_dimensions[i] = + rhs_shape.dimensions(dimension_numbers.kernel_spatial_dimensions(i)); + } + TF_ASSIGN_OR_RETURN(*instr.mutable_window(), + MakeWindow(window_dimensions, window_strides, padding, + lhs_dilation, rhs_dilation)); + + TF_ASSIGN_OR_RETURN( + *instr.mutable_shape(), + ShapeInference::InferConvolveShape(lhs_shape, rhs_shape, instr.window(), + dimension_numbers)); + + *instr.mutable_convolution_dimension_numbers() = dimension_numbers; + + return AddInstruction(std::move(instr), HloOpcode::kConvolution, + {lhs, rhs}); + }); +} + +StatusOr XlaBuilder::MakeWindow( + tensorflow::gtl::ArraySlice window_dimensions, + tensorflow::gtl::ArraySlice window_strides, + tensorflow::gtl::ArraySlice> padding, + tensorflow::gtl::ArraySlice lhs_dilation, + tensorflow::gtl::ArraySlice rhs_dilation) const { + const auto verify_size = [&](const size_t x, const char* x_name) { + if (x == 0 || x == window_dimensions.size()) { + return Status::OK(); + } else { + return InvalidArgument( + "%s", tensorflow::strings::StrCat( + "Window has different number of window dimensions than of ", + x_name, + "\nNumber of window dimensions: ", window_dimensions.size(), + "\nNumber of ", x_name, ": ", x, "\n") + .c_str()); + } + }; + TF_RETURN_IF_ERROR(verify_size(window_strides.size(), "window strides")); + TF_RETURN_IF_ERROR(verify_size(padding.size(), "padding entries")); + TF_RETURN_IF_ERROR(verify_size(lhs_dilation.size(), "lhs dilation factors")); + TF_RETURN_IF_ERROR(verify_size(rhs_dilation.size(), "rhs dilation factors")); + + Window window; + for (size_t i = 0; i < window_dimensions.size(); i++) { + auto dim = window.add_dimensions(); + dim->set_size(window_dimensions[i]); + if (!window_strides.empty()) { + dim->set_stride(window_strides[i]); + } else { + dim->set_stride(1); + } + if (!padding.empty()) { + dim->set_padding_low(padding[i].first); + dim->set_padding_high(padding[i].second); + } else { + dim->set_padding_low(0); + dim->set_padding_high(0); + } + if (!lhs_dilation.empty()) { + dim->set_base_dilation(lhs_dilation[i]); + } else { + dim->set_base_dilation(1); + } + if (!rhs_dilation.empty()) { + dim->set_window_dilation(rhs_dilation[i]); + } else { + dim->set_window_dilation(1); + } + dim->set_window_reversal(false); + } + return window; } XlaOp XlaBuilder::Fft(const XlaOp& operand, const FftType fft_type, diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.h b/tensorflow/compiler/xla/client/xla_client/xla_builder.h index d747691f16..24e0be2ac1 100644 --- a/tensorflow/compiler/xla/client/xla_client/xla_builder.h +++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.h @@ -835,6 +835,20 @@ class XlaBuilder { void IsConstantVisitor(const int64 op_handle, std::set* visited, bool* is_constant) const; + // Checks bounds for convolution parameters. + Status VerifyConvolution( + const Shape& lhs_shape, const Shape& rhs_shape, + const ConvolutionDimensionNumbers& dimension_numbers) const; + + // Helper function for creating a Window proto from user-supplied data. + // Returns error if the user-supplied data was invalid. + StatusOr MakeWindow( + tensorflow::gtl::ArraySlice window_dimensions, + tensorflow::gtl::ArraySlice window_strides, + tensorflow::gtl::ArraySlice> padding, + tensorflow::gtl::ArraySlice lhs_dilation, + tensorflow::gtl::ArraySlice rhs_dilation) const; + string name_; // Name to use for the built computation. // The first error encountered while building the computation. diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD index 19fb4886db..67c53c6ac0 100644 --- a/tensorflow/compiler/xla/tests/BUILD +++ b/tensorflow/compiler/xla/tests/BUILD @@ -781,10 +781,10 @@ xla_test( "//tensorflow/compiler/xla:statusor", "//tensorflow/compiler/xla:util", "//tensorflow/compiler/xla:xla_data_proto", - "//tensorflow/compiler/xla/client:computation_builder", "//tensorflow/compiler/xla/client:global_data", "//tensorflow/compiler/xla/client:local_client", "//tensorflow/compiler/xla/client:padding", + "//tensorflow/compiler/xla/client/xla_client:xla_builder", "//tensorflow/compiler/xla/tests:client_library_test_base", "//tensorflow/compiler/xla/tests:literal_test_util", "//tensorflow/compiler/xla/tests:xla_internal_test_main", diff --git a/tensorflow/compiler/xla/tests/convolution_test.cc b/tensorflow/compiler/xla/tests/convolution_test.cc index 72715398de..5eb3136abe 100644 --- a/tensorflow/compiler/xla/tests/convolution_test.cc +++ b/tensorflow/compiler/xla/tests/convolution_test.cc @@ -20,10 +20,10 @@ limitations under the License. #include "tensorflow/compiler/xla/array2d.h" #include "tensorflow/compiler/xla/array4d.h" -#include "tensorflow/compiler/xla/client/computation_builder.h" #include "tensorflow/compiler/xla/client/global_data.h" #include "tensorflow/compiler/xla/client/local_client.h" #include "tensorflow/compiler/xla/client/padding.h" +#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h" #include "tensorflow/compiler/xla/layout_util.h" #include "tensorflow/compiler/xla/literal_util.h" #include "tensorflow/compiler/xla/ptr_util.h" @@ -88,12 +88,12 @@ class ForwardPassConvolution_3x3x256_256_OutputZ_Iota : public ConvolutionTest { ASSERT_EQ(2, arhs->width()); ASSERT_EQ(2, arhs->height()); - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto lhs = builder.ConstantR4FromArray4D(*alhs); auto rhs = builder.ConstantR4FromArray4D(*arhs); - auto conv = builder.Conv(lhs, rhs, {1, 1}, Padding::kValid); + builder.Conv(lhs, rhs, {1, 1}, Padding::kValid); - ComputeAndCompare(&builder, conv, {}, error_spec_); + ComputeAndCompare(&builder, {}, error_spec_); } }; @@ -106,12 +106,12 @@ template class Convolve_1x1x1x2_1x1x1x2_Valid : public ConvolutionTest { public: void RunTest() { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); Shape input_shape = ShapeUtil::MakeShapeWithType({1, 1, 1, 2}); Shape filter_shape = ShapeUtil::MakeShapeWithType({1, 1, 1, 2}); auto input = builder.Parameter(0, input_shape, "input"); auto filter = builder.Parameter(1, filter_shape, "filter"); - auto conv = builder.Conv(input, filter, {1, 1}, Padding::kValid); + builder.Conv(input, filter, {1, 1}, Padding::kValid); Array4D input_data(1, 1, 1, 2); input_data.FillWithYX(Array2D({ @@ -122,7 +122,7 @@ class Convolve_1x1x1x2_1x1x1x2_Valid : public ConvolutionTest { {5.0f, 6.0f}, })); - ComputeAndCompare(&builder, conv, + ComputeAndCompare(&builder, {std::move(*Literal::CreateFromArray(input_data)), std::move(*Literal::CreateFromArray(filter_data))}, error_spec_); @@ -137,12 +137,12 @@ template class Convolve_1x1x4x4_1x1x2x2_Valid : public ConvolutionTest { public: void RunTest() { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); Shape input_shape = ShapeUtil::MakeShapeWithType({1, 1, 4, 4}); Shape filter_shape = ShapeUtil::MakeShapeWithType({1, 1, 2, 2}); auto input = builder.Parameter(0, input_shape, "input"); auto filter = builder.Parameter(1, filter_shape, "filter"); - auto conv = builder.Conv(input, filter, {1, 1}, Padding::kValid); + builder.Conv(input, filter, {1, 1}, Padding::kValid); Array4D input_data(1, 1, 4, 4); input_data.FillWithYX(Array2D({ @@ -156,7 +156,7 @@ class Convolve_1x1x4x4_1x1x2x2_Valid : public ConvolutionTest { {5.0f, 6.0f}, {7.0f, 8.0f}, })); - ComputeAndCompare(&builder, conv, + ComputeAndCompare(&builder, {std::move(*Literal::CreateFromArray(input_data)), std::move(*Literal::CreateFromArray(filter_data))}, error_spec_); @@ -171,12 +171,12 @@ template class Convolve_1x1x4x4_1x1x2x2_Same : public ConvolutionTest { public: void RunTest() { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); Shape input_shape = ShapeUtil::MakeShapeWithType({1, 1, 4, 4}); Shape filter_shape = ShapeUtil::MakeShapeWithType({1, 1, 2, 2}); auto input = builder.Parameter(0, input_shape, "input"); auto filter = builder.Parameter(1, filter_shape, "filter"); - auto conv = builder.Conv(input, filter, {1, 1}, Padding::kSame); + builder.Conv(input, filter, {1, 1}, Padding::kSame); Array4D input_data(1, 1, 4, 4); input_data.FillWithYX(Array2D({ @@ -191,7 +191,7 @@ class Convolve_1x1x4x4_1x1x2x2_Same : public ConvolutionTest { {7.0f, 8.0f}, })); - ComputeAndCompare(&builder, conv, + ComputeAndCompare(&builder, {std::move(*Literal::CreateFromArray(input_data)), std::move(*Literal::CreateFromArray(filter_data))}, error_spec_); @@ -207,12 +207,12 @@ template class Convolve_1x1x4x4_1x1x3x3_Same : public ConvolutionTest { public: void RunTest() { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); Shape input_shape = ShapeUtil::MakeShapeWithType({1, 1, 4, 4}); Shape filter_shape = ShapeUtil::MakeShapeWithType({1, 1, 3, 3}); auto input = builder.Parameter(0, input_shape, "input"); auto filter = builder.Parameter(1, filter_shape, "filter"); - auto conv = builder.Conv(input, filter, {1, 1}, Padding::kSame); + builder.Conv(input, filter, {1, 1}, Padding::kSame); Array4D input_data(1, 1, 4, 4); input_data.FillWithYX(Array2D({{1.0f, 2.0f, 3.0f, 4.0f}, @@ -223,7 +223,7 @@ class Convolve_1x1x4x4_1x1x3x3_Same : public ConvolutionTest { filter_data.FillWithYX(Array2D( {{5.0f, 6.0f, 7.0f}, {8.0f, 9.0f, 10.0f}, {11.0f, 12.0f, 13.0f}})); // clang-format on - ComputeAndCompare(&builder, conv, + ComputeAndCompare(&builder, {std::move(*Literal::CreateFromArray(input_data)), std::move(*Literal::CreateFromArray(filter_data))}, error_spec_); @@ -234,7 +234,7 @@ TYPED_TEST_CASE(Convolve_1x1x4x4_1x1x3x3_Same, TestTypes); TYPED_TEST(Convolve_1x1x4x4_1x1x3x3_Same, Types) { this->RunTest(); } XLA_TEST_F(ConvolutionTest, Convolve1D_1x2x5_1x2x2_Valid) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); { Shape input_shape = ShapeUtil::MakeShape(F32, {1, 2, 5}); Shape filter_shape = ShapeUtil::MakeShape(F32, {1, 2, 2}); @@ -264,7 +264,7 @@ template class Convolve1D_1x2x5_1x2x2_WithRHSDilation : public ConvolutionTest { public: void RunTest() { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); { Shape input_shape = ShapeUtil::MakeShapeWithType({1, 2, 5}); Shape filter_shape = ShapeUtil::MakeShapeWithType({1, 2, 2}); @@ -300,7 +300,7 @@ TYPED_TEST_CASE(Convolve1D_1x2x5_1x2x2_WithRHSDilation, TestTypes); TYPED_TEST(Convolve1D_1x2x5_1x2x2_WithRHSDilation, Types) { this->RunTest(); } XLA_TEST_F(ConvolutionTest, Convolve1D_1x2x5_1x2x2_WithLHSDilation) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); { Shape input_shape = ShapeUtil::MakeShape(F32, {1, 2, 5}); Shape filter_shape = ShapeUtil::MakeShape(F32, {1, 2, 2}); @@ -331,7 +331,7 @@ XLA_TEST_F(ConvolutionTest, Convolve1D_1x2x5_1x2x2_WithLHSDilation) { } XLA_TEST_F(ConvolutionTest, Convolve1D_1x2x5_1x2x2_WithLHSAndRHSDilation) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); { Shape input_shape = ShapeUtil::MakeShape(F32, {1, 2, 5}); Shape filter_shape = ShapeUtil::MakeShape(F32, {1, 2, 2}); @@ -365,7 +365,7 @@ template class Convolve1D_1x2x5_1x2x2_WithPadding : public ConvolutionTest { public: void RunTest() { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); { Shape input_shape = ShapeUtil::MakeShapeWithType({1, 2, 5}); Shape filter_shape = ShapeUtil::MakeShapeWithType({1, 2, 2}); @@ -402,7 +402,7 @@ TYPED_TEST_CASE(Convolve1D_1x2x5_1x2x2_WithPadding, TestTypes); TYPED_TEST(Convolve1D_1x2x5_1x2x2_WithPadding, Types) { this->RunTest(); } XLA_TEST_F(ConvolutionTest, Convolve3D_1x4x2x3x3_2x2x2x3x3_Valid) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); std::vector input_dims = {1, 4, 2, 3, 3}; std::vector filter_dims = {2, 2, 2, 3, 3}; Shape input_shape = ShapeUtil::MakeShape(F32, input_dims); @@ -469,7 +469,7 @@ template class Convolve2D_1x3x3x5_3x3x5x5_Valid : public ConvolutionTest { public: void RunTest() { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); std::vector input_dims = {1, 3, 3, 5}; std::vector filter_dims = {3, 3, 5, 3}; Shape input_shape = ShapeUtil::MakeShapeWithType(input_dims); @@ -537,7 +537,7 @@ XLA_TEST_P(ConvolveWithAndWithoutCanonicalization, execution_options_.mutable_debug_options()->add_xla_disable_hlo_passes( "convolution-canonicalization"); } - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); Shape input_shape = ShapeUtil::MakeShape(F32, {4, 29}); Shape filter_shape = ShapeUtil::MakeShape(F32, {4, 10}); @@ -551,8 +551,7 @@ XLA_TEST_P(ConvolveWithAndWithoutCanonicalization, dnums.set_kernel_output_feature_dimension(1); dnums.set_output_batch_dimension(0); dnums.set_output_feature_dimension(1); - auto conv = builder.ConvWithGeneralDimensions(input, filter, {}, - Padding::kValid, dnums); + builder.ConvWithGeneralDimensions(input, filter, {}, Padding::kValid, dnums); Array2D param0(4, 29); param0.FillUnique(); @@ -563,7 +562,7 @@ XLA_TEST_P(ConvolveWithAndWithoutCanonicalization, Array2D expected_result(29, 10); expected_result.Fill(0); - ComputeAndCompare(&builder, conv, + ComputeAndCompare(&builder, {std::move(*Literal::CreateFromArray(param0)), std::move(*Literal::CreateFromArray(param1))}, error_spec_); @@ -587,7 +586,7 @@ class Convolve1D1WindowTestBase protected: template void TestImpl() { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); int64 input_feature = GetParam().input_feature; int64 output_feature = GetParam().output_feature; int64 batch = GetParam().batch; @@ -724,12 +723,12 @@ INSTANTIATE_TEST_CASE_P( #endif XLA_TEST_F(ConvolutionTest, Convolve_bf16_1x1x1x2_1x1x1x2_Valid) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); Shape input_shape = ShapeUtil::MakeShape(BF16, {1, 1, 1, 2}); Shape filter_shape = ShapeUtil::MakeShape(BF16, {1, 1, 1, 2}); auto input = builder.Parameter(0, input_shape, "input"); auto filter = builder.Parameter(1, filter_shape, "filter"); - auto conv = builder.Conv(input, filter, {1, 1}, Padding::kValid); + builder.Conv(input, filter, {1, 1}, Padding::kValid); Array4D input_data(1, 1, 1, 2); input_data.FillWithYX(Array2D({ @@ -740,7 +739,7 @@ XLA_TEST_F(ConvolutionTest, Convolve_bf16_1x1x1x2_1x1x1x2_Valid) { {bfloat16(5), bfloat16(6)}, })); - ComputeAndCompare(&builder, conv, + ComputeAndCompare(&builder, {std::move(*Literal::CreateFromArray(input_data)), std::move(*Literal::CreateFromArray(filter_data))}, error_spec_); -- GitLab From 15b104a047c1ec8ec07045047d46a300ebc6b2e3 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 15:45:37 -0700 Subject: [PATCH 117/791] Small changes to testing code, plus a new binary to check diff from command line. PiperOrigin-RevId: 192360373 --- tensorflow/contrib/lite/testing/BUILD | 13 +++-- .../contrib/lite/testing/generate_testspec.cc | 49 +++++++++++++------ .../contrib/lite/testing/generate_testspec.h | 2 +- tensorflow/contrib/lite/testing/tf_driver.cc | 9 +++- .../lite/testing/tflite_diff_example_test.cc | 7 ++- .../contrib/lite/testing/tflite_diff_flags.h | 4 +- .../contrib/lite/testing/tflite_diff_util.cc | 10 ++-- .../contrib/lite/testing/tflite_driver.cc | 1 - 8 files changed, 65 insertions(+), 30 deletions(-) diff --git a/tensorflow/contrib/lite/testing/BUILD b/tensorflow/contrib/lite/testing/BUILD index 9f0ba43252..198984e7e7 100644 --- a/tensorflow/contrib/lite/testing/BUILD +++ b/tensorflow/contrib/lite/testing/BUILD @@ -196,7 +196,6 @@ cc_library( cc_library( name = "util", - testonly = 1, hdrs = ["util.h"], ) @@ -251,7 +250,6 @@ cc_test( cc_library( name = "generate_testspec", - testonly = 1, srcs = ["generate_testspec.cc"], hdrs = ["generate_testspec.h"], deps = [ @@ -277,7 +275,6 @@ cc_test( cc_library( name = "tflite_diff_util", - testonly = 1, srcs = ["tflite_diff_util.cc"], hdrs = ["tflite_diff_util.h"], deps = [ @@ -295,7 +292,6 @@ cc_library( cc_library( name = "tflite_diff_flags", - testonly = 1, hdrs = ["tflite_diff_flags.h"], deps = [ ":split", @@ -338,6 +334,15 @@ tf_cc_test( ], ) +cc_binary( + name = "tflite_diff", + srcs = ["tflite_diff_example_test.cc"], + deps = [ + ":tflite_diff_flags", + ":tflite_diff_util", + ], +) + tf_cc_test( name = "generated_examples_zip_test", size = "large", diff --git a/tensorflow/contrib/lite/testing/generate_testspec.cc b/tensorflow/contrib/lite/testing/generate_testspec.cc index eb3deafb69..6580845af4 100644 --- a/tensorflow/contrib/lite/testing/generate_testspec.cc +++ b/tensorflow/contrib/lite/testing/generate_testspec.cc @@ -22,7 +22,22 @@ limitations under the License. namespace tflite { namespace testing { -void GenerateTestSpecFromTensorflowModel( +template +void GenerateCsv(const std::vector& shape, float min, float max, + string* out) { + auto random_float = [](int min, int max) { + static unsigned int seed; + return min + (max - min) * static_cast(rand_r(&seed)) / RAND_MAX; + }; + + std::function random_t = [&](int) { + return static_cast(random_float(min, max)); + }; + std::vector data = GenerateRandomTensor(shape, random_t); + *out = Join(data.data(), data.size(), ","); +} + +bool GenerateTestSpecFromTensorflowModel( std::iostream& stream, const string& tensorflow_model_path, const string& tflite_model_path, const std::vector& input_layer, const std::vector& input_layer_type, @@ -31,12 +46,6 @@ void GenerateTestSpecFromTensorflowModel( CHECK_EQ(input_layer.size(), input_layer_type.size()); CHECK_EQ(input_layer.size(), input_layer_shape.size()); - // Initialize random functions. - static unsigned int seed = 0; - std::function float_rand = [](int idx) { - return static_cast(rand_r(&seed)) / RAND_MAX - 0.5f; - }; - // Generate inputs. std::vector input_values; input_values.resize(input_layer.size()); @@ -46,15 +55,25 @@ void GenerateTestSpecFromTensorflowModel( auto shape = Split(input_layer_shape[i], ","); switch (type) { - case tensorflow::DT_FLOAT: { - const auto& data = GenerateRandomTensor(shape, float_rand); - input_values[i] = Join(data.data(), data.size(), ","); + case tensorflow::DT_FLOAT: + GenerateCsv(shape, -0.5, 0.5, &input_values[i]); + break; + case tensorflow::DT_UINT8: + GenerateCsv(shape, 0, 255, &input_values[i]); + break; + case tensorflow::DT_INT32: + GenerateCsv(shape, -100, 100, &input_values[i]); + break; + case tensorflow::DT_INT64: + GenerateCsv(shape, -100, 100, &input_values[i]); + break; + case tensorflow::DT_BOOL: + GenerateCsv(shape, 0.01, 1.99, &input_values[i]); break; - } default: - - fprintf(stderr, "Unsupported type %d when generating testspec\n", type); - return; + fprintf(stderr, "Unsupported type %d (%s) when generating testspec.\n", + type, input_layer_type[i].c_str()); + return false; } } @@ -82,6 +101,8 @@ void GenerateTestSpecFromTensorflowModel( stream << " output: \"" << runner.ReadOutput(i) << "\"\n"; } stream << "}\n"; + + return true; } } // namespace testing diff --git a/tensorflow/contrib/lite/testing/generate_testspec.h b/tensorflow/contrib/lite/testing/generate_testspec.h index 3529ee709b..6e31a853c3 100644 --- a/tensorflow/contrib/lite/testing/generate_testspec.h +++ b/tensorflow/contrib/lite/testing/generate_testspec.h @@ -34,7 +34,7 @@ namespace testing { // input_layer_type: datatypes of input tensors. Example: float // input_layer_shape: shapes of input tensors, separated by comma. example: // 1,3,4 output_layer: names of output tensors. Example: output -void GenerateTestSpecFromTensorflowModel( +bool GenerateTestSpecFromTensorflowModel( std::iostream& stream, const string& tensorflow_model_path, const string& tflite_model_path, const std::vector& input_layer, const std::vector& input_layer_type, diff --git a/tensorflow/contrib/lite/testing/tf_driver.cc b/tensorflow/contrib/lite/testing/tf_driver.cc index 2c253bb198..7b295875aa 100644 --- a/tensorflow/contrib/lite/testing/tf_driver.cc +++ b/tensorflow/contrib/lite/testing/tf_driver.cc @@ -87,10 +87,9 @@ TfDriver::TfDriver(const std::vector& input_layer, void TfDriver::LoadModel(const string& bin_file_path) { if (!IsValid()) return; - std::cout << std::endl << "Loading model: " << bin_file_path << std::endl; std::ifstream model(bin_file_path); if (model.fail()) { - Invalidate("Failed to find the model"); + Invalidate("Failed to find the model " + bin_file_path); return; } @@ -121,6 +120,10 @@ void TfDriver::SetInput(int id, const string& csv_values) { FillTensorWithData(&tensor, csv_values); break; } + case tensorflow::DT_UINT8: { + FillTensorWithData(&tensor, csv_values); + break; + } default: fprintf(stderr, "Unsupported type %d in SetInput\n", input_types_[id]); Invalidate("Unsupported tensor data type"); @@ -162,6 +165,8 @@ string TfDriver::ReadOutput(int id) { return TensorDataToCsvString(output_tensors_[id]); case tensorflow::DT_INT32: return TensorDataToCsvString(output_tensors_[id]); + case tensorflow::DT_UINT8: + return TensorDataToCsvString(output_tensors_[id]); default: fprintf(stderr, "Unsupported type %d in ResetTensor\n", input_types_[id]); Invalidate("Unsupported tensor data type"); diff --git a/tensorflow/contrib/lite/testing/tflite_diff_example_test.cc b/tensorflow/contrib/lite/testing/tflite_diff_example_test.cc index 3817e68111..5afa0f800c 100644 --- a/tensorflow/contrib/lite/testing/tflite_diff_example_test.cc +++ b/tensorflow/contrib/lite/testing/tflite_diff_example_test.cc @@ -19,10 +19,13 @@ limitations under the License. int main(int argc, char** argv) { ::tflite::testing::DiffOptions options = ::tflite::testing::ParseTfliteDiffFlags(&argc, argv); + if (options.tensorflow_model.empty()) return 1; + int failure_count = 0; for (int i = 0; i < 100; i++) { if (!tflite::testing::RunDiffTest(options)) { - return 1; + ++failure_count; } } - return 0; + fprintf(stderr, "Num errors: %d\n", failure_count); + return failure_count != 0 ? 1 : 0; } diff --git a/tensorflow/contrib/lite/testing/tflite_diff_flags.h b/tensorflow/contrib/lite/testing/tflite_diff_flags.h index 5f1129d501..706108ed73 100644 --- a/tensorflow/contrib/lite/testing/tflite_diff_flags.h +++ b/tensorflow/contrib/lite/testing/tflite_diff_flags.h @@ -51,9 +51,11 @@ DiffOptions ParseTfliteDiffFlags(int* argc, char** argv) { "output_1,output_2"), }; + bool no_inputs = *argc == 1; bool success = tensorflow::Flags::Parse(argc, argv, flags); - if (!success || (*argc == 2 && !strcmp(argv[1], "--helpfull"))) { + if (!success || no_inputs || (*argc == 2 && !strcmp(argv[1], "--helpfull"))) { fprintf(stderr, "%s", tensorflow::Flags::Usage(argv[0], flags).c_str()); + return {}; } return {values.tensorflow_model, diff --git a/tensorflow/contrib/lite/testing/tflite_diff_util.cc b/tensorflow/contrib/lite/testing/tflite_diff_util.cc index 9ef4e1f66c..f601d3752d 100644 --- a/tensorflow/contrib/lite/testing/tflite_diff_util.cc +++ b/tensorflow/contrib/lite/testing/tflite_diff_util.cc @@ -27,13 +27,13 @@ namespace testing { bool RunDiffTest(const DiffOptions& options) { std::stringstream tflite_stream; - GenerateTestSpecFromTensorflowModel( - tflite_stream, options.tensorflow_model, options.tflite_model, - options.input_layer, options.input_layer_type, options.input_layer_shape, - options.output_layer); + if (!GenerateTestSpecFromTensorflowModel( + tflite_stream, options.tensorflow_model, options.tflite_model, + options.input_layer, options.input_layer_type, + options.input_layer_shape, options.output_layer)) + return false; TfLiteDriver tflite_driver(/*use_nnapi=*/true); tflite_driver.LoadModel(options.tflite_model); - std::cout << tflite_stream.str(); return tflite::testing::ParseAndRunTests(&tflite_stream, &tflite_driver); } } // namespace testing diff --git a/tensorflow/contrib/lite/testing/tflite_driver.cc b/tensorflow/contrib/lite/testing/tflite_driver.cc index c399f4f2b7..3764bab035 100644 --- a/tensorflow/contrib/lite/testing/tflite_driver.cc +++ b/tensorflow/contrib/lite/testing/tflite_driver.cc @@ -143,7 +143,6 @@ void TfLiteDriver::AllocateTensors() { void TfLiteDriver::LoadModel(const string& bin_file_path) { if (!IsValid()) return; - std::cout << std::endl << "Loading model: " << bin_file_path << std::endl; model_ = FlatBufferModel::BuildFromFile(GetFullPath(bin_file_path).c_str()); if (!model_) { -- GitLab From 21e1bd6fcd671f41858fca47306e07c76ada7e9a Mon Sep 17 00:00:00 2001 From: Akshay Agrawal Date: Tue, 10 Apr 2018 15:48:15 -0700 Subject: [PATCH 118/791] In `get_variable`, nest the choice to use `ResourceVariable` under an `init_scope`. This makes sure that, when executing eagerly, calls to `get_variable` in a `defun`-compiled function retrieve `ResourceVariable`s instead of `Variables`. PiperOrigin-RevId: 192360775 --- tensorflow/python/kernel_tests/BUILD | 2 + .../kernel_tests/variable_scope_test.py | 118 +++++++++++------- tensorflow/python/ops/variable_scope.py | 12 +- 3 files changed, 89 insertions(+), 43 deletions(-) diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index 3033b48977..1827a26902 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -1029,12 +1029,14 @@ tf_py_test( "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:framework_test_lib", "//tensorflow/python:init_ops", + "//tensorflow/python:layers", "//tensorflow/python:math_ops", "//tensorflow/python:variable_scope", "//tensorflow/python:resource_variable_ops", "//tensorflow/python:state_ops", "//tensorflow/python:variables", "//tensorflow/python/eager:context", + "//tensorflow/python/eager:function", ], tags = ["no_windows"], ) diff --git a/tensorflow/python/kernel_tests/variable_scope_test.py b/tensorflow/python/kernel_tests/variable_scope_test.py index 86ab9fbb70..51aa671098 100644 --- a/tensorflow/python/kernel_tests/variable_scope_test.py +++ b/tensorflow/python/kernel_tests/variable_scope_test.py @@ -24,11 +24,13 @@ import threading import numpy from tensorflow.python.eager import context +from tensorflow.python.eager import function from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import test_util +from tensorflow.python.layers import core as core_layers from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import init_ops @@ -118,6 +120,16 @@ class VariableScopeTest(test.TestCase): w = variable_scope.get_variable("w", []) self.assertEqual(w.dtype.base_dtype, dtypes.float16) + def testGetVariableInGraphNestedUnderEagerContext(self): + with context.eager_mode(): + + @function.defun + def f(): + v = variable_scope.get_variable("should_be_resource", []) + self.assertEqual(type(v), resource_variable_ops.ResourceVariable) + + f() + def testEagerVariableStore(self): with context.eager_mode(): store = variable_scope.EagerVariableStore() @@ -156,6 +168,28 @@ class VariableScopeTest(test.TestCase): for v in new_store.variables(): self.assertEqual(v.numpy(), 1) + def testEagerVariableStoreWithEagerDefun(self): + with context.eager_mode(): + + @function.defun + def f(): + x = constant_op.constant([[2.0]]) + d1 = core_layers.Dense( + 1, name="my_dense", kernel_initializer=init_ops.ones_initializer()) + _ = d1(x) # create variables + self.assertEqual(len(d1.variables), 2) + v1, v2 = d1.variables + d2 = core_layers.Dense( + 1, + name="my_dense", + kernel_initializer=init_ops.ones_initializer(), + _reuse=True) + _ = d2(x) + self.assertEqual(len(d2.variables), 2) + v3, v4 = d2.variables + self.assertAllEqual([v1, v2], [v3, v4]) + f() + @test_util.run_in_graph_and_eager_modes() def testInitFromNonTensorValue(self): v = variable_scope.get_variable("v4", initializer=4, dtype=dtypes.int32) @@ -209,15 +243,15 @@ class VariableScopeTest(test.TestCase): with variable_scope.variable_scope("not_cached", caching_device=""): v2_not_cached = variable_scope.get_variable("v", []) - self.assertFalse(v2_not_cached.value().device.startswith( - caching_device)) + self.assertFalse( + v2_not_cached.value().device.startswith(caching_device)) with variable_scope.variable_scope( "not_cached_identity_device", caching_device=lambda op: op.device): v2_identity_device = variable_scope.get_variable("v", []) - self.assertFalse(v2_identity_device.value().device.startswith( - caching_device)) + self.assertFalse( + v2_identity_device.value().device.startswith(caching_device)) with variable_scope.variable_scope("we_will_do_it_live") as vs_live: vs_live.set_caching_device("/job:live") @@ -484,15 +518,19 @@ class VariableScopeTest(test.TestCase): def testVarScopeGetOrCreateReuse(self): with self.test_session(): + def test_value(value): x = constant_op.constant(value) - with variable_scope.variable_scope("testVarScopeGetOrCreateReuse_bar", - reuse=variable_scope.AUTO_REUSE): + with variable_scope.variable_scope( + "testVarScopeGetOrCreateReuse_bar", + reuse=variable_scope.AUTO_REUSE): _ = state_ops.assign(variable_scope.get_variable("var", []), x) - with variable_scope.variable_scope("testVarScopeGetOrCreateReuse_bar", - reuse=variable_scope.AUTO_REUSE): + with variable_scope.variable_scope( + "testVarScopeGetOrCreateReuse_bar", + reuse=variable_scope.AUTO_REUSE): _ = variable_scope.get_variable("var", []) self.assertEqual(value, x.eval()) + test_value(42.) # Variable is created. test_value(13.) # Variable is reused hereafter. test_value(17.) @@ -551,19 +589,16 @@ class VariableScopeTest(test.TestCase): with variable_scope.variable_scope("default") as default: with variable_scope.variable_scope(None, "layer"): self.assertEqual( - variable_scope.get_variable("w", []).name, - "default/layer/w:0") + variable_scope.get_variable("w", []).name, "default/layer/w:0") with variable_scope.variable_scope(None, "layer"): self.assertEqual( - variable_scope.get_variable("w", []).name, - "default/layer_1/w:0") + variable_scope.get_variable("w", []).name, "default/layer_1/w:0") with variable_scope.variable_scope(default): pass # No matter the jump in the middle, unique numbering continues. with variable_scope.variable_scope(None, "layer"): self.assertEqual( - variable_scope.get_variable("w", []).name, - "default/layer_2/w:0") + variable_scope.get_variable("w", []).name, "default/layer_2/w:0") def testVarOpScopeReuse(self): with self.test_session(): @@ -935,12 +970,12 @@ class VariableScopeTest(test.TestCase): def testGetCollection(self): with self.test_session(): _ = variable_scope.get_variable("testGetCollection_a", []) - _ = variable_scope.get_variable("testGetCollection_b", [], - trainable=False) + _ = variable_scope.get_variable( + "testGetCollection_b", [], trainable=False) with variable_scope.variable_scope("testGetCollection_foo_") as scope1: _ = variable_scope.get_variable("testGetCollection_a", []) - _ = variable_scope.get_variable("testGetCollection_b", [], - trainable=False) + _ = variable_scope.get_variable( + "testGetCollection_b", [], trainable=False) self.assertEqual([ v.name for v in scope1.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES) @@ -954,8 +989,8 @@ class VariableScopeTest(test.TestCase): ]) with variable_scope.variable_scope("testGetCollection_foo") as scope2: _ = variable_scope.get_variable("testGetCollection_a", []) - _ = variable_scope.get_variable("testGetCollection_b", [], - trainable=False) + _ = variable_scope.get_variable( + "testGetCollection_b", [], trainable=False) self.assertEqual([ v.name for v in scope2.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES) @@ -992,22 +1027,22 @@ class VariableScopeTest(test.TestCase): with variable_scope.variable_scope( "testGetTrainableVariables_foo") as scope: _ = variable_scope.get_variable("testGetTrainableVariables_b", []) - _ = variable_scope.get_variable("testGetTrainableVariables_c", [], - trainable=False) - self.assertEqual([v.name - for v in scope.trainable_variables()], - ["testGetTrainableVariables_foo/" - "testGetTrainableVariables_b:0"]) + _ = variable_scope.get_variable( + "testGetTrainableVariables_c", [], trainable=False) + self.assertEqual( + [v.name for v in scope.trainable_variables()], + ["testGetTrainableVariables_foo/" + "testGetTrainableVariables_b:0"]) def testGetGlobalVariables(self): with self.test_session(): _ = variable_scope.get_variable("testGetGlobalVariables_a", []) with variable_scope.variable_scope("testGetGlobalVariables_foo") as scope: _ = variable_scope.get_variable("testGetGlobalVariables_b", []) - self.assertEqual([v.name - for v in scope.global_variables()], - ["testGetGlobalVariables_foo/" - "testGetGlobalVariables_b:0"]) + self.assertEqual( + [v.name for v in scope.global_variables()], + ["testGetGlobalVariables_foo/" + "testGetGlobalVariables_b:0"]) def testGetLocalVariables(self): with self.test_session(): @@ -1016,10 +1051,8 @@ class VariableScopeTest(test.TestCase): with variable_scope.variable_scope("foo") as scope: _ = variable_scope.get_variable( "b", [], collections=[ops.GraphKeys.LOCAL_VARIABLES]) - _ = variable_scope.get_variable( - "c", []) - self.assertEqual([v.name - for v in scope.local_variables()], ["foo/b:0"]) + _ = variable_scope.get_variable("c", []) + self.assertEqual([v.name for v in scope.local_variables()], ["foo/b:0"]) def testGetVariableWithRefDtype(self): v = variable_scope.get_variable("v", shape=[3, 4], dtype=dtypes.float32) @@ -1242,10 +1275,8 @@ class VariableScopeWithCustomGetterTest(test.TestCase): with ops.name_scope("prod_getter"): return g_0 * g_1 - with variable_scope.variable_scope( - "prod_scope", custom_getter=prod_getter): - with variable_scope.variable_scope( - "sum_scope", custom_getter=sum_getter): + with variable_scope.variable_scope("prod_scope", custom_getter=prod_getter): + with variable_scope.variable_scope("sum_scope", custom_getter=sum_getter): with variable_scope.variable_scope( "inner_sum_scope", custom_getter=sum_getter): # take sums of sums of products @@ -1270,9 +1301,8 @@ class VariableScopeWithCustomGetterTest(test.TestCase): np_vars, np_v = sess.run([true_vars, v]) # take products of sums of products self.assertAllClose( - np_v, - (((np_vars[0] * np_vars[1]) + (np_vars[2] * np_vars[3])) - + ((np_vars[4] * np_vars[5]) + (np_vars[6] * np_vars[7])))) + np_v, (((np_vars[0] * np_vars[1]) + (np_vars[2] * np_vars[3])) + ( + (np_vars[4] * np_vars[5]) + (np_vars[6] * np_vars[7])))) def testVariableCreator(self): @@ -1368,7 +1398,11 @@ class VariableScopeMultithreadedTest(test.TestCase): graph = ops.get_default_graph() threads = [ - threading.Thread(target=thread_fn, args=(i, graph,)) for i in range(2)] + threading.Thread(target=thread_fn, args=( + i, + graph, + )) for i in range(2) + ] threads[0].start() # Allow thread 0 to finish before starting thread 1. diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py index e33085ba62..ba213ef884 100644 --- a/tensorflow/python/ops/variable_scope.py +++ b/tensorflow/python/ops/variable_scope.py @@ -307,6 +307,17 @@ class _VariableStore(object): raise ValueError( "Passed a custom_getter which is not callable: %s" % custom_getter) + with ops.init_scope(): + if context.executing_eagerly(): + # Variable creation and initialization takes place in `init_scope`s; + # as such, if an `init_scope` lifts us into the eager context, then we + # need to use `ResourceVariable`s. + use_resource = True + + # Note that it's fine to reuse eager variables whose initialization was + # lifted from a function-building graph into the eager context (that's why + # the following clause is not wrapped in an `init_scope`); lifted variables + # are tracked by the graph's `VariableStore`. if context.executing_eagerly(): if not self._store_eager_variables and reuse: raise RuntimeError( @@ -315,7 +326,6 @@ class _VariableStore(object): " EagerVariableStore for example usage.") if self._store_eager_variables: reuse = AUTO_REUSE - use_resource = True # If a *_ref type is passed in an error would be triggered further down the # stack. We prevent this using base_dtype to get a non-ref version of the -- GitLab From 4a2420589da03ed8d1af9fa92073d2973d315ee4 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Tue, 10 Apr 2018 15:49:03 -0700 Subject: [PATCH 119/791] Cleaning up _distributed_apply now the device policy is unnecessary PiperOrigin-RevId: 192360913 --- tensorflow/python/training/optimizer.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py index 75665fc284..46a58a9adf 100644 --- a/tensorflow/python/training/optimizer.py +++ b/tensorflow/python/training/optimizer.py @@ -689,9 +689,7 @@ class Optimizer( # device_policy is set because non-mirrored tensors will be read in # `update_op`. `_resource_apply_dense`, `lr_t`, `beta1_t` and `beta2_t` # is an example. - with ops.name_scope( - "update_" + scope_name), context.context().device_policy( - context.DEVICE_PLACEMENT_SILENT): + with ops.name_scope("update_" + scope_name): return p.update_op(self, g) with ops.name_scope(name, self._name) as name: @@ -707,11 +705,8 @@ class Optimizer( return self._finish(update_ops, "update") non_slot_devices = distribution.non_slot_devices(var_list) - # Device policy is needed because hyperparameter tensors (such as - # AdamOptimizer's beta1_t) need to be copied across devices in Eager. - with context.context().device_policy(context.DEVICE_PLACEMENT_SILENT): - finish_updates = distribution.update_non_slot( - non_slot_devices, finish, self, update_ops) + finish_updates = distribution.update_non_slot( + non_slot_devices, finish, self, update_ops) if global_step is None: apply_updates = distribution.group(finish_updates, name=name) else: -- GitLab From 47d72205f3c58d31bfec52eb331e89edc562106c Mon Sep 17 00:00:00 2001 From: Mingxing Tan Date: Tue, 10 Apr 2018 15:59:39 -0700 Subject: [PATCH 120/791] Allow passing allow_custom_ops for toco_convert. PiperOrigin-RevId: 192362688 --- tensorflow/contrib/lite/python/lite.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/lite/python/lite.py b/tensorflow/contrib/lite/python/lite.py index ed6dd036f9..cf50f9d4d6 100644 --- a/tensorflow/contrib/lite/python/lite.py +++ b/tensorflow/contrib/lite/python/lite.py @@ -145,7 +145,8 @@ def toco_convert(input_data, input_format=TENSORFLOW_GRAPHDEF, output_format=TFLITE, quantized_input_stats=None, - drop_control_dependency=True): + drop_control_dependency=True, + allow_custom_ops=None): """Convert a model using TOCO from `input_format` to `output_format`. Typically this is to convert from TensorFlow GraphDef to TFLite, in which @@ -178,9 +179,12 @@ def toco_convert(input_data, toco = _toco_flags_pb2.TocoFlags() toco.input_format = input_format toco.output_format = output_format + toco.inference_type = inference_type toco.drop_control_dependency = drop_control_dependency + if allow_custom_ops is not None: + toco.allow_custom_ops = allow_custom_ops + model = _model_flags_pb2.ModelFlags() - toco.inference_type = inference_type for idx, input_tensor in enumerate(input_tensors): if input_tensor.dtype == _dtypes.float32: tflite_input_type = FLOAT -- GitLab From 9846c26ddd2b163ead837b0e1150ab385f2e20b6 Mon Sep 17 00:00:00 2001 From: Amit Patankar Date: Tue, 10 Apr 2018 16:10:13 -0700 Subject: [PATCH 121/791] Updating the sed command for docker parameterized build. --- tensorflow/tools/docker/parameterized_docker_build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/tools/docker/parameterized_docker_build.sh b/tensorflow/tools/docker/parameterized_docker_build.sh index b4fba5b8f5..05de25f2cb 100755 --- a/tensorflow/tools/docker/parameterized_docker_build.sh +++ b/tensorflow/tools/docker/parameterized_docker_build.sh @@ -284,7 +284,7 @@ if [[ "${TF_DOCKER_BUILD_IS_DEVEL}" == "no" ]]; then if sed -i -e 's/python /python3 /g' "${DOCKERFILE}" && \ sed -i -e 's/python-dev/python3-dev/g' "${DOCKERFILE}" && \ sed -i -e 's/pip /pip3 /g' "${DOCKERFILE}" && \ - sed -i -e 's^# RUN ln -s /usr/bin/python3 /usr/bin/python#^RUN ln -s /usr/bin/python3 /usr/bin/python^' "${DOCKERFILE}" + sed -i -e 's^# RUN ln -s -f /usr/bin/python3 /usr/bin/python#^RUN ln -s -f /usr/bin/python3 /usr/bin/python^' "${DOCKERFILE}" then echo "Modified Dockerfile for python version "\ "${TF_DOCKER_BUILD_PYTHON_VERSION} at: ${DOCKERFILE}" @@ -306,7 +306,7 @@ else sed -i -e 's^/tmp/pip^/tmp/pip3^g' "${DOCKERFILE}" && \ sed -i -e 's/pip /pip3 /g' "${DOCKERFILE}" && \ sed -i -e 's/ENV CI_BUILD_PYTHON python/ENV CI_BUILD_PYTHON python3/g' "${DOCKERFILE}" && \ - sed -i -e 's^# RUN ln -s /usr/bin/python3 /usr/bin/python#^RUN ln -s /usr/bin/python3 /usr/bin/python^' "${DOCKERFILE}" + sed -i -e 's^# RUN ln -s -f /usr/bin/python3 /usr/bin/python#^RUN ln -s -f /usr/bin/python3 /usr/bin/python^' "${DOCKERFILE}" then echo "Modified Dockerfile further for python version ${TF_DOCKER_BUILD_PYTHON_VERSION} at: ${DOCKERFILE}" else -- GitLab From dc8aa019ba27d65789bcecbc776d1ccc9359c011 Mon Sep 17 00:00:00 2001 From: Francois Chollet Date: Tue, 10 Apr 2018 16:11:38 -0700 Subject: [PATCH 122/791] Fix `nn` module RNN namespace issues. PiperOrigin-RevId: 192364808 --- tensorflow/python/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py index da836aca6f..13f8420a67 100644 --- a/tensorflow/python/__init__.py +++ b/tensorflow/python/__init__.py @@ -157,6 +157,9 @@ from tensorflow.python.ops import rnn_cell # Required due to `rnn` and `rnn_cell` not being imported in `nn` directly # (due to a circular dependency issue: rnn depends on layers). nn.dynamic_rnn = rnn.dynamic_rnn +nn.static_rnn = rnn.static_rnn +nn.raw_rnn = rnn.raw_rnn +nn.bidirectional_dynamic_rnn = rnn.bidirectional_dynamic_rnn nn.rnn_cell = rnn_cell # Symbols whitelisted for export without documentation. -- GitLab From 2891e0930eba15c7f27b0ab5732554e6b2c474d5 Mon Sep 17 00:00:00 2001 From: Chris Leary Date: Tue, 10 Apr 2018 16:12:19 -0700 Subject: [PATCH 123/791] [XLA] GRPC service definition. PiperOrigin-RevId: 192364932 --- tensorflow/compiler/xla/rpc/BUILD | 79 ++++++ .../compiler/xla/rpc/grpc_client_test.cc | 109 ++++++++ tensorflow/compiler/xla/rpc/grpc_service.cc | 192 ++++++++++++++ tensorflow/compiler/xla/rpc/grpc_service.h | 126 +++++++++ .../compiler/xla/rpc/grpc_service_main.cc | 62 +++++ tensorflow/compiler/xla/rpc/grpc_stub.cc | 244 ++++++++++++++++++ tensorflow/compiler/xla/rpc/grpc_stub.h | 141 ++++++++++ tensorflow/compiler/xla/rpc/xla_service.proto | 225 ++++++++++++++++ tensorflow/compiler/xla/xla.bzl | 13 +- .../core/platform/default/build_config.bzl | 5 + 10 files changed, 1194 insertions(+), 2 deletions(-) create mode 100644 tensorflow/compiler/xla/rpc/BUILD create mode 100644 tensorflow/compiler/xla/rpc/grpc_client_test.cc create mode 100644 tensorflow/compiler/xla/rpc/grpc_service.cc create mode 100644 tensorflow/compiler/xla/rpc/grpc_service.h create mode 100644 tensorflow/compiler/xla/rpc/grpc_service_main.cc create mode 100644 tensorflow/compiler/xla/rpc/grpc_stub.cc create mode 100644 tensorflow/compiler/xla/rpc/grpc_stub.h create mode 100644 tensorflow/compiler/xla/rpc/xla_service.proto diff --git a/tensorflow/compiler/xla/rpc/BUILD b/tensorflow/compiler/xla/rpc/BUILD new file mode 100644 index 0000000000..977f863787 --- /dev/null +++ b/tensorflow/compiler/xla/rpc/BUILD @@ -0,0 +1,79 @@ +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//tensorflow:internal"]) + +load("//tensorflow:tensorflow.bzl", "tf_cc_test") +load("//tensorflow:tensorflow.bzl", "tf_cc_binary") +load( + "//tensorflow/compiler/xla:xla.bzl", + "xla_proto_library", + "xla_py_grpc_library", +) + +xla_proto_library( + name = "xla_service_proto", + srcs = ["xla_service.proto"], + use_grpc_plugin = True, + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/compiler/xla:xla_data_proto", + "//tensorflow/compiler/xla:xla_proto", + ], +) + +cc_library( + name = "grpc_stub", + srcs = ["grpc_stub.cc"], + hdrs = ["grpc_stub.h"], + deps = [ + ":xla_service_proto", + "//tensorflow/compiler/xla:service_interface", + "//tensorflow/compiler/xla:xla_data_proto", + "//tensorflow/core:lib", + "//tensorflow/core/distributed_runtime/rpc:grpc_util", + ], +) + +tf_cc_binary( + name = "grpc_service_main_cpu", + srcs = ["grpc_service_main.cc"], + deps = [ + ":grpc_service", + "//tensorflow/compiler/xla/service:cpu_plugin", + "//tensorflow/core:framework_internal", + "//tensorflow/core:lib", + "@grpc//:grpc++_unsecure", + ], +) + +tf_cc_test( + name = "grpc_client_test", + srcs = ["grpc_client_test.cc"], + data = [ + "//tensorflow/compiler/xla/rpc:grpc_service_main_cpu", + ], + deps = [ + ":grpc_stub", + "//tensorflow/compiler/xla/client", + "//tensorflow/compiler/xla/client:computation_builder", + "//tensorflow/compiler/xla/tests:literal_test_util", + "//tensorflow/core:framework_internal", + "//tensorflow/core:lib", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "@grpc//:grpc++_unsecure", + ], +) + +cc_library( + name = "grpc_service", + srcs = ["grpc_service.cc"], + hdrs = ["grpc_service.h"], + deps = [ + ":xla_service_proto", + "//tensorflow/compiler/xla/service", + "//tensorflow/compiler/xla/service:platform_util", + "//tensorflow/core/distributed_runtime/rpc:grpc_util", + "@grpc//:grpc++_unsecure", + ], +) diff --git a/tensorflow/compiler/xla/rpc/grpc_client_test.cc b/tensorflow/compiler/xla/rpc/grpc_client_test.cc new file mode 100644 index 0000000000..b559ee4b5a --- /dev/null +++ b/tensorflow/compiler/xla/rpc/grpc_client_test.cc @@ -0,0 +1,109 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Simple C++ test to exercise the GRPC capabilities of XLA. +// +// Launches an RPC service in a subprocess and connects to it over a socket +// using an RPCStub. +#include +#include + +#include "grpc++/create_channel.h" +#include "grpc++/security/credentials.h" + +#include "tensorflow/compiler/xla/client/client.h" +#include "tensorflow/compiler/xla/client/computation_builder.h" +#include "tensorflow/compiler/xla/rpc/grpc_stub.h" +#include "tensorflow/compiler/xla/tests/literal_test_util.h" +#include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/lib/strings/stringprintf.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/net.h" +#include "tensorflow/core/platform/subprocess.h" +#include "tensorflow/core/platform/test.h" + +namespace xla { +namespace { + +class GRPCClientTestBase : public ::testing::Test { + protected: + GRPCClientTestBase() { + string test_srcdir = tensorflow::testing::TensorFlowSrcRoot(); + string service_main_path = tensorflow::io::JoinPath( + test_srcdir, "compiler/xla/rpc/grpc_service_main_cpu"); + int port = tensorflow::internal::PickUnusedPortOrDie(); + subprocess_.SetProgram( + service_main_path, + {service_main_path, tensorflow::strings::Printf("--port=%d", port)}); + subprocess_.SetChannelAction(tensorflow::CHAN_STDOUT, + tensorflow::ACTION_DUPPARENT); + subprocess_.SetChannelAction(tensorflow::CHAN_STDERR, + tensorflow::ACTION_DUPPARENT); + CHECK(subprocess_.Start()); + LOG(INFO) << "Launched subprocess"; + + auto channel = + ::grpc::CreateChannel(tensorflow::strings::Printf("localhost:%d", port), + ::grpc::InsecureChannelCredentials()); + channel->WaitForConnected(gpr_time_add( + gpr_now(GPR_CLOCK_REALTIME), gpr_time_from_seconds(10, GPR_TIMESPAN))); + LOG(INFO) << "Channel to server is connected on port " << port; + + xla_service_ = grpc::XlaService::NewStub(channel); + stub_.reset(new GRPCStub(xla_service_.get())); + client_.reset(new Client(stub_.get())); + } + + ~GRPCClientTestBase() override { + LOG(INFO) << "Killing subprocess"; + subprocess_.Kill(SIGKILL); + } + + tensorflow::SubProcess subprocess_; + std::unique_ptr xla_service_; + std::unique_ptr stub_; + std::unique_ptr client_; +}; + +TEST_F(GRPCClientTestBase, ItsAlive) { + ASSERT_NE(xla_service_, nullptr); + ASSERT_NE(stub_, nullptr); + ASSERT_NE(client_, nullptr); +} + +TEST_F(GRPCClientTestBase, AxpyTenValues) { + ComputationBuilder builder(client_.get(), "axpy_10"); + auto alpha = builder.ConstantR0(3.1415926535); + auto x = builder.ConstantR1( + {-1.0, 1.0, 2.0, -2.0, -3.0, 3.0, 4.0, -4.0, -5.0, 5.0}); + auto y = builder.ConstantR1( + {5.0, -5.0, -4.0, 4.0, 3.0, -3.0, -2.0, 2.0, 1.0, -1.0}); + auto ax = builder.Mul(alpha, x); + auto axpy = builder.Add(ax, y); + + std::vector expected = { + 1.85840735, -1.85840735, 2.28318531, -2.28318531, -6.42477796, + 6.42477796, 10.56637061, -10.56637061, -14.70796327, 14.70796327}; + std::unique_ptr expected_literal = + Literal::CreateR1(expected); + TF_ASSERT_OK_AND_ASSIGN(auto computation, builder.Build()); + TF_ASSERT_OK_AND_ASSIGN(auto result_literal, client_->ExecuteAndTransfer( + computation, {}, nullptr)); + LiteralTestUtil::ExpectNear(*expected_literal, *result_literal, + ErrorSpec(0.0001)); +} + +} // namespace +} // namespace xla diff --git a/tensorflow/compiler/xla/rpc/grpc_service.cc b/tensorflow/compiler/xla/rpc/grpc_service.cc new file mode 100644 index 0000000000..414829d6e7 --- /dev/null +++ b/tensorflow/compiler/xla/rpc/grpc_service.cc @@ -0,0 +1,192 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/rpc/grpc_service.h" +#include "tensorflow/compiler/xla/service/platform_util.h" +#include "tensorflow/core/distributed_runtime/rpc/grpc_util.h" + +namespace xla { + +/* static */ StatusOr> GRPCService::NewService( + perftools::gputools::Platform* platform) { + std::unique_ptr grpc_service(new GRPCService()); + TF_ASSIGN_OR_RETURN(grpc_service->service_, + ::xla::Service::NewService(platform)); + return std::move(grpc_service); +} + +::grpc::Status DelegateRPC(std::function op) { + tensorflow::Status s = op(); + return tensorflow::ToGrpcStatus(s); +} + +::grpc::Status GRPCService::Computation(::grpc::ServerContext* context, + const ComputationRequest* arg, + ComputationResponse* result) { + return DelegateRPC( + [this, arg, result]() { return service_->Computation(arg, result); }); +} + +::grpc::Status GRPCService::CreateOp(::grpc::ServerContext* context, + const OpRequest* arg, OpResponse* result) { + return DelegateRPC( + [this, arg, result]() { return service_->Op(arg, result); }); +} + +::grpc::Status GRPCService::Unregister(::grpc::ServerContext* context, + const UnregisterRequest* arg, + UnregisterResponse* result) { + return DelegateRPC( + [this, arg, result]() { return service_->Unregister(arg, result); }); +} + +::grpc::Status GRPCService::DeconstructTuple(::grpc::ServerContext* context, + const DeconstructTupleRequest* arg, + DeconstructTupleResponse* result) { + return DelegateRPC([this, arg, result]() { + return service_->DeconstructTuple(arg, result); + }); +} + +::grpc::Status GRPCService::SetReturnValue(::grpc::ServerContext* context, + const SetReturnValueRequest* arg, + SetReturnValueResponse* results) { + return DelegateRPC([this, arg, results]() { + return service_->SetReturnValue(arg, results); + }); +} + +::grpc::Status GRPCService::Execute(::grpc::ServerContext* context, + const ExecuteRequest* arg, + ExecuteResponse* result) { + return DelegateRPC( + [this, arg, result]() { return service_->Execute(arg, result); }); +} + +::grpc::Status GRPCService::ExecuteAsync(::grpc::ServerContext* context, + const ExecuteAsyncRequest* arg, + ExecuteAsyncResponse* result) { + return DelegateRPC( + [this, arg, result]() { return service_->ExecuteAsync(arg, result); }); +} + +::grpc::Status GRPCService::WaitForExecution(::grpc::ServerContext* context, + const WaitForExecutionRequest* arg, + WaitForExecutionResponse* result) { + return DelegateRPC([this, arg, result]() { + return service_->WaitForExecution(arg, result); + }); +} + +::grpc::Status GRPCService::TransferToClient(::grpc::ServerContext* context, + const TransferToClientRequest* arg, + TransferToClientResponse* result) { + return DelegateRPC([this, arg, result]() { + return service_->TransferToClient(arg, result); + }); +} + +::grpc::Status GRPCService::TransferToServer(::grpc::ServerContext* context, + const TransferToServerRequest* arg, + TransferToServerResponse* result) { + return DelegateRPC([this, arg, result]() { + return service_->TransferToServer(arg, result); + }); +} + +::grpc::Status GRPCService::TransferToInfeed(::grpc::ServerContext* context, + const TransferToInfeedRequest* arg, + TransferToInfeedResponse* result) { + return DelegateRPC([this, arg, result]() { + return service_->TransferToInfeed(arg, result); + }); +} + +::grpc::Status GRPCService::TransferFromOutfeed( + ::grpc::ServerContext* context, const TransferFromOutfeedRequest* arg, + TransferFromOutfeedResponse* result) { + return DelegateRPC([this, arg, result]() { + return service_->TransferFromOutfeed(arg, result); + }); +} + +::grpc::Status GRPCService::ResetDevice(::grpc::ServerContext* context, + const ResetDeviceRequest* arg, + ResetDeviceResponse* result) { + return DelegateRPC( + [this, arg, result]() { return service_->ResetDevice(arg, result); }); +} + +::grpc::Status GRPCService::IsConstant(::grpc::ServerContext* context, + const IsConstantRequest* arg, + IsConstantResponse* result) { + return DelegateRPC( + [this, arg, result]() { return service_->IsConstant(arg, result); }); +} + +::grpc::Status GRPCService::ComputeConstant(::grpc::ServerContext* context, + const ComputeConstantRequest* arg, + ComputeConstantResponse* result) { + return DelegateRPC( + [this, arg, result]() { return service_->ComputeConstant(arg, result); }); +} + +::grpc::Status GRPCService::GetShape(::grpc::ServerContext* context, + const GetShapeRequest* arg, + GetShapeResponse* result) { + return DelegateRPC( + [this, arg, result]() { return service_->GetShape(arg, result); }); +} + +::grpc::Status GRPCService::GetComputationShape( + ::grpc::ServerContext* context, const GetComputationShapeRequest* arg, + GetComputationShapeResponse* result) { + return DelegateRPC([this, arg, result]() { + return service_->GetComputationShape(arg, result); + }); +} + +::grpc::Status GRPCService::GetLocalShape(::grpc::ServerContext* context, + const GetLocalShapeRequest* arg, + GetLocalShapeResponse* result) { + return DelegateRPC( + [this, arg, result]() { return service_->GetLocalShape(arg, result); }); +} + +::grpc::Status GRPCService::GetComputationStats( + ::grpc::ServerContext* context, const ComputationStatsRequest* arg, + ComputationStatsResponse* result) { + return DelegateRPC([this, arg, result]() { + return service_->GetComputationStats(arg, result); + }); +} + +::grpc::Status GRPCService::SnapshotComputation( + ::grpc::ServerContext* context, const SnapshotComputationRequest* arg, + SnapshotComputationResponse* result) { + return DelegateRPC([this, arg, result]() { + return service_->SnapshotComputation(arg, result); + }); +} + +::grpc::Status GRPCService::LoadComputationSnapshot( + ::grpc::ServerContext* context, const LoadComputationSnapshotRequest* arg, + LoadComputationSnapshotResponse* result) { + return DelegateRPC([this, arg, result]() { + return service_->LoadComputationSnapshot(arg, result); + }); +} + +} // namespace xla diff --git a/tensorflow/compiler/xla/rpc/grpc_service.h b/tensorflow/compiler/xla/rpc/grpc_service.h new file mode 100644 index 0000000000..7c9e484517 --- /dev/null +++ b/tensorflow/compiler/xla/rpc/grpc_service.h @@ -0,0 +1,126 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_RPC_GRPC_SERVICE_H_ +#define TENSORFLOW_COMPILER_XLA_RPC_GRPC_SERVICE_H_ + +#include "grpc++/server_context.h" +#include "tensorflow/compiler/xla/rpc/xla_service.grpc.pb.h" +#include "tensorflow/compiler/xla/service/service.h" + +namespace xla { + +// Service implementation which wraps a XLA Service with a GRPC interface. +class GRPCService : public grpc::XlaService::Service { + public: + // Factory for creating a RPCService. The parameter platform is the platform + // that the service should target. If platform is null then the default + // platform is used. + static StatusOr> NewService( + perftools::gputools::Platform* platform = nullptr); + + ::grpc::Status Computation(::grpc::ServerContext* context, + const ComputationRequest* arg, + ComputationResponse* result) override; + + ::grpc::Status CreateOp(::grpc::ServerContext* context, const OpRequest* arg, + OpResponse* result) override; + + ::grpc::Status Unregister(::grpc::ServerContext* context, + const UnregisterRequest* arg, + UnregisterResponse* result) override; + + ::grpc::Status DeconstructTuple(::grpc::ServerContext* context, + const DeconstructTupleRequest* arg, + DeconstructTupleResponse* result) override; + + ::grpc::Status SetReturnValue(::grpc::ServerContext* context, + const SetReturnValueRequest* arg, + SetReturnValueResponse* results) override; + + ::grpc::Status Execute(::grpc::ServerContext* context, + const ExecuteRequest* arg, + ExecuteResponse* result) override; + + ::grpc::Status ExecuteAsync(::grpc::ServerContext* context, + const ExecuteAsyncRequest* arg, + ExecuteAsyncResponse* result) override; + + ::grpc::Status WaitForExecution(::grpc::ServerContext* context, + const WaitForExecutionRequest* arg, + WaitForExecutionResponse* result) override; + + ::grpc::Status TransferToClient(::grpc::ServerContext* context, + const TransferToClientRequest* arg, + TransferToClientResponse* result) override; + + ::grpc::Status TransferToServer(::grpc::ServerContext* context, + const TransferToServerRequest* arg, + TransferToServerResponse* result) override; + + ::grpc::Status TransferToInfeed(::grpc::ServerContext* context, + const TransferToInfeedRequest* arg, + TransferToInfeedResponse* result) override; + + ::grpc::Status TransferFromOutfeed( + ::grpc::ServerContext* context, const TransferFromOutfeedRequest* arg, + TransferFromOutfeedResponse* result) override; + + ::grpc::Status ResetDevice(::grpc::ServerContext* context, + const ResetDeviceRequest* arg, + ResetDeviceResponse* result) override; + + ::grpc::Status IsConstant(::grpc::ServerContext* context, + const IsConstantRequest* arg, + IsConstantResponse* result) override; + + ::grpc::Status ComputeConstant(::grpc::ServerContext* context, + const ComputeConstantRequest* arg, + ComputeConstantResponse* result) override; + + ::grpc::Status GetShape(::grpc::ServerContext* context, + const GetShapeRequest* arg, + GetShapeResponse* result) override; + + ::grpc::Status GetComputationShape( + ::grpc::ServerContext* context, const GetComputationShapeRequest* arg, + GetComputationShapeResponse* result) override; + + ::grpc::Status GetLocalShape(::grpc::ServerContext* context, + const GetLocalShapeRequest* arg, + GetLocalShapeResponse* result) override; + + ::grpc::Status GetComputationStats(::grpc::ServerContext* context, + const ComputationStatsRequest* arg, + ComputationStatsResponse* result) override; + + ::grpc::Status SnapshotComputation( + ::grpc::ServerContext* context, const SnapshotComputationRequest* arg, + SnapshotComputationResponse* result) override; + + ::grpc::Status LoadComputationSnapshot( + ::grpc::ServerContext* context, const LoadComputationSnapshotRequest* arg, + LoadComputationSnapshotResponse* result) override; + + private: + std::unique_ptr<::xla::Service> service_; + + GRPCService() {} + GRPCService(const GRPCService&) = delete; + void operator=(const GRPCService&) = delete; +}; +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_RPC_GRPC_SERVICE_H_ diff --git a/tensorflow/compiler/xla/rpc/grpc_service_main.cc b/tensorflow/compiler/xla/rpc/grpc_service_main.cc new file mode 100644 index 0000000000..e29908ccec --- /dev/null +++ b/tensorflow/compiler/xla/rpc/grpc_service_main.cc @@ -0,0 +1,62 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Basic server binary that exposes a xla::Service through a GRPC interface +// on a configurable port. +#include "grpc++/security/server_credentials.h" +#include "grpc++/server.h" +#include "grpc++/server_builder.h" +#include "tensorflow/compiler/xla/rpc/grpc_service.h" +#include "tensorflow/core/lib/strings/stringprintf.h" +#include "tensorflow/core/platform/init_main.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/util/command_line_flags.h" + +namespace xla { +namespace { + +int RealMain(int argc, char** argv) { + int32 port = 1685; + std::vector flag_list = { + tensorflow::Flag("port", &port, "port to listen on"), + }; + string usage = tensorflow::Flags::Usage(argv[0], flag_list); + bool parsed_values_ok = tensorflow::Flags::Parse(&argc, argv, flag_list); + if (!parsed_values_ok) { + LOG(ERROR) << usage; + return 2; + } + tensorflow::port::InitMain(argv[0], &argc, &argv); + + std::unique_ptr service = + xla::GRPCService::NewService().ConsumeValueOrDie(); + + ::grpc::ServerBuilder builder; + string server_address(tensorflow::strings::Printf("localhost:%d", port)); + + builder.AddListeningPort(server_address, ::grpc::InsecureServerCredentials()); + builder.RegisterService(service.get()); + std::unique_ptr<::grpc::Server> server(builder.BuildAndStart()); + + LOG(INFO) << "Server listening on " << server_address; + server->Wait(); + + return 0; +} + +} // namespace +} // namespace xla + +int main(int argc, char** argv) { return xla::RealMain(argc, argv); } diff --git a/tensorflow/compiler/xla/rpc/grpc_stub.cc b/tensorflow/compiler/xla/rpc/grpc_stub.cc new file mode 100644 index 0000000000..e1f2b0abe3 --- /dev/null +++ b/tensorflow/compiler/xla/rpc/grpc_stub.cc @@ -0,0 +1,244 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/rpc/grpc_stub.h" +#include "tensorflow/core/distributed_runtime/rpc/grpc_util.h" + +namespace xla { + +GRPCStub::~GRPCStub() = default; + +tensorflow::Status MakeRPC( + const std::function<::grpc::Status(::grpc::ClientContext*)>& rpc_method) { + ::grpc::ClientContext context; + ::grpc::Status s = rpc_method(&context); + return tensorflow::FromGrpcStatus(s); +} + +tensorflow::Status GRPCStub::TransferToClient( + const TransferToClientRequest* request, + TransferToClientResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->TransferToClient(context, *request, response); + }); +} + +tensorflow::Status GRPCStub::TransferToServer( + const TransferToServerRequest* request, + TransferToServerResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->TransferToServer(context, *request, response); + }); +} + +tensorflow::Status GRPCStub::TransferToInfeed( + const TransferToInfeedRequest* request, + TransferToInfeedResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->TransferToInfeed(context, *request, response); + }); +} + +tensorflow::Status GRPCStub::TransferFromOutfeed( + const TransferFromOutfeedRequest* request, + TransferFromOutfeedResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->TransferFromOutfeed(context, *request, response); + }); +} + +tensorflow::Status GRPCStub::ResetDevice(const ResetDeviceRequest* request, + ResetDeviceResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->ResetDevice(context, *request, response); + }); +} + +tensorflow::Status GRPCStub::LoadComputationSnapshot( + const LoadComputationSnapshotRequest* request, + LoadComputationSnapshotResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->LoadComputationSnapshot(context, *request, response); + }); +} + +tensorflow::Status GRPCStub::Execute(const ExecuteRequest* request, + ExecuteResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->Execute(context, *request, response); + }); +} + +tensorflow::Status GRPCStub::ExecuteGraph(const ExecuteGraphRequest* request, + ExecuteResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->ExecuteGraph(context, *request, response); + }); +} + +tensorflow::Status GRPCStub::ExecuteParallel( + const ExecuteParallelRequest* request, ExecuteParallelResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->ExecuteParallel(context, *request, response); + }); +} + +tensorflow::Status GRPCStub::ExecuteGraphParallel( + const ExecuteGraphParallelRequest* request, + ExecuteParallelResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->ExecuteGraphParallel(context, *request, response); + }); +} + +tensorflow::Status GRPCStub::ExecuteAsync(const ExecuteAsyncRequest* request, + ExecuteAsyncResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->ExecuteAsync(context, *request, response); + }); +} + +tensorflow::Status GRPCStub::WaitForExecution( + const WaitForExecutionRequest* request, + WaitForExecutionResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->WaitForExecution(context, *request, response); + }); +} + +tensorflow::Status GRPCStub::DeconstructTuple( + const DeconstructTupleRequest* request, + DeconstructTupleResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->DeconstructTuple(context, *request, response); + }); +} + +tensorflow::Status GRPCStub::GetComputationStats( + const ComputationStatsRequest* request, + ComputationStatsResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->GetComputationStats(context, *request, response); + }); +} + +tensorflow::Status GRPCStub::GetComputationGraphStats( + const ComputationGraphStatsRequest* request, + ComputationStatsResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->GetComputationGraphStats(context, *request, response); + }); +} + +tensorflow::Status GRPCStub::GetComputationShape( + const GetComputationShapeRequest* request, + GetComputationShapeResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->GetComputationShape(context, *request, response); + }); +} + +tensorflow::Status GRPCStub::GetShape(const GetShapeRequest* request, + GetShapeResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->GetShape(context, *request, response); + }); +} + +tensorflow::Status GRPCStub::GetDeviceHandles( + const GetDeviceHandlesRequest* request, + GetDeviceHandlesResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->GetDeviceHandles(context, *request, response); + }); +} + +tensorflow::Status GRPCStub::CreateChannelHandle( + const CreateChannelHandleRequest* request, + CreateChannelHandleResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->CreateChannelHandle(context, *request, response); + }); +} + +// Methods used by ComputationBuilder. +tensorflow::Status GRPCStub::Computation(const ComputationRequest* request, + ComputationResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->Computation(context, *request, response); + }); +} + +tensorflow::Status GRPCStub::Op(const OpRequest* request, + OpResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->CreateOp(context, *request, response); + }); +} + +tensorflow::Status GRPCStub::GetLocalShape(const GetLocalShapeRequest* request, + GetLocalShapeResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->GetLocalShape(context, *request, response); + }); +} + +tensorflow::Status GRPCStub::SetReturnValue( + const SetReturnValueRequest* request, SetReturnValueResponse* responses) { + return MakeRPC([this, request, responses](::grpc::ClientContext* context) { + return grpc_stub_->SetReturnValue(context, *request, responses); + }); +} + +tensorflow::Status GRPCStub::IsConstant(const IsConstantRequest* request, + IsConstantResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->IsConstant(context, *request, response); + }); +} + +tensorflow::Status GRPCStub::ComputeConstant( + const ComputeConstantRequest* request, ComputeConstantResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->ComputeConstant(context, *request, response); + }); +} + +tensorflow::Status GRPCStub::ComputeConstantGraph( + const ComputeConstantGraphRequest* request, + ComputeConstantResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->ComputeConstantGraph(context, *request, response); + }); +} + +// Methods used by Computation. +tensorflow::Status GRPCStub::SnapshotComputation( + const SnapshotComputationRequest* request, + SnapshotComputationResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->SnapshotComputation(context, *request, response); + }); +} + +// Methods used by GlobalData. +tensorflow::Status GRPCStub::Unregister(const UnregisterRequest* request, + UnregisterResponse* response) { + return MakeRPC([this, request, response](::grpc::ClientContext* context) { + return grpc_stub_->Unregister(context, *request, response); + }); +} + +} // namespace xla diff --git a/tensorflow/compiler/xla/rpc/grpc_stub.h b/tensorflow/compiler/xla/rpc/grpc_stub.h new file mode 100644 index 0000000000..fd9810d4f1 --- /dev/null +++ b/tensorflow/compiler/xla/rpc/grpc_stub.h @@ -0,0 +1,141 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_RPC_GRPC_STUB_H_ +#define TENSORFLOW_COMPILER_XLA_RPC_GRPC_STUB_H_ + +#include "tensorflow/compiler/xla/rpc/xla_service.grpc.pb.h" +#include "tensorflow/compiler/xla/service_interface.h" +#include "tensorflow/compiler/xla/xla_data.pb.h" +#include "tensorflow/core/platform/macros.h" + +namespace xla { + +class GRPCStub : public ServiceInterface { + public: + explicit GRPCStub(grpc::XlaService::Stub* stub) : grpc_stub_(stub) {} + ~GRPCStub() override; + + tensorflow::Status TransferToClient( + const TransferToClientRequest* arg, + TransferToClientResponse* result) override; + + tensorflow::Status TransferToServer( + const TransferToServerRequest* arg, + TransferToServerResponse* result) override; + + tensorflow::Status TransferToInfeed( + const TransferToInfeedRequest* arg, + TransferToInfeedResponse* result) override; + + tensorflow::Status TransferFromOutfeed( + const TransferFromOutfeedRequest* arg, + TransferFromOutfeedResponse* result) override; + + tensorflow::Status ResetDevice(const ResetDeviceRequest* arg, + ResetDeviceResponse* result) override; + + tensorflow::Status LoadComputationSnapshot( + const LoadComputationSnapshotRequest* request, + LoadComputationSnapshotResponse* result) override; + + tensorflow::Status Execute(const ExecuteRequest* arg, + ExecuteResponse* result) override; + + tensorflow::Status ExecuteGraph(const ExecuteGraphRequest* request, + ExecuteResponse* response) override; + + tensorflow::Status ExecuteParallel(const ExecuteParallelRequest* arg, + ExecuteParallelResponse* result) override; + + tensorflow::Status ExecuteGraphParallel( + const ExecuteGraphParallelRequest* request, + ExecuteParallelResponse* response) override; + + tensorflow::Status ExecuteAsync(const ExecuteAsyncRequest* arg, + ExecuteAsyncResponse* result) override; + + tensorflow::Status WaitForExecution( + const WaitForExecutionRequest* arg, + WaitForExecutionResponse* result) override; + + tensorflow::Status DeconstructTuple( + const DeconstructTupleRequest* arg, + DeconstructTupleResponse* result) override; + + tensorflow::Status GetComputationStats( + const ComputationStatsRequest* arg, + ComputationStatsResponse* result) override; + + tensorflow::Status GetComputationGraphStats( + const ComputationGraphStatsRequest* request, + ComputationStatsResponse* response) override; + + tensorflow::Status GetComputationShape( + const GetComputationShapeRequest* arg, + GetComputationShapeResponse* result) override; + + tensorflow::Status GetShape(const GetShapeRequest* arg, + GetShapeResponse* result) override; + + tensorflow::Status GetDeviceHandles( + const GetDeviceHandlesRequest* arg, + GetDeviceHandlesResponse* result) override; + + tensorflow::Status CreateChannelHandle( + const CreateChannelHandleRequest* arg, + CreateChannelHandleResponse* result) override; + + // Methods used by ComputationBuilder. + tensorflow::Status Computation(const ComputationRequest* arg, + ComputationResponse* result) override; + + tensorflow::Status Op(const OpRequest* arg, OpResponse* result) override; + tensorflow::Status GetLocalShape(const GetLocalShapeRequest* arg, + GetLocalShapeResponse* result) override; + + tensorflow::Status SetReturnValue(const SetReturnValueRequest* arg, + SetReturnValueResponse* results) override; + + tensorflow::Status IsConstant(const IsConstantRequest* arg, + IsConstantResponse* result) override; + + tensorflow::Status ComputeConstant(const ComputeConstantRequest* arg, + ComputeConstantResponse* result) override; + + tensorflow::Status ComputeConstantGraph( + const ComputeConstantGraphRequest* arg, + ComputeConstantResponse* result) override; + + // Methods used by Computation. + tensorflow::Status SnapshotComputation( + const SnapshotComputationRequest* ag, + SnapshotComputationResponse* result) override; + + // Methods used by GlobalData. + tensorflow::Status Unregister(const UnregisterRequest* arg, + UnregisterResponse* result) override; + + grpc::XlaService::Stub* service() { return grpc_stub_; } + + private: + grpc::XlaService::Stub* grpc_stub_; + + TF_DISALLOW_COPY_AND_ASSIGN(GRPCStub); +}; + +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_RPC_GRPC_STUB_H_ diff --git a/tensorflow/compiler/xla/rpc/xla_service.proto b/tensorflow/compiler/xla/rpc/xla_service.proto new file mode 100644 index 0000000000..c47164ee1b --- /dev/null +++ b/tensorflow/compiler/xla/rpc/xla_service.proto @@ -0,0 +1,225 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// XLA service API. +// +// Users 1) build up computations and 2) create allocations via this API. +// Computations are composed of data flowing between arbitrarily-sized +// vector-oriented operations. +// +// Users build up computations using a ComputationHandle, and talk about +// allocations using GlobalDataHandles. +// +// There are currently no checkpointing capabilities or distribution/replication +// guarantees. The service runs on a single machine (e.g. one task) and that is +// its failure domain. +// +// Canonical example of "alpha * X + Y": +// * Make a computation. +// * Add alpha and X and Y as parameters. +// * Request the multiplication of alpha and X. +// * Request the addition of that result and Y. +// +// Then, pass the computation and appropriately shaped inputs to the XLA +// service's Execute method, which provides a result as a GlobalDataHandle. +// +// All data in XLA computations are conceptually immutable. +// +// Note: this API is subject to change / refinement over time -- use the +// provided client libraries to insulate code from changes to this service API. + +syntax = "proto3"; + +import "tensorflow/compiler/xla/xla.proto"; +import "tensorflow/compiler/xla/xla_data.proto"; + +package xla; + +service XlaService { + ///////////////////////// + // Global data requests + + // Unregisters a global allocation. + // + // If the handle given is not currently allocated, a NOT_FOUND status is + // returned. + rpc Unregister(UnregisterRequest) returns (UnregisterResponse) { + } + + // Deconstructs a tuple. Returns a newly created GlobalDataHandle for each + // element in the tuple. + rpc DeconstructTuple(DeconstructTupleRequest) + returns (DeconstructTupleResponse) { + } + + // Unpack requests that a global data handle, with a tuple shape, has global + // data handles created for each of its constituent members. This is the + // equivalent of the "destructuring assignment" present in various programming + // languages. + rpc Unpack(UnpackRequest) returns (UnpackResponse) { + } + + // Requests the shape of the referenced global data. + rpc GetShape(GetShapeRequest) returns (GetShapeResponse) { + } + + // Requests the program shape of the referenced computation. + rpc GetComputationShape(GetComputationShapeRequest) + returns (GetComputationShapeResponse) { + } + + // Requests the statistics of the given computation. + rpc GetComputationStats(ComputationStatsRequest) + returns (ComputationStatsResponse) { + } + + // Requests the statistics of the given computation. + // + // TODO(b/74197823): This is a part of a NOT YET ready refactor. + rpc GetComputationGraphStats(ComputationGraphStatsRequest) + returns (ComputationStatsResponse) { + } + + // Loads a variable number of values with a given element type from ColumnIO. + rpc LoadData(LoadDataRequest) returns (LoadDataResponse) { + } + + // Transfers the given global data to the client in the form of a Literal. + rpc TransferToClient(TransferToClientRequest) + returns (TransferToClientResponse) { + } + + // Transfers the given literal to the server to be stored in a global + // allocation, which is returned. + rpc TransferToServer(TransferToServerRequest) + returns (TransferToServerResponse) { + } + + // Transfers the given literal to the Infeed buffer of the device. + rpc TransferToInfeed(TransferToInfeedRequest) + returns (TransferToInfeedResponse) { + } + + // Transferred literal from the Outfeed buffer of the device. + rpc TransferFromOutfeed(TransferFromOutfeedRequest) + returns (TransferFromOutfeedResponse) { + } + + // Resets the device, clearing all existing state on the device. + rpc ResetDevice(ResetDeviceRequest) returns (ResetDeviceResponse) { + } + + // Tests if an expression is a compile-time constant. + rpc IsConstant(IsConstantRequest) returns (IsConstantResponse) { + } + + // Computes the value of a constant expression. + rpc ComputeConstant(ComputeConstantRequest) + returns (ComputeConstantResponse) { + } + + // Computes the value of a constant expression. The request contains the + // computation graph for the constant expression. + rpc ComputeConstantGraph(ComputeConstantGraphRequest) + returns (ComputeConstantResponse) { + } + + // Retrieves the inferred shape for a value within a computation. + rpc GetLocalShape(GetLocalShapeRequest) returns (GetLocalShapeResponse) { + } + + // Requests one or more device handles from the target. The returned device + // handles can be used to specify the device on which to execute computations + // or transfer data. + rpc GetDeviceHandles(GetDeviceHandlesRequest) + returns (GetDeviceHandlesResponse) { + } + + // Creates a channel handle that can be used to transfer data between + // two computations via a pair of Send and Recv instructions. + rpc CreateChannelHandle(CreateChannelHandleRequest) + returns (CreateChannelHandleResponse) { + } + + // Requests that the referenced computation be specialized for the provided + // arguments for subsequent execution. This permits things such as value + // specialization. + rpc Specialize(SpecializeRequest) returns (SpecializeResponse) { + } + + // Modifies the provided computation so that subsequent executions + // will compute the provided ComputationDataHandle, rather than the + // last expression enqueued on that Computation. + rpc SetReturnValue(SetReturnValueRequest) returns (SetReturnValueResponse) { + } + + // Computation creates a new computation with the given name. + // A unique ComputationHandle is returned. + rpc Computation(ComputationRequest) returns (ComputationResponse) { + } + + // Adds a new op to a computation. + rpc CreateOp(OpRequest) returns (OpResponse) { + } + + // Invokes the provided computation with the provided global data passed as + // immutable arguments. Returns global data output and execution timing. + rpc Execute(ExecuteRequest) returns (ExecuteResponse) { + } + + // Invokes the provided computation with the provided global data passed as + // immutable arguments. The request contains the whole computation graph. + // Returns global data output and execution timing. + rpc ExecuteGraph(ExecuteGraphRequest) returns (ExecuteResponse) { + } + + // Invokes the provided list of computations in parallel with the provided + // global data for each computation. Returns a list of global data output and + // execution timing. + rpc ExecuteParallel(ExecuteParallelRequest) + returns (ExecuteParallelResponse) { + } + + // Invokes the provided list of computations in parallel with the provided + // global data for each computation. Returns a list of global data output and + // execution timing. + // + // TODO(b/74197823): This is a part of a NOT YET ready refactor. + rpc ExecuteGraphParallel(ExecuteGraphParallelRequest) + returns (ExecuteParallelResponse) { + } + + // Invokes the provided computation with the provided global data passed as + // immutable arguments. Returns a handle to the execution. + rpc ExecuteAsync(ExecuteAsyncRequest) returns (ExecuteAsyncResponse) { + } + + // Waits until the given execution (aysnchronously launched) is complete, and + // returns the global data output. + rpc WaitForExecution(WaitForExecutionRequest) + returns (WaitForExecutionResponse) { + } + + // Serializes a computation to proto form, so it can be loaded via + // LoadComputationSnapshot. + rpc SnapshotComputation(SnapshotComputationRequest) + returns (SnapshotComputationResponse) { + } + + // Loads a computation from a captured snapshot. + rpc LoadComputationSnapshot(LoadComputationSnapshotRequest) + returns (LoadComputationSnapshotResponse) { + } +} diff --git a/tensorflow/compiler/xla/xla.bzl b/tensorflow/compiler/xla/xla.bzl index 6b136d333b..1439f1bcc5 100644 --- a/tensorflow/compiler/xla/xla.bzl +++ b/tensorflow/compiler/xla/xla.bzl @@ -6,7 +6,9 @@ load("//tensorflow/core:platform/default/build_config_root.bzl", "if_static") # xla_proto_library() is a convenience wrapper around cc_proto_library. -def xla_proto_library(name, srcs=[], deps=[], visibility=None, testonly=0): +def xla_proto_library(name, srcs=[], deps=[], visibility=None, testonly=0, **kwargs): + if kwargs.get('use_grpc_plugin'): + kwargs['use_grpc_namespace'] = True cc_proto_library(name=name, srcs=srcs, deps=deps, @@ -16,6 +18,13 @@ def xla_proto_library(name, srcs=[], deps=[], visibility=None, testonly=0): ), protoc="@protobuf_archive//:protoc", testonly=testonly, - visibility=visibility,) + visibility=visibility, + **kwargs) + +def xla_py_grpc_library(**kwargs): + # Note: we don't currently define any special targets for Python GRPC in OSS. + _ignore = kwargs + pass + ORC_JIT_MEMORY_MAPPER_TARGETS = [] diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl index e01e076bcf..4cfa25bf66 100644 --- a/tensorflow/core/platform/default/build_config.bzl +++ b/tensorflow/core/platform/default/build_config.bzl @@ -122,6 +122,7 @@ def cc_proto_library( protoc="@protobuf_archive//:protoc", internal_bootstrap_hack=False, use_grpc_plugin=False, + use_grpc_namespace=False, default_header=False, **kargs): """Bazel rule to create a C++ protobuf library from proto source files. @@ -169,8 +170,11 @@ def cc_proto_library( return grpc_cpp_plugin = None + plugin_options = [] if use_grpc_plugin: grpc_cpp_plugin = "//external:grpc_cpp_plugin" + if use_grpc_namespace: + plugin_options = ["services_namespace=grpc"] gen_srcs = _proto_cc_srcs(srcs, use_grpc_plugin) gen_hdrs = _proto_cc_hdrs(srcs, use_grpc_plugin) @@ -184,6 +188,7 @@ def cc_proto_library( protoc=protoc, plugin=grpc_cpp_plugin, plugin_language="grpc", + plugin_options=plugin_options, gen_cc=1, outs=outs, visibility=["//visibility:public"], -- GitLab From bb1dec54a63ad0a5f43208fa7617f090bc5be2e9 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 16:26:05 -0700 Subject: [PATCH 124/791] Supporting FakeQuant num_bits and getting the fake quant op matching tensorflow. PiperOrigin-RevId: 192367307 --- .../contrib/lite/kernels/internal/BUILD | 2 + .../internal/optimized/optimized_ops.h | 79 +++++-------------- .../kernels/internal/quantization_util.cc | 21 +++++ .../lite/kernels/internal/quantization_util.h | 8 ++ .../internal/reference/reference_ops.h | 79 +++++-------------- .../contrib/lite/toco/export_tensorflow.cc | 3 + .../contrib/lite/toco/import_tensorflow.cc | 3 + tensorflow/contrib/lite/toco/model.h | 4 +- .../contrib/lite/toco/tflite/operator.cc | 3 + .../contrib/lite/toco/tflite/operator_test.cc | 2 + 10 files changed, 81 insertions(+), 123 deletions(-) diff --git a/tensorflow/contrib/lite/kernels/internal/BUILD b/tensorflow/contrib/lite/kernels/internal/BUILD index 167c0f1fde..32a0acf888 100644 --- a/tensorflow/contrib/lite/kernels/internal/BUILD +++ b/tensorflow/contrib/lite/kernels/internal/BUILD @@ -154,6 +154,7 @@ cc_library( ], copts = tflite_copts(), deps = [ + ":quantization_util", ":types", ":round", "//third_party/eigen3", @@ -238,6 +239,7 @@ cc_library( "reference/reference_ops.h", ], deps = [ + ":quantization_util", ":round", ":types", "//third_party/eigen3", diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h index 22c0504ad2..5f60b2d6a0 100644 --- a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h +++ b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h @@ -30,6 +30,7 @@ limitations under the License. #include "fixedpoint/fixedpoint.h" #include "public/gemmlowp.h" #include "tensorflow/contrib/lite/kernels/internal/common.h" +#include "tensorflow/contrib/lite/kernels/internal/quantization_util.h" #include "tensorflow/contrib/lite/kernels/internal/round.h" #include "tensorflow/contrib/lite/kernels/internal/types.h" @@ -4750,66 +4751,23 @@ inline void Dequantize(const uint8* input_data, const Dims<4>& input_dims, } inline void FakeQuant(const float* input_data, const Dims<4>& input_dims, - float rmin, float rmax, float* output_data, + float rmin, float rmax, int num_bits, float* output_data, const Dims<4>& output_dims) { gemmlowp::ScopedProfilingLabel label("FakeQuant"); // 0 should always be a representable value. Let's assume that the initial // min,max range contains 0. - TFLITE_DCHECK_LE(rmin, 0.); - TFLITE_DCHECK_GE(rmax, 0.); - - // Determine quantization parameters: zero_point, scale. - using Integer = uint8; - const Integer qmin = std::numeric_limits::min(); - const Integer qmax = std::numeric_limits::max(); - const float qmin_float = qmin; - const float qmax_float = qmax; - int32 zero_point = 0; - float scale = 0.f; - // If rmin==rmax, both must be zero per the above assertion, - // so we are done. - if (rmin != rmax) { - // First determine the scale. - scale = (rmax - rmin) / (qmax_float - qmin_float); - - // Zero-point computation. - // First the initial floating-point computation. The zero-point can be - // determined from solving an affine equation for any known pair - // (real value, corresponding quantized value). - // We know two such pairs: (rmin, qmin) and (rmax, qmax). - // The arithmetic error on the zero point computed from either pair - // will be roughly machine_epsilon * (sum of absolute values of terms) - // so we want to use the variant that adds the smaller terms. - const float zero_point_from_min = qmin_float - rmin / scale; - const float zero_point_from_max = qmax_float - rmax / scale; - const float zero_point_from_min_error = - std::abs(qmin_float) + std::abs(rmin / scale); - const float zero_point_from_max_error = - std::abs(qmax_float) + std::abs(rmax / scale); - - const float zero_point_float = - zero_point_from_min_error < zero_point_from_max_error - ? zero_point_from_min - : zero_point_from_max; - - // Now we need to nudge the zero point to be an integer - // (our zero points are integer, and this is motivated by the requirement - // to be able to represent the real value "0" exactly as a quantized value, - // which is required in multiple places, for example in Im2col with SAME - // padding). - if (zero_point_float < qmin_float) { - zero_point = qmin; - } else if (zero_point_float > qmax_float) { - zero_point = qmax; - } else { - zero_point = static_cast(TfLiteRound(zero_point_float)); - } - // The zero point should always be in the range of quantized value, - // [qmin, qmax]. - TFLITE_DCHECK_GE(zero_point, qmin); - TFLITE_DCHECK_LE(zero_point, qmax); - } + TFLITE_DCHECK_LE(rmin, 0.0f); + TFLITE_DCHECK_GE(rmax, 0.0f); + TFLITE_DCHECK_LT(rmin, rmax); + + // Code matches tensorflow's FakeQuantWithMinMaxArgsFunctor. + int quant_min = 0; + int quant_max = (1 << num_bits) - 1; + float nudged_min, nudged_max, nudged_scale; + NudgeQuantizationRange(rmin, rmax, quant_min, quant_max, &nudged_min, + &nudged_max, &nudged_scale); + const float inv_nudged_scale = 1.0f / nudged_scale; const int batches = MatchingArraySize(input_dims, 3, output_dims, 3); const int height = MatchingArraySize(input_dims, 2, output_dims, 2); @@ -4820,11 +4778,12 @@ inline void FakeQuant(const float* input_data, const Dims<4>& input_dims, for (int x = 0; x < width; ++x) { for (int c = 0; c < depth; ++c) { const float src_val = input_data[Offset(input_dims, c, x, y, b)]; - const float unclamped_quantized_val = - TfLiteRound(zero_point + src_val / scale); - const float quantized_val = std::min( - qmax_float, std::max(qmin_float, unclamped_quantized_val)); - const float dst_val = scale * (quantized_val - zero_point); + const float clamped = + std::min(nudged_max, std::max(nudged_min, src_val)); + const float clamped_shifted = clamped - nudged_min; + const float dst_val = + TfLiteRound(clamped_shifted * inv_nudged_scale) * nudged_scale + + nudged_min; output_data[Offset(output_dims, c, x, y, b)] = dst_val; } } diff --git a/tensorflow/contrib/lite/kernels/internal/quantization_util.cc b/tensorflow/contrib/lite/kernels/internal/quantization_util.cc index dd86313726..b0951aac8c 100644 --- a/tensorflow/contrib/lite/kernels/internal/quantization_util.cc +++ b/tensorflow/contrib/lite/kernels/internal/quantization_util.cc @@ -104,4 +104,25 @@ int CalculateInputRadius(int input_integer_bits, int input_left_shift) { return static_cast(std::floor(max_input_rescaled)); } +void NudgeQuantizationRange(const float min, const float max, + const int quant_min, const int quant_max, + float* nudged_min, float* nudged_max, + float* scale) { + // This code originates from tensorflow/core/kernels/fake_quant_ops_functor.h. + const float quant_min_float = static_cast(quant_min); + const float quant_max_float = static_cast(quant_max); + *scale = (max - min) / (quant_max_float - quant_min_float); + const float zero_point_from_min = quant_min_float - min / *scale; + uint16 nudged_zero_point; + if (zero_point_from_min < quant_min_float) { + nudged_zero_point = static_cast(quant_min); + } else if (zero_point_from_min > quant_max_float) { + nudged_zero_point = static_cast(quant_max); + } else { + nudged_zero_point = static_cast(TfLiteRound(zero_point_from_min)); + } + *nudged_min = (quant_min_float - nudged_zero_point) * (*scale); + *nudged_max = (quant_max_float - nudged_zero_point) * (*scale); +} + } // namespace tflite diff --git a/tensorflow/contrib/lite/kernels/internal/quantization_util.h b/tensorflow/contrib/lite/kernels/internal/quantization_util.h index 1f6f5d3b15..4a217515f1 100644 --- a/tensorflow/contrib/lite/kernels/internal/quantization_util.h +++ b/tensorflow/contrib/lite/kernels/internal/quantization_util.h @@ -209,6 +209,14 @@ void PreprocessLogSoftmaxScaling(double beta, double input_scale, // Softmax. int CalculateInputRadius(int input_integer_bits, int input_left_shift); +// Nudges a min/max quantization range to ensure zero is zero. +// Gymnastics with nudged zero point is to ensure that real zero maps to +// an integer, which is required for e.g. zero-padding in convolutional layers. +// Outputs nudged_min, nudged_max, nudged_scale. +void NudgeQuantizationRange(const float min, const float max, + const int quant_min, const int quant_max, + float* nudged_min, float* nudged_max, float* scale); + } // namespace tflite #endif // TENSORFLOW_CONTRIB_LITE_KERNELS_INTERNAL_QUANTIZATION_UTIL_H_ diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h index 93b4eb5504..0912f5928c 100644 --- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h +++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h @@ -27,6 +27,7 @@ limitations under the License. #include "fixedpoint/fixedpoint.h" #include "public/gemmlowp.h" #include "tensorflow/contrib/lite/kernels/internal/common.h" +#include "tensorflow/contrib/lite/kernels/internal/quantization_util.h" #include "tensorflow/contrib/lite/kernels/internal/round.h" #include "tensorflow/contrib/lite/kernels/internal/types.h" @@ -2697,74 +2698,30 @@ inline void Dequantize(const uint8* input_data, const Dims<4>& input_dims, } inline void FakeQuant(const float* input_data, const Dims<4>& input_dims, - float rmin, float rmax, float* output_data, + float rmin, float rmax, int num_bits, float* output_data, const Dims<4>& output_dims) { // 0 should always be a representable value. Let's assume that the initial // min,max range contains 0. - TFLITE_DCHECK_LE(rmin, 0.); - TFLITE_DCHECK_GE(rmax, 0.); - - // Determine quantization parameters: zero_point, scale. - using Integer = uint8; - const Integer qmin = std::numeric_limits::min(); - const Integer qmax = std::numeric_limits::max(); - const float qmin_float = qmin; - const float qmax_float = qmax; - int32 zero_point = 0; - float scale = 0.f; - // If rmin==rmax, both must be zero per the above assertion, - // so we are done. - if (rmin != rmax) { - // First determine the scale. - scale = (rmax - rmin) / (qmax_float - qmin_float); - - // Zero-point computation. - // First the initial floating-point computation. The zero-point can be - // determined from solving an affine equation for any known pair - // (real value, corresponding quantized value). - // We know two such pairs: (rmin, qmin) and (rmax, qmax). - // The arithmetic error on the zero point computed from either pair - // will be roughly machine_epsilon * (sum of absolute values of terms) - // so we want to use the variant that adds the smaller terms. - const float zero_point_from_min = qmin_float - rmin / scale; - const float zero_point_from_max = qmax_float - rmax / scale; - const float zero_point_from_min_error = - std::abs(qmin_float) + std::abs(rmin / scale); - const float zero_point_from_max_error = - std::abs(qmax_float) + std::abs(rmax / scale); - - const float zero_point_float = - zero_point_from_min_error < zero_point_from_max_error - ? zero_point_from_min - : zero_point_from_max; - - // Now we need to nudge the zero point to be an integer - // (our zero points are integer, and this is motivated by the requirement - // to be able to represent the real value "0" exactly as a quantized value, - // which is required in multiple places, for example in Im2col with SAME - // padding). - if (zero_point_float < qmin_float) { - zero_point = qmin; - } else if (zero_point_float > qmax_float) { - zero_point = qmax; - } else { - zero_point = static_cast(TfLiteRound(zero_point_float)); - } - // The zero point should always be in the range of quantized value, - // [qmin, qmax]. - TFLITE_DCHECK_GE(zero_point, qmin); - TFLITE_DCHECK_LE(zero_point, qmax); - } + TFLITE_DCHECK_LE(rmin, 0.0f); + TFLITE_DCHECK_GE(rmax, 0.0f); + TFLITE_DCHECK_LT(rmin, rmax); + + // Code matches tensorflow's FakeQuantWithMinMaxArgsFunctor. + int quant_min = 0; + int quant_max = (1 << num_bits) - 1; + float nudged_min, nudged_max, nudged_scale; + NudgeQuantizationRange(rmin, rmax, quant_min, quant_max, &nudged_min, + &nudged_max, &nudged_scale); + const float inv_nudged_scale = 1.0f / nudged_scale; const int flat_size = MatchingFlatSize(output_dims, input_dims); - for (int i = 0; i < flat_size; i++) { const float src_val = input_data[i]; - const float unclamped_quantized_val = - TfLiteRound(zero_point + src_val / scale); - const float quantized_val = - std::min(qmax_float, std::max(qmin_float, unclamped_quantized_val)); - const float dst_val = scale * (quantized_val - zero_point); + const float clamped = std::min(nudged_max, std::max(nudged_min, src_val)); + const float clamped_shifted = clamped - nudged_min; + const float dst_val = + TfLiteRound(clamped_shifted * inv_nudged_scale) * nudged_scale + + nudged_min; output_data[i] = dst_val; } } diff --git a/tensorflow/contrib/lite/toco/export_tensorflow.cc b/tensorflow/contrib/lite/toco/export_tensorflow.cc index 4a85f3c5a4..99ccfaea64 100644 --- a/tensorflow/contrib/lite/toco/export_tensorflow.cc +++ b/tensorflow/contrib/lite/toco/export_tensorflow.cc @@ -883,6 +883,9 @@ void ConvertFakeQuantOperator(const FakeQuantOperator& src_op, CHECK(src_op.minmax); (*fakequant_op->mutable_attr())["min"].set_f(src_op.minmax->min); (*fakequant_op->mutable_attr())["max"].set_f(src_op.minmax->max); + if (src_op.num_bits) { + (*fakequant_op->mutable_attr())["num_bits"].set_i(src_op.num_bits); + } } void ConvertMaxPoolOperator(const MaxPoolOperator& src_op, diff --git a/tensorflow/contrib/lite/toco/import_tensorflow.cc b/tensorflow/contrib/lite/toco/import_tensorflow.cc index 6b62eeb638..155d890c9f 100644 --- a/tensorflow/contrib/lite/toco/import_tensorflow.cc +++ b/tensorflow/contrib/lite/toco/import_tensorflow.cc @@ -694,6 +694,8 @@ void ConvertFakeQuantWithMinMaxArgs( minmax.min = GetFloatAttr(node, "min"); minmax.max = GetFloatAttr(node, "max"); op->outputs.push_back(node.name()); + // tf.fake_quant_with_min_max_args num_bits defaults to 8. + op->num_bits = HasAttr(node, "num_bits") ? GetIntAttr(node, "num_bits") : 8; model->operators.emplace_back(op); } @@ -711,6 +713,7 @@ void ConvertFakeQuantWithMinMaxVars( op->inputs.push_back(node.input(i)); } op->outputs.push_back(node.name()); + op->num_bits = HasAttr(node, "num_bits") ? GetIntAttr(node, "num_bits") : 8; model->operators.emplace_back(op); } diff --git a/tensorflow/contrib/lite/toco/model.h b/tensorflow/contrib/lite/toco/model.h index 54c3a59506..616d53ae3e 100644 --- a/tensorflow/contrib/lite/toco/model.h +++ b/tensorflow/contrib/lite/toco/model.h @@ -724,8 +724,7 @@ struct L2PoolOperator : Operator { // The expected [min, max] range of values in a given array. // Used for quantization only. // This information typically comes from special nodes found in quantized -// models, -// see FakeQuantOperator, and is used during quantization to resolve +// models, see FakeQuantOperator, and is used during quantization to resolve // actual quantization parameters (see QuantizationParams). struct MinMax { double min = 0.; @@ -753,6 +752,7 @@ inline bool operator==(const MinMax& m1, const MinMax& m2) { struct FakeQuantOperator : Operator { FakeQuantOperator() : Operator(OperatorType::kFakeQuant) {} std::unique_ptr minmax; + int num_bits = 8; }; // Element-wise division operator. diff --git a/tensorflow/contrib/lite/toco/tflite/operator.cc b/tensorflow/contrib/lite/toco/tflite/operator.cc index e015108120..0e057fd252 100644 --- a/tensorflow/contrib/lite/toco/tflite/operator.cc +++ b/tensorflow/contrib/lite/toco/tflite/operator.cc @@ -260,12 +260,15 @@ class FakeQuant : public CustomOperator { flexbuffers::Builder* fbb) const override { fbb->Float("min", op.minmax->min); fbb->Float("max", op.minmax->max); + fbb->Int("num_bits", op.num_bits); } void ReadOptions(const flexbuffers::Map& m, TocoOperator* op) const override { auto* minmax = new MinMax; minmax->min = m["min"].AsFloat(); minmax->max = m["max"].AsFloat(); op->minmax.reset(minmax); + const auto& num_bits = m["num_bits"]; + op->num_bits = num_bits.IsInt() ? num_bits.AsInt32() : 8; } }; diff --git a/tensorflow/contrib/lite/toco/tflite/operator_test.cc b/tensorflow/contrib/lite/toco/tflite/operator_test.cc index 24ba71e459..a947630e28 100644 --- a/tensorflow/contrib/lite/toco/tflite/operator_test.cc +++ b/tensorflow/contrib/lite/toco/tflite/operator_test.cc @@ -165,10 +165,12 @@ TEST_F(OperatorTest, CustomFakeQuant) { minmax->min = -10; minmax->max = 200; op.minmax.reset(minmax); + op.num_bits = 16; auto output_toco_op = SerializeAndDeserialize( GetOperator("FAKE_QUANT", OperatorType::kFakeQuant), op); EXPECT_EQ(op.minmax->min, output_toco_op->minmax->min); EXPECT_EQ(op.minmax->max, output_toco_op->minmax->max); + EXPECT_EQ(op.num_bits, output_toco_op->num_bits); } TEST_F(OperatorTest, CustomFullyConnected) { -- GitLab From c2f265493879a86b3ce200f9af56747bfb9dd653 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 16:31:46 -0700 Subject: [PATCH 125/791] Update programmers guide PiperOrigin-RevId: 192368335 --- tensorflow/docs_src/programmers_guide/faq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/docs_src/programmers_guide/faq.md b/tensorflow/docs_src/programmers_guide/faq.md index 392ac6f7f1..51c1a1e032 100644 --- a/tensorflow/docs_src/programmers_guide/faq.md +++ b/tensorflow/docs_src/programmers_guide/faq.md @@ -121,7 +121,7 @@ dimensions: devices, which makes it possible to speed up @{$deep_cnn$CIFAR-10 training using multiple GPUs}. * The Session API allows multiple concurrent steps (i.e. calls to - @{tf.Session.run} in parallel. This + @{tf.Session.run} in parallel). This enables the runtime to get higher throughput, if a single step does not use all of the resources in your computer. -- GitLab From 16eec071ea0a83dc5303758ac0e528f59337a1ce Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 16:32:05 -0700 Subject: [PATCH 126/791] [XLA] Redesign: implement and test ReduceWindow. PiperOrigin-RevId: 192368401 --- .../xla/client/xla_client/xla_builder.cc | 36 +++++++++++++- tensorflow/compiler/xla/tests/BUILD | 3 +- .../compiler/xla/tests/reduce_window_test.cc | 48 +++++++++---------- 3 files changed, 60 insertions(+), 27 deletions(-) diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc index 9e4b9ccd25..c869eb2ec5 100644 --- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc +++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc @@ -1425,7 +1425,21 @@ XlaOp XlaBuilder::ReduceWindow( const XlaComputation& computation, tensorflow::gtl::ArraySlice window_dimensions, tensorflow::gtl::ArraySlice window_strides, Padding padding) { - return UnimplementedOp(); + return NoteErrorOrReturn([&]() -> StatusOr { + HloInstructionProto instr; + + TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); + TF_RETURN_IF_ERROR( + ValidatePaddingValues(AsInt64Slice(operand_shape.dimensions()), + window_dimensions, window_strides)); + + std::vector> padding_values = + MakePadding(AsInt64Slice(operand_shape.dimensions()), window_dimensions, + window_strides, padding); + return ReduceWindowWithGeneralPadding(operand, init_value, computation, + window_dimensions, window_strides, + padding_values); + }); } XlaOp XlaBuilder::ReduceWindowWithGeneralPadding( @@ -1434,7 +1448,25 @@ XlaOp XlaBuilder::ReduceWindowWithGeneralPadding( tensorflow::gtl::ArraySlice window_dimensions, tensorflow::gtl::ArraySlice window_strides, tensorflow::gtl::ArraySlice> padding) { - return UnimplementedOp(); + return NoteErrorOrReturn([&]() -> StatusOr { + HloInstructionProto instr; + + TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); + TF_ASSIGN_OR_RETURN(const Shape& init_shape, GetShape(init_value)); + TF_ASSIGN_OR_RETURN(const ProgramShape& to_apply_shape, + computation.GetProgramShape()); + TF_ASSIGN_OR_RETURN(*instr.mutable_window(), + MakeWindow(window_dimensions, window_strides, padding, + /*lhs_dilation=*/{}, /*rhs_dilation=*/{})); + TF_ASSIGN_OR_RETURN( + *instr.mutable_shape(), + ShapeInference::InferReduceWindowShape(operand_shape, init_shape, + instr.window(), to_apply_shape)); + + AddCalledComputation(computation, &instr); + return AddInstruction(std::move(instr), HloOpcode::kReduceWindow, + {operand, init_value}); + }); } XlaOp XlaBuilder::BatchNormTraining(const XlaOp& operand, const XlaOp& scale, diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD index 67c53c6ac0..a615acdbb8 100644 --- a/tensorflow/compiler/xla/tests/BUILD +++ b/tensorflow/compiler/xla/tests/BUILD @@ -1091,10 +1091,11 @@ xla_test_library( "//tensorflow/compiler/xla:reference_util", "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:xla_data_proto", - "//tensorflow/compiler/xla/client:computation_builder", "//tensorflow/compiler/xla/client:local_client", "//tensorflow/compiler/xla/client:padding", "//tensorflow/compiler/xla/client/lib:arithmetic", + "//tensorflow/compiler/xla/client/xla_client:xla_builder", + "//tensorflow/compiler/xla/client/xla_client:xla_computation", "//tensorflow/compiler/xla/tests:client_library_test_base", "//tensorflow/compiler/xla/tests:hlo_test_base", "//tensorflow/compiler/xla/tests:literal_test_util", diff --git a/tensorflow/compiler/xla/tests/reduce_window_test.cc b/tensorflow/compiler/xla/tests/reduce_window_test.cc index 8dd24f1237..8ef980ebd9 100644 --- a/tensorflow/compiler/xla/tests/reduce_window_test.cc +++ b/tensorflow/compiler/xla/tests/reduce_window_test.cc @@ -21,10 +21,11 @@ limitations under the License. #include "tensorflow/compiler/xla/array2d.h" #include "tensorflow/compiler/xla/array3d.h" #include "tensorflow/compiler/xla/array4d.h" -#include "tensorflow/compiler/xla/client/computation_builder.h" #include "tensorflow/compiler/xla/client/lib/arithmetic.h" #include "tensorflow/compiler/xla/client/local_client.h" #include "tensorflow/compiler/xla/client/padding.h" +#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h" +#include "tensorflow/compiler/xla/client/xla_client/xla_computation.h" #include "tensorflow/compiler/xla/reference_util.h" #include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/tests/client_library_test_base.h" @@ -63,11 +64,9 @@ class ReduceWindowTestBase : public ClientLibraryTestBase { class ReduceWindowTest : public ::testing::WithParamInterface, public ReduceWindowTestBase { public: - ReduceWindowTest() : builder_(client_, TestName()) { - set_use_bfloat16(GetParam()); - } + ReduceWindowTest() : builder_(TestName()) { set_use_bfloat16(GetParam()); } - void ReduceWindowAdd(const ComputationDataHandle& input, + void ReduceWindowAdd(const XlaOp& input, tensorflow::gtl::ArraySlice window_dimensions, tensorflow::gtl::ArraySlice window_strides, Padding padding) { @@ -78,16 +77,17 @@ class ReduceWindowTest : public ::testing::WithParamInterface, window_dimensions, window_strides, padding); } - void ReduceWindowMax(const ComputationDataHandle& input, + void ReduceWindowMax(const XlaOp& input, tensorflow::gtl::ArraySlice window_dimensions, tensorflow::gtl::ArraySlice window_strides, Padding padding) { auto init = CreateConstantFromLiteral(Literal::MinValue(F32), &builder_); - builder_.ReduceWindow(input, init, CreateScalarMax(), window_dimensions, - window_strides, padding); + builder_.ReduceWindow(input, init, + CreateScalarMaxComputation(FloatType(), &builder_), + window_dimensions, window_strides, padding); } - void ReduceWindowMin(const ComputationDataHandle& input, + void ReduceWindowMin(const XlaOp& input, tensorflow::gtl::ArraySlice window_dimensions, tensorflow::gtl::ArraySlice window_strides, Padding padding) { @@ -97,7 +97,7 @@ class ReduceWindowTest : public ::testing::WithParamInterface, window_dimensions, window_strides, padding); } - ComputationBuilder builder_; + XlaBuilder builder_; }; TEST_P(ReduceWindowTest, MismatchedRanksGivesErrorStatus) { @@ -310,7 +310,7 @@ XLA_TEST_P(ReduceWindowTest, NonstandardReduceFunction) { auto rhs = b->Parameter(1, scalar, "rhs"); b->Min(b->Add(lhs, rhs), CreateConstantFromLiteral(*Literal::CreateR0(8.0f), b.get())); - Computation reduce_fn = b->BuildAndNoteError(); + XlaComputation reduce_fn = b->BuildAndNoteError(); builder_.ReduceWindow( input, @@ -338,7 +338,7 @@ TEST_P(ReduceWindowTest, R4UnitWindow) { std::unique_ptr input_literal = Literal::CreateR4FromArray4DWithLayout( input_array, LayoutUtil::MakeLayout({0, 3, 2, 1})); - ComputationDataHandle input; + XlaOp input; auto input_data = CreateParameterAndTransferLiteral( 0, *input_literal, "parameter", &builder_, &input); @@ -406,7 +406,7 @@ XLA_TEST_P(ReduceWindowTest, R4SecondMinorStride) { std::unique_ptr input_literal = Literal::CreateR4FromArray4DWithLayout( input_array, LayoutUtil::MakeLayout({3, 2, 1, 0})); - ComputationDataHandle input; + XlaOp input; auto input_data = CreateParameterAndTransferLiteral( 0, *input_literal, "parameter", &builder_, &input); @@ -428,7 +428,7 @@ XLA_TEST_P(ReduceWindowTest, R4SecondMinorUnitStride) { std::unique_ptr input_literal = Literal::CreateR4FromArray4DWithLayout( input_array, LayoutUtil::MakeLayout({3, 2, 1, 0})); - ComputationDataHandle input; + XlaOp input; auto input_data = CreateParameterAndTransferLiteral( 0, *input_literal, "parameter", &builder_, &input); @@ -450,7 +450,7 @@ XLA_TEST_P(ReduceWindowTest, R4SecondMinorWin) { std::unique_ptr input_literal = Literal::CreateR4FromArray4DWithLayout( input_array, LayoutUtil::MakeLayout({3, 2, 1, 0})); - ComputationDataHandle input; + XlaOp input; auto input_data = CreateParameterAndTransferLiteral( 0, *input_literal, "parameter", &builder_, &input); @@ -551,7 +551,7 @@ TEST_P(ReduceWindowTest, R2ReduceWindowInceptionFromBroadcast) { TEST_P(ReduceWindowTest, R2ReduceWindowNonOverlappingFromBroadcast) { Array2D input_array(6, 4, 1.0f); - ComputationDataHandle input = builder_.Broadcast( + XlaOp input = builder_.Broadcast( CreateConstantFromLiteral(Literal::One(F32), &builder_), {6, 4}); Padding padding = Padding::kSame; @@ -610,7 +610,7 @@ class R4ReduceWindowTest : public ReduceWindowTestBase, R4ReduceWindowTest() { set_use_bfloat16(::testing::get<1>(GetParam())); } void DoIt() { - ComputationBuilder b(client_, TestName()); + XlaBuilder b(TestName()); const auto& param = ::testing::get<0>(GetParam()); const float kInitValue = 0.0f; @@ -621,7 +621,7 @@ class R4ReduceWindowTest : public ReduceWindowTestBase, std::unique_ptr input_literal = Literal::CreateR4FromArray4DWithLayout( input, LayoutUtil::MakeLayout(param.layout)); - ComputationDataHandle parameter; + XlaOp parameter; auto input_arg = CreateParameterAndTransferLiteral(0, *input_literal, "p0", &b, ¶meter); @@ -962,7 +962,7 @@ class R3ReduceWindowTest : public ReduceWindowTestBase, }; TEST_P(R3ReduceWindowTest, Add) { - ComputationBuilder b(client_, TestName()); + XlaBuilder b(TestName()); const auto& param = ::testing::get<0>(GetParam()); CHECK(param.reducer == kAdd); @@ -973,7 +973,7 @@ TEST_P(R3ReduceWindowTest, Add) { Literal::CreateR3FromArray3DWithLayout( input, LayoutUtil::MakeLayout(param.layout)); - ComputationDataHandle parameter; + XlaOp parameter; auto input_arg = CreateParameterAndTransferLiteral(0, *input_literal, "p0", &b, ¶meter); auto init_value = @@ -1100,7 +1100,7 @@ class R2ReduceWindowTest : public ReduceWindowTestBase, R2ReduceWindowTest() { set_use_bfloat16(::testing::get<1>(GetParam())); } void DoIt() { - ComputationBuilder b(client_, TestName()); + XlaBuilder b(TestName()); const auto& param = ::testing::get<0>(GetParam()); CHECK(param.reducer == kAdd); @@ -1110,7 +1110,7 @@ class R2ReduceWindowTest : public ReduceWindowTestBase, Literal::CreateR2FromArray2DWithLayout( input, LayoutUtil::MakeLayout(param.layout)); - ComputationDataHandle parameter; + XlaOp parameter; auto input_arg = CreateParameterAndTransferLiteral(0, *input_literal, "p0", &b, ¶meter); std::vector> padding(2); @@ -1298,7 +1298,7 @@ class R1ReduceWindowTest : public ReduceWindowTestBase, }; TEST_P(R1ReduceWindowTest, DoIt) { - ComputationBuilder b(client_, TestName()); + XlaBuilder b(TestName()); const auto& param = ::testing::get<0>(GetParam()); CHECK(param.reducer == kAdd || param.reducer == kMax); @@ -1307,7 +1307,7 @@ TEST_P(R1ReduceWindowTest, DoIt) { std::iota(std::begin(input_vector), std::end(input_vector), 0); std::unique_ptr input_literal = Literal::CreateR1(tensorflow::gtl::ArraySlice(input_vector)); - ComputationDataHandle parameter; + XlaOp parameter; auto input_arg = CreateParameterAndTransferLiteral(0, *input_literal, "p0", &b, ¶meter); -- GitLab From f5c2e5d968d371c0855c6d7b2cc4f050615d4bc4 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Tue, 10 Apr 2018 16:35:58 -0700 Subject: [PATCH 127/791] Fix issue with gradients of resource variables in cond. PiperOrigin-RevId: 192369091 --- tensorflow/python/ops/control_flow_grad.py | 6 ++++++ tensorflow/python/ops/gradients_test.py | 25 ++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/tensorflow/python/ops/control_flow_grad.py b/tensorflow/python/ops/control_flow_grad.py index 45955554ca..6a551deb5b 100644 --- a/tensorflow/python/ops/control_flow_grad.py +++ b/tensorflow/python/ops/control_flow_grad.py @@ -20,6 +20,7 @@ from __future__ import print_function from six.moves import xrange # pylint: disable=redefined-builtin +from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor from tensorflow.python.ops import control_flow_ops @@ -74,6 +75,11 @@ def _SwitchGrad(op, *grad): # At this point, we have created zero_grad guarded by the right switch. # Unfortunately, we may still get None here for not trainable data types. if zero_grad is None: + # For resource variables we get None always on the other branch, so bypass + # this. + if op.inputs[0].dtype == dtypes.resource: + return merge( + [grad[op_ctxt.branch]] * 2, name="cond_resource_grad")[0], None return None, None return merge(grad, name="cond_grad")[0], None else: diff --git a/tensorflow/python/ops/gradients_test.py b/tensorflow/python/ops/gradients_test.py index c94f1396b2..0603d3b670 100644 --- a/tensorflow/python/ops/gradients_test.py +++ b/tensorflow/python/ops/gradients_test.py @@ -44,6 +44,7 @@ from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import math_grad # pylint: disable=unused-import from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn_grad # pylint: disable=unused-import +from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import state_grad # pylint: disable=unused-import from tensorflow.python.ops import tensor_array_grad # pylint: disable=unused-import from tensorflow.python.ops import tensor_array_ops @@ -810,5 +811,29 @@ class OnlyRealGradientsTest(test_util.TensorFlowTestCase): gradients.gradients(y, x) +class ResourceCondTest(test_util.TensorFlowTestCase): + + def testBasic(self): + gamma = resource_variable_ops.ResourceVariable( + np.random.random((3,)), + dtype="float32", name="gamma") + + inputs = array_ops.ones(shape=(3,), dtype="float32") + + def TestFn(): + output = inputs + gamma + return output + + training = array_ops.placeholder_with_default(True, shape=()) + output = control_flow_ops.cond( + training, TestFn, lambda: inputs) + + loss = output + + grads = gradients.gradients( + loss, [gamma]) + self.assertTrue(None not in grads) + + if __name__ == "__main__": googletest.main() -- GitLab From f83843a4b8dde5e9306c2b91da8ccbd438a7265f Mon Sep 17 00:00:00 2001 From: Michael Kuperstein Date: Tue, 10 Apr 2018 16:45:19 -0700 Subject: [PATCH 128/791] Add a thread-safe producer-consumer queue. PiperOrigin-RevId: 192370670 --- tensorflow/compiler/jit/BUILD | 19 +++ .../compiler/jit/producer_consumer_queue.h | 132 +++++++++++++++++ .../jit/producer_consumer_queue_test.cc | 139 ++++++++++++++++++ 3 files changed, 290 insertions(+) create mode 100644 tensorflow/compiler/jit/producer_consumer_queue.h create mode 100644 tensorflow/compiler/jit/producer_consumer_queue_test.cc diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD index a492fc6b9b..4cefc08645 100644 --- a/tensorflow/compiler/jit/BUILD +++ b/tensorflow/compiler/jit/BUILD @@ -318,6 +318,25 @@ cc_library( hdrs = ["union_find.h"], ) +cc_library( + name = "producer_consumer_queue", + hdrs = ["producer_consumer_queue.h"], + deps = ["//tensorflow/core:lib"], +) + +tf_cc_test( + name = "producer_consumer_queue_test", + size = "small", + srcs = ["producer_consumer_queue_test.cc"], + deps = [ + ":producer_consumer_queue", + "//tensorflow/core:lib", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + ], +) + tf_cc_test( name = "graph_to_functiondef_test", size = "small", diff --git a/tensorflow/compiler/jit/producer_consumer_queue.h b/tensorflow/compiler/jit/producer_consumer_queue.h new file mode 100644 index 0000000000..7c8c04152d --- /dev/null +++ b/tensorflow/compiler/jit/producer_consumer_queue.h @@ -0,0 +1,132 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_JIT_PRODUCER_CONSUMER_QUEUE_H_ +#define TENSORFLOW_COMPILER_JIT_PRODUCER_CONSUMER_QUEUE_H_ + +#include +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/mutex.h" + +namespace tensorflow { + +// A thread-safe, first-in-first-out queue. +template +class ProducerConsumerQueue { + public: + ProducerConsumerQueue() + : capacity_(std::numeric_limits::max()) {} + ~ProducerConsumerQueue() = default; + + // Wait until the queue is non-full, then append a copy of v. + void Put(const T &v); + + // Wait until the queue is non-empty, then remove and return the head value. + T Get(); + + // If the queue is non-empty, remove the head value, placing it in *pv, and + // return true; otherwise return false. + bool TryGet(T *pv); + + // Set the capacity of the queue; the queue is full whenever count() >= + // capacity(). The initial value is the maximum size_t. Requires size > 0. + void set_capacity(std::size_t size); + + // Return the capacity of the queue. + std::size_t capacity() const; + + // Return the number of elements in the queue. + std::size_t count() const; + + // Implementation details follow. Clients should ignore. + private: + mutable tensorflow::mutex mu_; // protects all fields below + tensorflow::condition_variable non_empty_ GUARDED_BY(mu_); + tensorflow::condition_variable non_full_ GUARDED_BY(mu_); + std::size_t capacity_ GUARDED_BY(mu_); + std::deque queue_ GUARDED_BY(mu_); + + TF_DISALLOW_COPY_AND_ASSIGN(ProducerConsumerQueue); +}; + +// ------------------------------------------------------ +// Implementation details follow. Clients should ignore. + +// Wait until the queue is non-full, then append a copy of v. +template +void ProducerConsumerQueue::Put(const T &v) { + mutex_lock lock(mu_); + while (queue_.size() >= capacity_) { + non_full_.wait(lock); + } + queue_.push_back(v); + non_empty_.notify_one(); +} + +// Wait until the queue is non-empty, then remove and return the head value. +template +T ProducerConsumerQueue::Get() { + mutex_lock lock(mu_); + while (queue_.empty()) { + non_empty_.wait(lock); + } + non_full_.notify_one(); + T result_value = queue_.front(); + queue_.pop_front(); + return result_value; +} + +// If the queue is non-empty, remove the head value, placing it in *pv, and +// return true; otherwise return false. +template +bool ProducerConsumerQueue::TryGet(T *pv) { + mutex_lock lock(mu_); + bool got_element = !queue_.empty(); + if (got_element) { + non_full_.notify_one(); + *pv = queue_.front(); + queue_.pop_front(); + } + return got_element; +} + +// Set the capacity of the queue; the queue is full whenever count() >= +// capacity(). The initial value is the maximum size_t. Requires size > 0. +template +void ProducerConsumerQueue::set_capacity(std::size_t size) { + mutex_lock lock(mu_); + CHECK_NE(size, 0); + capacity_ = size; + non_full_.notify_all(); +} + +// Return the capacity of the queue. +template +std::size_t ProducerConsumerQueue::capacity() const { + mutex_lock lock(mu_); + std::size_t max_elements = capacity_; + return max_elements; +} + +// Return the number of elements in the queue. +template +std::size_t ProducerConsumerQueue::count() const { + mutex_lock lock(mu_); + std::size_t num_elements = queue_.size(); + return num_elements; +} +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_JIT_PRODUCER_CONSUMER_QUEUE_H_ diff --git a/tensorflow/compiler/jit/producer_consumer_queue_test.cc b/tensorflow/compiler/jit/producer_consumer_queue_test.cc new file mode 100644 index 0000000000..f61260c6e5 --- /dev/null +++ b/tensorflow/compiler/jit/producer_consumer_queue_test.cc @@ -0,0 +1,139 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/jit/producer_consumer_queue.h" + +#include "tensorflow/core/lib/core/threadpool.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/mutex.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace { + +typedef ProducerConsumerQueue IntQueue; + +// Insert integers between low inclusive and high exclusive into q. +void PushRange(IntQueue *q, int low, int high) { + while (low != high) { + q->Put(low); + VLOG(2) << "Pushing " << low; + ++low; + } +} + +// Push the numbers between 0 and 999 inclusive from several threads in the +// pool. +void PushRanges(IntQueue *queue, thread::ThreadPool *pool) { + VLOG(1) << "Adding 20-36"; + pool->Schedule([queue] { PushRange(queue, 20, 36); }); + VLOG(1) << "Adding 7-20"; + pool->Schedule([queue] { PushRange(queue, 7, 20); }); + VLOG(1) << "Adding 36-501"; + pool->Schedule([queue] { PushRange(queue, 36, 501); }); + VLOG(1) << "Adding 501-1000"; + pool->Schedule([queue] { PushRange(queue, 501, 1000); }); + VLOG(1) << "Adding 0-5"; + pool->Schedule([queue] { PushRange(queue, 0, 5); }); + VLOG(1) << "Adding 5-7"; + pool->Schedule([queue] { PushRange(queue, 5, 7); }); +} + +// Pop elements from queue using Get(). Make sure that exactly elements +// were present and their values are all integers between 0 and high-1 +// inclusive. +void GetRange(IntQueue *queue, int high) { + VLOG(1) << "Testing Wait"; + std::vector results; + for (int i = 0; i != high; ++i) { + int r = queue->Get(); + VLOG(2) << "Waited and got " << r; + results.push_back(r); + } + CHECK_EQ(queue->count(), 0); + std::sort(results.begin(), results.end()); + for (int i = 0; i != high; ++i) { + CHECK(results[i] == i); + } +} + +// Pop elements from queue using TryGet(). Make sure that exactly +// elements were present and their values are all integers between 0 and high-1 +// inclusive. +void TryGetRange(IntQueue *queue, int high) { + std::vector results; + // Give up if we don't get all the elements back from the queue + // in 10 seconds. + int timeout = 10; + int r; + for (int i = 0; i != high; ++i) { + while (!queue->TryGet(&r)) { + if (!timeout--) { + LOG(FATAL) << "Can't find all elements in the queue"; + } + VLOG(1) << "Sleeping for a second..."; + sleep(1); + } + VLOG(2) << "Popped " << r; + results.push_back(r); + } + CHECK_EQ(queue->count(), 0); + CHECK(!queue->TryGet(&r)); + std::sort(results.begin(), results.end()); + for (int i = 0; i != high; ++i) { + CHECK_EQ(i, results[i]); + } +} + +const int kNumThreads = 15; + +TEST(ProducerConsumerQueue, GetRange) { + IntQueue queue; + { + thread::ThreadPool pool(Env::Default(), "test", kNumThreads); + PushRanges(&queue, &pool); + } + GetRange(&queue, 1000); +} + +TEST(ProducerConsumerQueue, TryGetRange) { + IntQueue queue; + { + thread::ThreadPool pool(Env::Default(), "test", kNumThreads); + PushRanges(&queue, &pool); + } + TryGetRange(&queue, 1000); +} + +TEST(ProducerConsumerQueue, ParallelGetRange) { + IntQueue queue; + { + thread::ThreadPool pool(Env::Default(), "test", kNumThreads); + pool.Schedule([&queue] { GetRange(&queue, 1000); }); + PushRanges(&queue, &pool); + } +} + +TEST(ProducerConsumerQueue, ParallelTryGetRange) { + IntQueue queue; + { + thread::ThreadPool pool(Env::Default(), "test", kNumThreads); + pool.Schedule([&queue] { TryGetRange(&queue, 1000); }); + PushRanges(&queue, &pool); + } +} + +} // namespace +} // namespace tensorflow -- GitLab From 408f524761e50b98159ad8ff3b18a0f6af08d867 Mon Sep 17 00:00:00 2001 From: Jacques Pienaar Date: Tue, 10 Apr 2018 16:47:08 -0700 Subject: [PATCH 129/791] Add types to error message in case of mismatch. NFC. PiperOrigin-RevId: 192370979 --- tensorflow/core/framework/tensor.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/framework/tensor.cc b/tensorflow/core/framework/tensor.cc index e2111d6038..d5a45c73c3 100644 --- a/tensorflow/core/framework/tensor.cc +++ b/tensorflow/core/framework/tensor.cc @@ -610,11 +610,15 @@ bool Tensor::IsInitialized() const { } void Tensor::CheckType(DataType expected_dtype) const { - CHECK_EQ(dtype(), expected_dtype); + CHECK_EQ(dtype(), expected_dtype) + << DataTypeString(expected_dtype) << " expected, got " + << DataTypeString(dtype()); } void Tensor::CheckTypeAndIsAligned(DataType expected_dtype) const { - CHECK_EQ(dtype(), expected_dtype); + CHECK_EQ(dtype(), expected_dtype) + << DataTypeString(expected_dtype) << " expected, got " + << DataTypeString(dtype()); CHECK(IsAligned()) << "CheckTypeAndIsAligned"; } -- GitLab From ffc651af58ebacdf3ddbe9537efda694c71a64f3 Mon Sep 17 00:00:00 2001 From: Austin Anderson Date: Tue, 10 Apr 2018 17:37:53 -0700 Subject: [PATCH 130/791] Update LogToSTDErr for TF Lite usage PiperOrigin-RevId: 192379483 --- tensorflow/contrib/lite/kernels/arg_max_test.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tensorflow/contrib/lite/kernels/arg_max_test.cc b/tensorflow/contrib/lite/kernels/arg_max_test.cc index f4e1da3a6e..31b15fe19a 100644 --- a/tensorflow/contrib/lite/kernels/arg_max_test.cc +++ b/tensorflow/contrib/lite/kernels/arg_max_test.cc @@ -100,8 +100,7 @@ TEST(ArgMaxOpTest, GetMaxArgOutput64) { } // namespace tflite int main(int argc, char** argv) { - // On Linux, add: FLAGS_logtostderr = true; - FLAGS_logtostderr = true; + ::tflite::LogToStderr(); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } -- GitLab From 0c219524a9b2ad82dfac1659d0957c0475d0cc25 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 17:41:56 -0700 Subject: [PATCH 131/791] [XLA] Redesign: implement and test SelectAndScatter. PiperOrigin-RevId: 192380121 --- .../xla/client/xla_client/xla_builder.cc | 34 +++++++++++++++++-- tensorflow/compiler/xla/tests/BUILD | 4 +-- .../xla/tests/select_and_scatter_test.cc | 29 ++++++++-------- 3 files changed, 48 insertions(+), 19 deletions(-) diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc index c869eb2ec5..b96421128e 100644 --- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc +++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc @@ -1499,7 +1499,14 @@ XlaOp XlaBuilder::SelectAndScatter( tensorflow::gtl::ArraySlice window_strides, Padding padding, const XlaOp& source, const XlaOp& init_value, const XlaComputation& scatter) { - return UnimplementedOp(); + return NoteErrorOrReturn([&]() -> StatusOr { + TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); + return SelectAndScatterWithGeneralPadding( + operand, select, window_dimensions, window_strides, + MakePadding(AsInt64Slice(operand_shape.dimensions()), window_dimensions, + window_strides, padding), + source, init_value, scatter); + }); } XlaOp XlaBuilder::SelectAndScatterWithGeneralPadding( @@ -1509,7 +1516,30 @@ XlaOp XlaBuilder::SelectAndScatterWithGeneralPadding( tensorflow::gtl::ArraySlice> padding, const XlaOp& source, const XlaOp& init_value, const XlaComputation& scatter) { - return UnimplementedOp(); + return NoteErrorOrReturn([&]() -> StatusOr { + HloInstructionProto instr; + + TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); + TF_ASSIGN_OR_RETURN(const Shape& source_shape, GetShape(source)); + TF_ASSIGN_OR_RETURN(const Shape& init_shape, GetShape(init_value)); + TF_ASSIGN_OR_RETURN(const ProgramShape& select_shape, + select.GetProgramShape()); + TF_ASSIGN_OR_RETURN(const ProgramShape& scatter_shape, + scatter.GetProgramShape()); + TF_ASSIGN_OR_RETURN(*instr.mutable_window(), + MakeWindow(window_dimensions, window_strides, padding, + /*lhs_dilation=*/{}, /*rhs_dilation=*/{})); + TF_ASSIGN_OR_RETURN(*instr.mutable_shape(), + ShapeInference::InferSelectAndScatterShape( + operand_shape, select_shape, instr.window(), + source_shape, init_shape, scatter_shape)); + + AddCalledComputation(select, &instr); + AddCalledComputation(scatter, &instr); + + return AddInstruction(std::move(instr), HloOpcode::kSelectAndScatter, + {operand, source, init_value}); + }); } XlaOp XlaBuilder::ReducePrecision(const XlaOp& operand, const int exponent_bits, diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD index a615acdbb8..2a2ef229ed 100644 --- a/tensorflow/compiler/xla/tests/BUILD +++ b/tensorflow/compiler/xla/tests/BUILD @@ -1132,11 +1132,11 @@ xla_test( "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:status_macros", "//tensorflow/compiler/xla:xla_data_proto", - "//tensorflow/compiler/xla/client:computation", - "//tensorflow/compiler/xla/client:computation_builder", "//tensorflow/compiler/xla/client:local_client", "//tensorflow/compiler/xla/client:padding", "//tensorflow/compiler/xla/client/lib:arithmetic", + "//tensorflow/compiler/xla/client/xla_client:xla_builder", + "//tensorflow/compiler/xla/client/xla_client:xla_computation", "//tensorflow/compiler/xla/tests:client_library_test_base", "//tensorflow/compiler/xla/tests:literal_test_util", "//tensorflow/compiler/xla/tests:xla_internal_test_main", diff --git a/tensorflow/compiler/xla/tests/select_and_scatter_test.cc b/tensorflow/compiler/xla/tests/select_and_scatter_test.cc index d268fdcace..7015e5a6a3 100644 --- a/tensorflow/compiler/xla/tests/select_and_scatter_test.cc +++ b/tensorflow/compiler/xla/tests/select_and_scatter_test.cc @@ -19,11 +19,11 @@ limitations under the License. #include #include "tensorflow/compiler/xla/array2d.h" -#include "tensorflow/compiler/xla/client/computation.h" -#include "tensorflow/compiler/xla/client/computation_builder.h" #include "tensorflow/compiler/xla/client/lib/arithmetic.h" #include "tensorflow/compiler/xla/client/local_client.h" #include "tensorflow/compiler/xla/client/padding.h" +#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h" +#include "tensorflow/compiler/xla/client/xla_client/xla_computation.h" #include "tensorflow/compiler/xla/layout_util.h" #include "tensorflow/compiler/xla/literal_util.h" #include "tensorflow/compiler/xla/reference_util.h" @@ -50,7 +50,7 @@ class SelectAndScatterTest : public ClientLibraryTestBase, public ::testing::WithParamInterface { public: - SelectAndScatterTest() : builder_(client_, TestName()) { + SelectAndScatterTest() : builder_(TestName()) { // Create S32 GE and ADD computations for select and scatter respectively. ge_s32_ = CreateScalarGeComputation(S32, &builder_); add_s32_ = CreateScalarAddComputation(S32, &builder_); @@ -60,13 +60,13 @@ class SelectAndScatterTest min_f32_ = CreateScalarMinComputation(F32, &builder_); } - ComputationBuilder builder_; - Computation ge_s32_; - Computation add_s32_; - Computation ge_f32_; - Computation add_f32_; - Computation max_f32_; - Computation min_f32_; + XlaBuilder builder_; + XlaComputation ge_s32_; + XlaComputation add_s32_; + XlaComputation ge_f32_; + XlaComputation add_f32_; + XlaComputation max_f32_; + XlaComputation min_f32_; }; XLA_TEST_P(SelectAndScatterTest, ParamTest) { @@ -80,12 +80,11 @@ XLA_TEST_P(SelectAndScatterTest, ParamTest) { s.FillRandom(12.0f); auto source = builder_.ConstantFromArray(s); - auto select_and_scatter = builder_.SelectAndScatter( - operand, ge_f32_, GetParam().window_dimensions, GetParam().window_strides, - GetParam().padding_type, source, builder_.ConstantR0(0.0f), - add_f32_); + builder_.SelectAndScatter(operand, ge_f32_, GetParam().window_dimensions, + GetParam().window_strides, GetParam().padding_type, + source, builder_.ConstantR0(0.0f), add_f32_); - ComputeAndCompare(&builder_, select_and_scatter, {}, ErrorSpec(1e-5)); + ComputeAndCompare(&builder_, {}, ErrorSpec(1e-5)); } INSTANTIATE_TEST_CASE_P( -- GitLab From 0f862770b3890a12d783c3fa31f4aaf8b6233a21 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 17:44:48 -0700 Subject: [PATCH 132/791] [XLA] Redesign: implement ReduceAll. PiperOrigin-RevId: 192380688 --- tensorflow/compiler/xla/client/xla_client/xla_builder.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc index b96421128e..a08ad0e30e 100644 --- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc +++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc @@ -1417,7 +1417,12 @@ XlaOp XlaBuilder::Reduce( XlaOp XlaBuilder::ReduceAll(const XlaOp& operand, const XlaOp& init_value, const XlaComputation& computation) { - return UnimplementedOp(); + return NoteErrorOrReturn([&]() -> StatusOr { + TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); + std::vector all_dimnos(ShapeUtil::Rank(operand_shape)); + std::iota(all_dimnos.begin(), all_dimnos.end(), 0); + return Reduce(operand, init_value, computation, all_dimnos); + }); } XlaOp XlaBuilder::ReduceWindow( -- GitLab From 462e799f0c2c3652b0cc712f34cf5142b487bad2 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 17:47:13 -0700 Subject: [PATCH 133/791] [XLA] Redesign: implement SliceInDim. PiperOrigin-RevId: 192381080 --- .../compiler/xla/client/xla_client/xla_builder.cc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc index a08ad0e30e..c7c303fe9d 100644 --- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc +++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc @@ -538,7 +538,17 @@ XlaOp XlaBuilder::Slice(const XlaOp& operand, XlaOp XlaBuilder::SliceInDim(const XlaOp& operand, int64 start_index, int64 limit_index, int64 stride, int64 dimno) { - return UnimplementedOp(); + return NoteErrorOrReturn([&]() -> StatusOr { + TF_ASSIGN_OR_RETURN(const Shape& shape, GetShape(operand)); + std::vector starts(ShapeUtil::Rank(shape), 0); + std::vector limits(shape.dimensions().begin(), + shape.dimensions().end()); + std::vector strides(ShapeUtil::Rank(shape), 1); + starts[dimno] = start_index; + limits[dimno] = limit_index; + strides[dimno] = stride; + return Slice(operand, starts, limits, strides); + }); } XlaOp XlaBuilder::DynamicSlice(const XlaOp& operand, const XlaOp& start_indices, -- GitLab From 874cee614d2baca210abe06e21f16632f3e4b97d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 17:49:51 -0700 Subject: [PATCH 134/791] [XLA] Redesign: implement Conj. PiperOrigin-RevId: 192381481 --- tensorflow/compiler/xla/client/xla_client/xla_builder.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc index c7c303fe9d..ba76001c78 100644 --- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc +++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc @@ -1083,7 +1083,9 @@ XlaOp XlaBuilder::Complex( return BinaryOp(HloOpcode::kComplex, real, imag, broadcast_dimensions); } -XlaOp XlaBuilder::Conj(const XlaOp& operand) { return UnimplementedOp(); } +XlaOp XlaBuilder::Conj(const XlaOp& operand) { + return Complex(Real(operand), Neg(Imag(operand))); +} XlaOp XlaBuilder::Sub(const XlaOp& lhs, const XlaOp& rhs, tensorflow::gtl::ArraySlice broadcast_dimensions) { -- GitLab From fe4dd168744b39daca4761d5e6ccf5c93458f023 Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Tue, 10 Apr 2018 17:57:19 -0700 Subject: [PATCH 135/791] Forward the status from LookupResource to GetInputTensorFromVariable rather than returning a generic error status PiperOrigin-RevId: 192382499 --- tensorflow/core/kernels/training_op_helpers.h | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/tensorflow/core/kernels/training_op_helpers.h b/tensorflow/core/kernels/training_op_helpers.h index 857daae177..7e56e15450 100644 --- a/tensorflow/core/kernels/training_op_helpers.h +++ b/tensorflow/core/kernels/training_op_helpers.h @@ -78,24 +78,21 @@ Status GetInputTensorFromVariable(OpKernelContext* ctx, int input, bool lock_held, bool sparse, Tensor* out) { if (ctx->input_dtype(input) == DT_RESOURCE) { Var* var; - if (LookupResource(ctx, HandleFromInput(ctx, input), &var).ok()) { - core::ScopedUnref unref_var(var); - if (lock_held) { + TF_RETURN_IF_ERROR(LookupResource(ctx, HandleFromInput(ctx, input), &var)); + core::ScopedUnref unref_var(var); + if (lock_held) { + TF_RETURN_IF_ERROR( + PrepareToUpdateVariable(ctx, var->tensor())); + *out = *var->tensor(); + } else { + mutex_lock ml(*var->mu()); + if (!sparse) { TF_RETURN_IF_ERROR( PrepareToUpdateVariable(ctx, var->tensor())); - *out = *var->tensor(); - } else { - mutex_lock ml(*var->mu()); - if (!sparse) { - TF_RETURN_IF_ERROR( - PrepareToUpdateVariable(ctx, var->tensor())); - } - *out = *var->tensor(); } - return Status::OK(); - } else { - return errors::Internal("Invalid variable reference."); + *out = *var->tensor(); } + return Status::OK(); } *out = ctx->mutable_input(input, lock_held); return Status::OK(); -- GitLab From 69136b4d2204b8e6dfd619bdb9a2a788c3c8b431 Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Tue, 10 Apr 2018 18:04:20 -0700 Subject: [PATCH 136/791] TFTS: De-flake the LSTM test Disabling the value-based check for now. Hopefully the shapes are deterministic. PiperOrigin-RevId: 192383553 --- tensorflow/contrib/timeseries/examples/lstm_test.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tensorflow/contrib/timeseries/examples/lstm_test.py b/tensorflow/contrib/timeseries/examples/lstm_test.py index ca56e38ca0..c58e24e6d9 100644 --- a/tensorflow/contrib/timeseries/examples/lstm_test.py +++ b/tensorflow/contrib/timeseries/examples/lstm_test.py @@ -36,17 +36,14 @@ class LSTMExampleTest(test.TestCase): def test_periodicity_learned(self): (observed_times, observed_values, all_times, predicted_values) = lstm.train_and_predict( - training_steps=100, estimator_config=_SeedRunConfig(), + training_steps=2, estimator_config=_SeedRunConfig(), export_directory=self.get_temp_dir()) self.assertAllEqual([100], observed_times.shape) self.assertAllEqual([100, 5], observed_values.shape) self.assertAllEqual([200], all_times.shape) self.assertAllEqual([200, 5], predicted_values.shape) - self.assertGreater( - predicted_values[100, 4] - - predicted_values[115, 4], # Amplitude of fifth component - 0.2) - + # TODO(allenl): Make the model deterministic so you can check something + # substantive. if __name__ == "__main__": test.main() -- GitLab From 695340d72acb786805837df0040332b81aafcaa9 Mon Sep 17 00:00:00 2001 From: Maciej Date: Tue, 10 Apr 2018 20:37:03 -0500 Subject: [PATCH 137/791] typo and readability fixes in CPU section (#18370) Fixed a typo in the Tuning MKL section, and modified punctuation for intra_op_parallelism_threads section for easier readability. --- tensorflow/docs_src/performance/performance_guide.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/docs_src/performance/performance_guide.md b/tensorflow/docs_src/performance/performance_guide.md index 580a899ac4..b1796cf9b2 100644 --- a/tensorflow/docs_src/performance/performance_guide.md +++ b/tensorflow/docs_src/performance/performance_guide.md @@ -475,7 +475,7 @@ optimizations. ### TensorFlow with Intel® MKL DNN Intel® has added optimizations to TensorFlow for Intel® Xeon® and Intel® Xeon -Phiâ„¢ though the use of Intel® Math Kernel Library for Deep Neural Networks +Phiâ„¢ through the use of the Intel® Math Kernel Library for Deep Neural Networks (Intel® MKL-DNN) optimized primitives. The optimizations also provide speedups for the consumer line of processors, e.g. i5 and i7 Intel processors. The Intel published paper @@ -581,9 +581,9 @@ Each variable that impacts performance is discussed below. for optimal settings. * **intra_op_parallelism_threads**: Setting this equal to the number of - physical cores is recommended. Setting the value to 0, which is the default - and will result in the value being set to the number of logical cores, is an - option to try for some architectures. This value and `OMP_NUM_THREADS` + physical cores is recommended. Setting the value to 0, which is the default, + results in the value being set to the number of logical cores - this is an + alternate option to try for some architectures. This value and `OMP_NUM_THREADS` should be equal. * **inter_op_parallelism_threads**: Setting this equal to the number of -- GitLab From 0899c019e404c0df17af70e50be95e1de1698b64 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 10 Apr 2018 18:37:35 -0700 Subject: [PATCH 138/791] Fix code block rendering issue in adding_an_op.md (#18368) * Fix code block rendering issue in adding_an_op.md In adding_an_op.md, html code was used in markdown for code blocks. However, this does not work very well as some of the code blocks includes incorrect rendering. This fix converts html into "```c++" (backticks) so that the rendering could be fixed. Signed-off-by: Yong Tang * Fix additional html code Signed-off-by: Yong Tang * Fix lang-cpp issue Signed-off-by: Yong Tang * Further clean up Signed-off-by: Yong Tang --- tensorflow/docs_src/extend/adding_an_op.md | 159 +++++++++++---------- 1 file changed, 84 insertions(+), 75 deletions(-) diff --git a/tensorflow/docs_src/extend/adding_an_op.md b/tensorflow/docs_src/extend/adding_an_op.md index 15075e1df8..84da2165b5 100644 --- a/tensorflow/docs_src/extend/adding_an_op.md +++ b/tensorflow/docs_src/extend/adding_an_op.md @@ -530,56 +530,58 @@ form [described below](#attr_types). For example, if you'd like the `ZeroOut` op to preserve a user-specified index, instead of only the 0th element, you can register the op like so: -

-REGISTER\_OP("ZeroOut")
-    .Attr("preserve\_index: int")
-    .Input("to\_zero: int32")
+```c++
+REGISTER_OP("ZeroOut")
+    .Attr("preserve_index: int")
+    .Input("to_zero: int32")
     .Output("zeroed: int32");
-
+``` (Note that the set of [attribute types](#attr_types) is different from the @{tf.DType$tensor types} used for inputs and outputs.) Your kernel can then access this attr in its constructor via the `context` parameter: -

+```c++
 class ZeroOutOp : public OpKernel {
  public:
-  explicit ZeroOutOp(OpKernelConstruction\* context) : OpKernel(context) {
+  explicit ZeroOutOp(OpKernelConstruction* context) : OpKernel(context) {
     // Get the index of the value to preserve
-    OP\_REQUIRES\_OK(context,
-                   context->GetAttr("preserve\_index", &preserve\_index\_));
-    // Check that preserve\_index is positive
-    OP\_REQUIRES(context, preserve\_index_ >= 0,
-                errors::InvalidArgument("Need preserve\_index >= 0, got ",
-                                        preserve\_index_));
-  }
-  void Compute(OpKernelContext\* context) override {
+    OP_REQUIRES_OK(context,
+                   context->GetAttr("preserve_index", &preserve_index_));
+    // Check that preserve_index is positive
+    OP_REQUIRES(context, preserve_index_ >= 0,
+                errors::InvalidArgument("Need preserve_index >= 0, got ",
+                                        preserve_index_));
+  }
+  void Compute(OpKernelContext* context) override {
     // ...
   }
- private:
-  int preserve\_index\_;
+ private:
+  int preserve_index_;
 };
-
+``` which can then be used in the `Compute` method: -

-  void Compute(OpKernelContext\* context) override {
+```c++
+  void Compute(OpKernelContext* context) override {
     // ...
-

-    // We're using saved attr to validate potentially dynamic input
-    // So we check that preserve\_index is in range
-    OP\_REQUIRES(context, preserve\_index_ < input.dimension(0),
-                errors::InvalidArgument("preserve\_index out of range"));

-    // Set all the elements of the output tensor to 0
+
+    // We're using saved attr to validate potentially dynamic input
+    // So we check that preserve_index is in range
+    OP_REQUIRES(context, preserve_index_ < input.dimension(0),
+                errors::InvalidArgument("preserve_index out of range"));
+
+    // Set all the elements of the output tensor to 0
     const int N = input.size();
     for (int i = 0; i < N; i++) {
       output\_flat(i) = 0;
-    }

-    // Preserve the requested input value
-    output\_flat(preserve\_index\_) = input(preserve\_index\_);
+    }
+
+    // Preserve the requested input value
+    output_flat(preserve_index_) = input(preserve_index_);
   }
-
+``` #### Attr types @@ -725,12 +727,12 @@ you would then register an `OpKernel` for each supported type. For instance, if you'd like the `ZeroOut` op to work on `float`s in addition to `int32`s, your op registration might look like: -

-REGISTER\_OP("ZeroOut")
-    .Attr("T: {float, int32}")
-    .Input("to\_zero: T")
-    .Output("zeroed: T");
-
+```c++ +REGISTER_OP("ZeroOut") + .Attr("T: {float, int32}") + .Input("to_zero: T") + .Output("zeroed: T"); +``` Your op registration now specifies that the input's type must be `float`, or `int32`, and that its output will be the same type, since both have type `T`. @@ -790,66 +792,73 @@ Your op registration now specifies that the input's type must be `float`, or > """ > ``` -

-\#include "tensorflow/core/framework/op_kernel.h"

-class ZeroOutInt32Op : public OpKernel {
+```c++
+#include "tensorflow/core/framework/op_kernel.h"
+
+class ZeroOutInt32Op : public OpKernel {
   // as before
-};

-class ZeroOutFloatOp : public OpKernel {
+};
+
+class ZeroOutFloatOp : public OpKernel {
  public:
-  explicit ZeroOutFloatOp(OpKernelConstruction\* context)
-      : OpKernel(context) {}

-  void Compute(OpKernelContext\* context) override {
+  explicit ZeroOutFloatOp(OpKernelConstruction* context)
+      : OpKernel(context) {}
+
+  void Compute(OpKernelContext* context) override {
     // Grab the input tensor
-    const Tensor& input\_tensor = context->input(0);
-    auto input = input\_tensor.flat<float>();

+    const Tensor& input_tensor = context->input(0);
+    auto input = input_tensor.flat();
+
     // Create an output tensor
     Tensor* output = NULL;
-    OP\_REQUIRES\_OK(context,
-                   context->allocate\_output(0, input_tensor.shape(), &output));
-    auto output\_flat = output->template flat<float>();

+    OP_REQUIRES_OK(context,
+                   context->allocate_output(0, input_tensor.shape(), &output));
+    auto output_flat = output->template flat();
+
     // Set all the elements of the output tensor to 0
     const int N = input.size();
-    for (int i = 0; i < N; i++) {
-      output\_flat(i) = 0;
-    }

+    for (int i = 0; i < N; i++) {
+      output_flat(i) = 0;
+    }
+
     // Preserve the first input value
-    if (N > 0) output\_flat(0) = input(0);
+    if (N > 0) output_flat(0) = input(0);
   }
-};

-// Note that TypeConstraint<int32>("T") means that attr "T" (defined
+};
+
+// Note that TypeConstraint("T") means that attr "T" (defined
 // in the op registration above) must be "int32" to use this template
-// instantiation.
-REGISTER\_KERNEL\_BUILDER(
+// instantiation.
+REGISTER_KERNEL_BUILDER(
     Name("ZeroOut")
-    .Device(DEVICE\_CPU)
-    .TypeConstraint<int32>("T"),
-    ZeroOutOpInt32);
-REGISTER\_KERNEL\_BUILDER(
+    .Device(DEVICE_CPU)
+    .TypeConstraint("T"),
+    ZeroOutOpInt32);
+REGISTER_KERNEL_BUILDER(
     Name("ZeroOut")
-    .Device(DEVICE\_CPU)
-    .TypeConstraint<float>("T"),
+    .Device(DEVICE_CPU)
+    .TypeConstraint("T"),
     ZeroOutFloatOp);
-
+``` > To preserve [backwards compatibility](#backwards-compatibility), you should > specify a [default value](#default-values-constraints) when adding an attr to > an existing op: > -> 

-> REGISTER\_OP("ZeroOut")
->   .Attr("T: {float, int32} = DT_INT32")
->   .Input("to\_zero: T")
+> ```c++
+> REGISTER_OP("ZeroOut")
+>   .Attr("T: {float, int32} = DT_INT32")
+>   .Input("to_zero: T")
 >   .Output("zeroed: T")
->
+> ``` Let's say you wanted to add more types, say `double`: -

-REGISTER\_OP("ZeroOut")
-    .Attr("T: {float, double, int32}")
-    .Input("to\_zero: T")
-    .Output("zeroed: T");
-
+```c++ +REGISTER_OP("ZeroOut") + .Attr("T: {float, double, int32}") + .Input("to_zero: T") + .Output("zeroed: T"); +``` Instead of writing another `OpKernel` with redundant code as above, often you will be able to use a C++ template instead. You will still have one kernel -- GitLab From 8a5a41f72a8f48d2bb337aca018bf1216b17a07b Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 10 Apr 2018 18:38:06 -0700 Subject: [PATCH 139/791] Fix incorrect math equation renderings in random_fourier_features.py (#18367) * Fix incorrect math equation renderings in random_fourier_features.py This fix fixes incorrect math equation renderings for markdown in random_fourier_features.py. The issue is that "```" backtick should not be added when mathjax quote is used ("\\(" or "$$"). Signed-off-by: Yong Tang * Additional fix. Signed-off-by: Yong Tang * MathJax fixes Signed-off-by: Yong Tang * Fix pylint errors Signed-off-by: Yong Tang --- .../python/mappers/random_fourier_features.py | 42 +++++++++---------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features.py b/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features.py index 091f0a1098..9a721a9d44 100644 --- a/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features.py +++ b/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features.py @@ -34,33 +34,31 @@ class RandomFourierFeatureMapper(dkm.DenseKernelMapper): r"""Class that implements Random Fourier Feature Mapping (RFFM) in TensorFlow. The RFFM mapping is used to approximate the Gaussian (RBF) kernel: - ``` $$(exp(-||x-y||_2^2 / (2 * \sigma^2))$$ - ``` The implementation of RFFM is based on the following paper: "Random Features for Large-Scale Kernel Machines" by Ali Rahimi and Ben Recht. (link: https://people.eecs.berkeley.edu/~brecht/papers/07.rah.rec.nips.pdf) - The mapping uses a matrix `\\(Omega \in R^{d x D}\\)` and a bias vector - `\\(b \in R^D\\)` where `d` is the input dimension (number of dense input - features) and `D` is the output dimension (i.e., dimension of the feature - space the input is mapped to). Each entry of `Omega` is sampled i.i.d. from a - (scaled) Gaussian distribution and each entry of `b` is sampled independently - and uniformly from [0, \\(2 * pi\\)]. - - For a single input feature vector x in R^d, its RFFM is defined as: - ``` - $$sqrt(2/D) * cos(x * Omega + b)$$ - ``` - where `cos` is the element-wise cosine function and `x, b` are represented as - row vectors. The aforementioned paper shows that the linear kernel of - RFFM-mapped vectors approximates the Gaussian kernel of the initial vectors. + The mapping uses a matrix \\(\Omega \in R^{d x D}\\) and a bias vector + \\(b \in R^D\\) where \\(d\\) is the input dimension (number of dense input + features) and \\(D\\) is the output dimension (i.e., dimension of the feature + space the input is mapped to). Each entry of \\(\Omega\\) is sampled i.i.d. + from a (scaled) Gaussian distribution and each entry of \\(b\\) is sampled + independently and uniformly from [0, \\(2 * \pi\\)]. + + For a single input feature vector \\(x \in R^d\\), its RFFM is defined as: + $$\sqrt(2/D) * cos(x * \Omega + b)$$ + + where \\(cos\\) is the element-wise cosine function and \\(x, b\\) are + represented as row vectors. The aforementioned paper shows that the linear + kernel of RFFM-mapped vectors approximates the Gaussian kernel of the initial + vectors. """ def __init__(self, input_dim, output_dim, stddev=1.0, seed=1, name=None): - """Constructs a RandomFourierFeatureMapper instance. + r"""Constructs a RandomFourierFeatureMapper instance. Args: input_dim: The dimension (number of features) of the tensors to be mapped. @@ -68,11 +66,11 @@ class RandomFourierFeatureMapper(dkm.DenseKernelMapper): stddev: The standard deviation of the Gaussian kernel to be approximated. The error of the classifier trained using this approximation is very sensitive to this parameter. - seed: An integer used to initialize the parameters (`Omega` and `b`) of - the mapper. For repeatable sequences across different invocations of the - mapper object (for instance, to ensure consistent mapping both at - training and eval/inference if these happen in different invocations), - set this to the same integer. + seed: An integer used to initialize the parameters (\\(\Omega\\) and + \\(b\\)) of the mapper. For repeatable sequences across different + invocations of the mapper object (for instance, to ensure consistent + mapping both at training and eval/inference if these happen in + different invocations), set this to the same integer. name: name for the mapper object. """ # TODO(sibyl-vie3Poto): Maybe infer input_dim and/or output_dim (if not explicitly -- GitLab From fad74785d12ea7463e5d0474522cd7d754699656 Mon Sep 17 00:00:00 2001 From: Francois Chollet Date: Tue, 10 Apr 2018 18:41:37 -0700 Subject: [PATCH 140/791] Fix for users who were passing `Dimension` type as `units` arg in `Dense`. PiperOrigin-RevId: 192387984 --- tensorflow/python/keras/_impl/keras/layers/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/keras/_impl/keras/layers/core.py b/tensorflow/python/keras/_impl/keras/layers/core.py index 87b997232e..f64174a23f 100644 --- a/tensorflow/python/keras/_impl/keras/layers/core.py +++ b/tensorflow/python/keras/_impl/keras/layers/core.py @@ -836,7 +836,7 @@ class Dense(Layer): super(Dense, self).__init__( activity_regularizer=regularizers.get(activity_regularizer), **kwargs) - self.units = units + self.units = int(units) self.activation = activations.get(activation) self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) -- GitLab From 5ad9e4588874f30d0d079acc60e07f2eddc0480f Mon Sep 17 00:00:00 2001 From: Michael Case Date: Tue, 10 Apr 2018 18:44:13 -0700 Subject: [PATCH 141/791] Merge changes from github. PiperOrigin-RevId: 192388250 --- README.md | 3 +- RELEASE.md | 11 + configure.py | 1 - tensorflow/compiler/xla/tests/build_defs.bzl | 3 +- tensorflow/compiler/xla/tests/slice_test.cc | 39 +- .../notebooks/dev_summit_2018_demo.ipynb | 1919 +++++++++++++++++ .../bayesflow/python/ops/monte_carlo_impl.py | 39 +- .../python/training/tpu_cluster_resolver.py | 2 +- .../training/tpu_cluster_resolver_test.py | 8 +- tensorflow/contrib/cmake/python_modules.txt | 1 + .../cudnn_rnn/python/ops/cudnn_rnn_ops.py | 16 +- .../kernel_tests/sequence_dataset_op_test.py | 6 + .../contrib/data/python/ops/resampling.py | 1 + .../distribute/python/cross_tower_ops.py | 4 +- .../distribute/python/cross_tower_utils.py | 2 +- .../python/shared_variable_creator.py | 2 +- .../bijectors/kumaraswamy_bijector_test.py | 2 +- .../distributions/python/ops/estimator.py | 2 +- .../distributions/python/ops/independent.py | 2 +- .../python/ops/onehot_categorical.py | 4 +- .../python/ops/relaxed_bernoulli.py | 8 +- .../python/ops/relaxed_onehot_categorical.py | 2 +- .../python/ops/vector_student_t.py | 2 +- .../python/ops/clustering_ops.py | 11 +- .../python/ops/factorization_ops.py | 71 +- .../factorization/python/ops/gmm_ops.py | 4 +- .../factorization/python/ops/kmeans.py | 8 +- .../contrib/factorization/python/ops/wals.py | 6 +- .../estimator/python/gan_estimator_impl.py | 4 +- .../gan/python/losses/python/losses_impl.py | 14 +- .../python/losses/python/losses_impl_test.py | 22 + tensorflow/contrib/gan/python/train.py | 4 + tensorflow/contrib/gan/python/train_test.py | 25 +- .../contrib/layers/python/layers/layers.py | 14 +- .../python/ops/linear_operator_block_diag.py | 3 +- .../unpartition_embedding_lookup.cc | 2 +- tensorflow/contrib/lite/toco/python/BUILD | 3 - .../contrib/lite/toco/python/toco_wrapper.py | 13 +- tensorflow/contrib/lookup/lookup_ops.py | 2 +- .../kernel_tests/attention_wrapper_test.py | 36 + .../seq2seq/python/ops/attention_wrapper.py | 3 +- .../contrib/tensorrt/convert/convert_nodes.cc | 9 +- .../timeseries/python/timeseries/BUILD | 1 + tensorflow/contrib/tpu/tpu_estimator.md | 2 +- .../training/python/training/evaluation.py | 10 +- .../python/training/evaluation_test.py | 16 +- tensorflow/contrib/verbs/rdma.h | 2 +- .../common_runtime/scoped_allocator_mgr.cc | 2 +- .../core/kernels/mkl_input_conversion_op.cc | 52 +- tensorflow/core/kernels/mkl_softmax_op.cc | 2 +- .../core/kernels/reduction_gpu_kernels.cu.h | 37 +- .../core/kernels/segment_reduction_ops.h | 8 + tensorflow/core/ops/dataset_ops.cc | 7 +- tensorflow/core/ops/nn_ops.cc | 26 +- tensorflow/core/public/version.h | 2 +- .../api_guides/python/contrib.graph_editor.md | 18 +- .../docs_src/api_guides/python/io_ops.md | 4 +- tensorflow/docs_src/api_guides/python/nn.md | 18 +- tensorflow/docs_src/get_started/index.md | 21 +- tensorflow/docs_src/get_started/leftnav_files | 5 +- .../get_started/premade_estimators.md | 2 +- tensorflow/docs_src/install/install_c.md | 2 +- tensorflow/docs_src/install/install_go.md | 2 +- tensorflow/docs_src/install/install_java.md | 22 +- tensorflow/docs_src/install/install_linux.md | 51 +- tensorflow/docs_src/install/install_mac.md | 10 +- .../docs_src/install/install_sources.md | 14 +- .../docs_src/programmers_guide/using_tpu.md | 10 +- tensorflow/docs_src/tutorials/layers.md | 54 +- tensorflow/java/BUILD | 3 + tensorflow/java/src/gen/cc/java_defs.h | 45 +- tensorflow/java/src/gen/cc/source_writer.cc | 305 ++- tensorflow/java/src/gen/cc/source_writer.h | 192 +- .../java/src/gen/cc/source_writer_test.cc | 369 +++- .../java/src/gen/resources/test.java.snippet | 2 + tensorflow/python/client/timeline_test.py | 5 +- .../python/eager/execution_callbacks.py | 2 +- .../python/kernel_tests/init_ops_test.py | 2 +- tensorflow/python/ops/control_flow_ops.py | 3 + tensorflow/python/ops/ctc_ops.py | 4 +- tensorflow/python/ops/custom_gradient.py | 2 +- tensorflow/python/ops/data_flow_ops.py | 11 +- .../python/ops/linalg/linear_operator.py | 3 +- .../ops/linalg/linear_operator_composition.py | 3 +- .../python/ops/linalg/linear_operator_diag.py | 3 +- .../ops/linalg/linear_operator_full_matrix.py | 3 +- .../ops/linalg/linear_operator_identity.py | 6 +- .../linear_operator_lower_triangular.py | 3 +- tensorflow/python/training/distribute.py | 2 +- tensorflow/python/training/session_manager.py | 10 +- .../tools/ci_build/install/install_golang.sh | 2 +- .../ci_build/windows/bazel/bazel_test_lib.sh | 4 +- .../tools/pip_package/build_pip_package.sh | 4 +- tensorflow/tools/pip_package/setup.py | 2 +- 94 files changed, 3314 insertions(+), 409 deletions(-) create mode 100644 tensorflow/contrib/autograph/examples/notebooks/dev_summit_2018_demo.ipynb create mode 100644 tensorflow/java/src/gen/resources/test.java.snippet diff --git a/README.md b/README.md index a69cf1ffea..29418dc2e9 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ **TensorFlow** is an open source software library for numerical computation using data flow graphs. The graph nodes represent mathematical operations, while the graph edges represent the multidimensional data arrays (tensors) that flow -between them. This flexible architecture lets you deploy computation to one +between them. This flexible architecture enables you to deploy computation to one or more CPUs or GPUs in a desktop, server, or mobile device without rewriting code. TensorFlow also includes TensorBoard, a data visualization toolkit. @@ -86,6 +86,7 @@ The TensorFlow project strives to abide by generally accepted best practices in * [TensorFlow Website](https://www.tensorflow.org) * [TensorFlow White Papers](https://www.tensorflow.org/about/bib) +* [TensorFlow YouTube Channel](https://www.youtube.com/channel/UC0rqucBdTuFTjJiefW5t-IQ) * [TensorFlow Model Zoo](https://github.com/tensorflow/models) * [TensorFlow MOOC on Udacity](https://www.udacity.com/course/deep-learning--ud730) * [TensorFlow Course at Stanford](https://web.stanford.edu/class/cs20si) diff --git a/RELEASE.md b/RELEASE.md index c63d9f20c9..e845953174 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -9,6 +9,8 @@ * Distributed Mutex / CriticalSection added to `tf.contrib.framework.CriticalSection`. * Better text processing with `tf.regex_replace`. * Easy, efficient sequence input with `tf.contrib.data.bucket_by_sequence_length` +* Initial support for `tf.contrib.tensorrt` that enables native TensorRT in + TensorFlow. ## Bug Fixes and Other Changes * Accelerated Linear Algebra (XLA): @@ -50,6 +52,15 @@ * Support `float16` `dtype` in `tf.linalg.*`. * Add `tf.estimator.export.TensorServingInputReceiver` that allows `tf.estimator.Estimator.export_savedmodel` to pass raw tensors to model functions. +## Deprecations + +* TensorFlow 1.7 may be the last time we support Cuda versions below 8.0. + Starting with TensorFlow 1.8 release, 8.0 will be the minimum supported + version. +* TensorFlow 1.7 may be the last time we support cuDNN versions below 6.0. + Starting with TensorFlow 1.8 release, 6.0 will be the minimum supported + version. + ## Thanks to our Contributors This release contains contributions from many people at Google, as well as: diff --git a/configure.py b/configure.py index da3f97ab30..81d5ad77ee 100644 --- a/configure.py +++ b/configure.py @@ -505,7 +505,6 @@ def set_cc_opt_flags(environ_cp): write_to_bazelrc('build --copt=-DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK') write_to_bazelrc('build --host_copt=-DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK') - def set_tf_cuda_clang(environ_cp): """set TF_CUDA_CLANG action_env. diff --git a/tensorflow/compiler/xla/tests/build_defs.bzl b/tensorflow/compiler/xla/tests/build_defs.bzl index 610302ac12..eac2eb286c 100644 --- a/tensorflow/compiler/xla/tests/build_defs.bzl +++ b/tensorflow/compiler/xla/tests/build_defs.bzl @@ -137,7 +137,8 @@ def xla_test(name, backend_deps += ["//tensorflow/compiler/xla/tests:test_macros_gpu"] this_backend_tags += ["requires-gpu-sm35"] elif backend in plugins: - backend_deps = plugins[backend]["deps"] + backend_deps = [] + backend_deps += plugins[backend]["deps"] this_backend_copts += plugins[backend]["copts"] this_backend_tags += plugins[backend]["tags"] this_backend_args += plugins[backend]["args"] diff --git a/tensorflow/compiler/xla/tests/slice_test.cc b/tensorflow/compiler/xla/tests/slice_test.cc index 8d9a9c7b73..52195db2aa 100644 --- a/tensorflow/compiler/xla/tests/slice_test.cc +++ b/tensorflow/compiler/xla/tests/slice_test.cc @@ -214,6 +214,9 @@ class SliceR1Test : public ClientLibraryTestBase, } }; +// A version of SliceR1Test used to label and disable 'large' tests +class SliceR1LargeTest : public SliceR1Test {}; + string SliceR1TestDataToString(const ::testing::TestParamInfo& data) { const R1Spec& spec = data.param; return ::tensorflow::strings::Printf("%lld_%lld_%lld_%lld", spec.input_dim0, @@ -233,8 +236,21 @@ XLA_TEST_P(SliceR1Test, DoIt_U64) { Run(GetParam()); } XLA_TEST_P(SliceR1Test, DoIt_S64) { Run(GetParam()); } +XLA_TEST_P(SliceR1LargeTest, DoIt_F32) { Run(GetParam()); } + +XLA_TEST_P(SliceR1LargeTest, DoIt_F64) { Run(GetParam()); } + +XLA_TEST_P(SliceR1LargeTest, DoIt_U32) { Run(GetParam()); } + +XLA_TEST_P(SliceR1LargeTest, DoIt_S32) { Run(GetParam()); } + +XLA_TEST_P(SliceR1LargeTest, DoIt_U64) { Run(GetParam()); } + +XLA_TEST_P(SliceR1LargeTest, DoIt_S64) { Run(GetParam()); } + XLA_TEST_P(SliceR1Test, DoIt_PRED) { Run(GetParam()); } + // Tests for R1 slice ops. // The format for each testcase is {input size, start, limit, stride}. // clang-format off @@ -242,12 +258,6 @@ INSTANTIATE_TEST_CASE_P( SliceR1TestInstantiation, SliceR1Test, ::testing::Values( -// TODO(b/69425338): This uses too much memory on GPU. -#ifndef XLA_TEST_BACKEND_GPU - R1Spec{16 * 1024 * 1024, 4 * 1024 * 1024, 12 * 1024 * 1024, 1}, - R1Spec{16 * 1024 * 1024, 4 * 1024 * 1024 + 1, 12 * 1024 * 1024 - 1, 1}, - R1Spec{16 * 1024 * 1024, 4 * 1024 * 1024 - 1, 12 * 1024 * 1024 + 1, 1}, -#endif R1Spec{10, 0, 0, 1}, R1Spec{10, 7, 7, 1}, R1Spec{10, 0, 5, 1}, @@ -283,6 +293,23 @@ INSTANTIATE_TEST_CASE_P( SliceR1TestDataToString ); +// TODO(b/69425338): This uses too much memory on GPU. +#ifndef XLA_TEST_BACKEND_GPU +INSTANTIATE_TEST_CASE_P( + SliceR1TestBigSlicesInstantiation, + SliceR1LargeTest, + ::testing::Values( + R1Spec{ + 16 * 1024 * 1024, 4 * 1024 * 1024, 12 * 1024 * 1024, 1}, + R1Spec{ + 16 * 1024 * 1024, 4 * 1024 * 1024 + 1, 12 * 1024 * 1024 - 1, 1}, + R1Spec{ + 16 * 1024 * 1024, 4 * 1024 * 1024 - 1, 12 * 1024 * 1024 + 1, 1} + ), + SliceR1TestDataToString +); +#endif + INSTANTIATE_TEST_CASE_P( SliceStridedR1TestInstantiation, SliceR1Test, diff --git a/tensorflow/contrib/autograph/examples/notebooks/dev_summit_2018_demo.ipynb b/tensorflow/contrib/autograph/examples/notebooks/dev_summit_2018_demo.ipynb new file mode 100644 index 0000000000..d62390494b --- /dev/null +++ b/tensorflow/contrib/autograph/examples/notebooks/dev_summit_2018_demo.ipynb @@ -0,0 +1,1919 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "Dev Summit 2018 - Autograph", + "version": "0.3.2", + "views": {}, + "default_view": {}, + "provenance": [ + { + "file_id": "1wCZUh73zTNs1jzzYjqoxMIdaBWCdKJ2K", + "timestamp": 1522238054357 + }, + { + "file_id": "1_HpC-RrmIv4lNaqeoslUeWaX8zH5IXaJ", + "timestamp": 1521743157199 + }, + { + "file_id": "1mjO2fQ2F9hxpAzw2mnrrUkcgfb7xSGW-", + "timestamp": 1520522344607 + } + ], + "collapsed_sections": [] + }, + "kernelspec": { + "name": "python2", + "display_name": "Python 2" + } + }, + "cells": [ + { + "metadata": { + "id": "g7nGs4mzVUHP", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Experimental: TF Autograph\n", + "**TensorFlow Dev Summit, 2018.**\n", + "\n", + "This interactive notebook demonstrates **autograph**, an experimental source-code transformation library to automatically convert TF.Eager and Python code to TensorFlow graphs.\n", + "\n", + "**Note: this is pre-alpha software!** The notebook works best with Python 2, for now.\n", + "\n", + "> ![alt text](https://lh3.googleusercontent.com/QOvy0clmg7siaVKzwmSPAjicWWNQ0OeyaB16plDjSJMf35WD3vLjF6mz4CGrhSHw60HnlZPJjkyDCBzw5XOI0oBGSewyYw=s688)\n", + "\n", + "### Table of Contents\n", + "1. _Write Eager code that is fast and scalable._\n", + "2. _Case study: complex control flow._\n", + "3. _Case study: training MNIST with Keras._\n", + "4. _Case study: building an RNN._" + ] + }, + { + "metadata": { + "id": "uFcgBENZqkB2", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "# Install TensorFlow; note that Colab notebooks run remotely, on virtual\n", + "# instances provided by Google.\n", + "!pip install -U -q tf-nightly" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "Pa2qpEmoVOGe", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "import os\n", + "import time\n", + "\n", + "import tensorflow as tf\n", + "from tensorflow.contrib import autograph\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import six\n", + "\n", + "from google.colab import widgets" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "ZVKfj5ttVkqz", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# 1. Write Eager code that is fast and scalable\n", + "\n", + "TF.Eager gives you more flexibility while coding, but at the cost of losing the benefits of TensorFlow graphs. For example, Eager does not currently support distributed training, exporting models, and a variety of memory and computation optimizations.\n", + "\n", + "Autograph gives you the best of both worlds: write your code in an Eager style, and we will automatically transform it into the equivalent TF graph code. The graph code can be executed eagerly (as a single op), included as part of a larger graph, or exported." + ] + }, + { + "metadata": { + "id": "snaZRFdWd9ym", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "For example, autograph can convert a function like this:" + ] + }, + { + "metadata": { + "id": "9__n8cSIeDnD", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "def g(x):\n", + " if x > 0:\n", + " x = x * x\n", + " else:\n", + " x = 0\n", + " return x" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "gq0eQcuReHET", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "... into a TF graph-building function:" + ] + }, + { + "metadata": { + "id": "sELSn599ePUF", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "output_extras": [ + {} + ], + "base_uri": "https://localhost:8080/", + "height": 413 + }, + "outputId": "bb0c7216-1ca3-4da1-d1fb-589902cdcd1a", + "executionInfo": { + "status": "ok", + "timestamp": 1522345737505, + "user_tz": 240, + "elapsed": 243, + "user": { + "displayName": "Dan Moldovan", + "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg", + "userId": "112023154726779574577" + } + } + }, + "cell_type": "code", + "source": [ + "print(autograph.to_code(g))" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "from __future__ import print_function\n", + "import tensorflow as tf\n", + "from tensorflow.contrib.autograph.impl import api as autograph_api\n", + "from tensorflow.contrib.autograph import utils as autograph_utils\n", + "\n", + "def tf__g(x):\n", + " with tf.name_scope('g'):\n", + "\n", + " def if_true():\n", + " with tf.name_scope('if_true'):\n", + " x_1, = x,\n", + " x_1 = x_1 * x_1\n", + " return x_1,\n", + "\n", + " def if_false():\n", + " with tf.name_scope('if_false'):\n", + " x_1, = x,\n", + " x_1 = 0\n", + " return x_1,\n", + " x = autograph_utils.run_cond(tf.greater(x, 0), if_true, if_false)\n", + " return x\n", + "\n" + ], + "name": "stdout" + } + ] + }, + { + "metadata": { + "id": "j74n-8hEe6dk", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "You can then use the converted function as you would any regular TF op -- you can pass `Tensor` arguments and it will return `Tensor`s:" + ] + }, + { + "metadata": { + "id": "AkVaY0-dfEbH", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "output_extras": [ + {} + ], + "base_uri": "https://localhost:8080/", + "height": 53 + }, + "outputId": "4ffe3757-c44d-424c-c2a8-7ddc973bfcce", + "executionInfo": { + "status": "ok", + "timestamp": 1522345737841, + "user_tz": 240, + "elapsed": 257, + "user": { + "displayName": "Dan Moldovan", + "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg", + "userId": "112023154726779574577" + } + } + }, + "cell_type": "code", + "source": [ + "tf_g = autograph.to_graph(g)\n", + "\n", + "with tf.Graph().as_default(): \n", + "\n", + " g_ops = tf_g(tf.constant(9))\n", + "\n", + " with tf.Session() as sess:\n", + " tf_g_result = sess.run(g_ops)\n", + "\n", + " print('g(9) = %s' % g(9))\n", + " print('tf_g(9) = %s' % tf_g_result)" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "g(9) = 81\n", + "tf_g(9) = 81\n" + ], + "name": "stdout" + } + ] + }, + { + "metadata": { + "id": "trrHQBM1VnD0", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# 2. Case study: complex control flow\n", + "\n", + "Autograph can convert a large chunk of the Python language into graph-equivalent code, and we're adding new supported language features all the time. In this section, we'll give you a taste of some of the functionality in autograph.\n", + "Autograph will automatically convert most Python control flow statements into their correct graph equivalent.\n", + " " + ] + }, + { + "metadata": { + "id": "u0YG3DPgZxoW", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "We support common statements like `while`, `for`, `if`, `break`, `return` and more. You can even nest them as much as you like. Imagine trying to write the graph version of this code by hand:" + ] + }, + { + "metadata": { + "id": "xJYDzOcrZ8pI", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "output_extras": [ + {} + ], + "base_uri": "https://localhost:8080/", + "height": 35 + }, + "outputId": "6c244ee4-b141-4ad6-eefa-cfffa71f33c6", + "executionInfo": { + "status": "ok", + "timestamp": 1522345738402, + "user_tz": 240, + "elapsed": 483, + "user": { + "displayName": "Dan Moldovan", + "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg", + "userId": "112023154726779574577" + } + } + }, + "cell_type": "code", + "source": [ + "def sum_even(numbers):\n", + " s = 0\n", + " for n in numbers:\n", + " if n % 2 > 0:\n", + " continue\n", + " s += n\n", + " return s\n", + "\n", + "\n", + "tf_sum_even = autograph.to_graph(sum_even)\n", + "\n", + "with tf.Graph().as_default(): \n", + " with tf.Session() as sess:\n", + " result = sess.run(tf_sum_even(tf.constant([10, 12, 15, 20])))\n", + "\n", + " print('Sum of even numbers: %s' % result)\n", + " \n", + "# Uncomment the line below to print the generated graph code\n", + "# print(autograph.to_code(sum_even))" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Sum of even numbers: 42\n" + ], + "name": "stdout" + } + ] + }, + { + "metadata": { + "id": "_YXo4KOcbKrn", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Try replacing the `continue` in the above code with `break` -- Autograph supports that as well!" + ] + }, + { + "metadata": { + "id": "xHmC0rBIavW_", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "The Python code above is much more readable than the matching graph code. Autograph takes care of tediously converting every piece of Python code into the matching TensorFlow graph version for you, so that you can quickly write maintainable code, but still benefit from the optimizations and deployment benefits of graphs." + ] + }, + { + "metadata": { + "id": "UEHWGpBXbS7g", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Let's try some other useful Python constructs, like `print` and `assert`. We automatically convert Python `assert` statements into the equivalent `tf.Assert` code. " + ] + }, + { + "metadata": { + "id": "qUU57xlEbauI", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "output_extras": [ + {} + ], + "base_uri": "https://localhost:8080/", + "height": 53 + }, + "outputId": "add3db4a-2077-4dd5-f7a7-a5b5a4529c26", + "executionInfo": { + "status": "ok", + "timestamp": 1522345738697, + "user_tz": 240, + "elapsed": 253, + "user": { + "displayName": "Dan Moldovan", + "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg", + "userId": "112023154726779574577" + } + } + }, + "cell_type": "code", + "source": [ + "def f(x):\n", + " assert x != 0, 'Do not pass zero!'\n", + " return x * x\n", + "\n", + "tf_f = autograph.to_graph(f)\n", + "with tf.Graph().as_default(): \n", + " with tf.Session() as sess:\n", + " try:\n", + " print(sess.run(tf_f(tf.constant(0))))\n", + " except tf.errors.InvalidArgumentError as e:\n", + " print('Got error message: %s' % e.message)\n", + " \n", + "# Uncomment the line below to print the generated graph code\n", + "# print(autograph.to_code(f))" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Got error message: assertion failed: [Do not pass zero!]\n", + "\t [[Node: f/Assert/Assert = Assert[T=[DT_STRING], summarize=3, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](f/NotEqual, f/Assert/Assert/data_0)]]\n" + ], + "name": "stdout" + } + ] + }, + { + "metadata": { + "id": "w5hBZaVJbck4", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "You can also use `print` functions in-graph:" + ] + }, + { + "metadata": { + "id": "6NdzRKLEboRv", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "output_extras": [ + {} + ], + "base_uri": "https://localhost:8080/", + "height": 35 + }, + "outputId": "fb82dfc3-790f-4127-87f6-361805be9e9b", + "executionInfo": { + "status": "ok", + "timestamp": 1522345739013, + "user_tz": 240, + "elapsed": 247, + "user": { + "displayName": "Dan Moldovan", + "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg", + "userId": "112023154726779574577" + } + } + }, + "cell_type": "code", + "source": [ + "def print_sign(n):\n", + " if n >= 0:\n", + " print(n, 'is positive!')\n", + " else:\n", + " print(n, 'is negative!')\n", + " return n\n", + "\n", + "\n", + "tf_print_sign = autograph.to_graph(print_sign)\n", + "with tf.Graph().as_default():\n", + " with tf.Session() as sess:\n", + " sess.run(tf_print_sign(tf.constant(1)))\n", + " \n", + "# Uncomment the line below to print the generated graph code\n", + "# print(autograph.to_code(print_sign))" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "1 is positive!\n" + ], + "name": "stdout" + } + ] + }, + { + "metadata": { + "id": "9u_Z3i3AivLA", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "We can convert lists to TensorArray, so appending to lists also works, with a few modifications:" + ] + }, + { + "metadata": { + "id": "MjhCQJVuiTNR", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "output_extras": [ + {} + ], + "base_uri": "https://localhost:8080/", + "height": 35 + }, + "outputId": "dc320b87-595b-4392-d29c-994486fd8a0a", + "executionInfo": { + "status": "ok", + "timestamp": 1522345744470, + "user_tz": 240, + "elapsed": 5391, + "user": { + "displayName": "Dan Moldovan", + "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg", + "userId": "112023154726779574577" + } + } + }, + "cell_type": "code", + "source": [ + "def f(n):\n", + " numbers = []\n", + " # We ask you to tell us about the element dtype.\n", + " autograph.utils.set_element_type(numbers, tf.int32)\n", + " for i in range(n):\n", + " numbers.append(i)\n", + " return numbers.stack() # Stack the list so that it can be used as a Tensor\n", + "\n", + "\n", + "tf_f = autograph.to_graph(f)\n", + "with tf.Graph().as_default():\n", + " with tf.Session() as sess:\n", + " print(sess.run(tf_f(tf.constant(5))))\n", + " \n", + "# Uncomment the line below to print the generated graph code\n", + "# print(autograph.to_code(f))" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "[0 1 2 3 4]\n" + ], + "name": "stdout" + } + ] + }, + { + "metadata": { + "id": "UdG8ZFrkTAF2", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "And all of these functionalities, and more, can be composed into more complicated code:\n" + ] + }, + { + "metadata": { + "id": "DVs6wt8NKaGQ", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "output_extras": [ + {} + ], + "base_uri": "https://localhost:8080/", + "height": 53 + }, + "cellView": "code", + "outputId": "0a4b8d08-8f65-4bbc-85ba-dc4c60563519", + "executionInfo": { + "status": "ok", + "timestamp": 1522345745186, + "user_tz": 240, + "elapsed": 658, + "user": { + "displayName": "Dan Moldovan", + "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg", + "userId": "112023154726779574577" + } + } + }, + "cell_type": "code", + "source": [ + "def print_primes(n):\n", + " \"\"\"Returns all the prime numbers less than n.\"\"\"\n", + " assert n > 0\n", + " \n", + " primes = []\n", + " autograph.utils.set_element_type(primes, tf.int32)\n", + " for i in range(2, n):\n", + " is_prime = True\n", + " for k in range(2, i):\n", + " if i % k == 0:\n", + " is_prime = False\n", + " break\n", + " if not is_prime:\n", + " continue\n", + " primes.append(i)\n", + " all_primes = primes.stack()\n", + "\n", + " print('The prime numbers less than', n, 'are:')\n", + " print(all_primes)\n", + " return tf.no_op()\n", + "\n", + " \n", + "tf_print_primes = autograph.to_graph(print_primes)\n", + "with tf.Graph().as_default(): \n", + " with tf.Session() as sess:\n", + " n = tf.constant(50)\n", + " sess.run(tf_print_primes(n))\n", + " \n", + "# Uncomment the line below to print the generated graph code\n", + "# print(autograph.to_code(print_primes))" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "The prime numbers less than 50 are:\n", + "[ 2 3 5 7 11 13 17 19 23 29 31 37 41 43 47]\n" + ], + "name": "stdout" + } + ] + }, + { + "metadata": { + "id": "JQ8kQT99VqDk", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# 3. Case study: training MNIST with Keras\n", + "\n", + "As we've seen, writing control flow in Autograph is easy. So running a training loop in graph should be easy as well!\n", + "\n", + "Here, we show an example of such a training loop for a simple Keras model that trains on MNIST." + ] + }, + { + "metadata": { + "id": "0CrtGWgwuLJr", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "import gzip\n", + "import shutil\n", + "\n", + "from six.moves import urllib\n", + "\n", + "\n", + "def download(directory, filename):\n", + " filepath = os.path.join(directory, filename)\n", + " if tf.gfile.Exists(filepath):\n", + " return filepath\n", + " if not tf.gfile.Exists(directory):\n", + " tf.gfile.MakeDirs(directory)\n", + " url = 'https://storage.googleapis.com/cvdf-datasets/mnist/' + filename + '.gz'\n", + " zipped_filepath = filepath + '.gz'\n", + " print('Downloading %s to %s' % (url, zipped_filepath))\n", + " urllib.request.urlretrieve(url, zipped_filepath)\n", + " with gzip.open(zipped_filepath, 'rb') as f_in, open(filepath, 'wb') as f_out:\n", + " shutil.copyfileobj(f_in, f_out)\n", + " os.remove(zipped_filepath)\n", + " return filepath\n", + "\n", + "\n", + "def dataset(directory, images_file, labels_file):\n", + " images_file = download(directory, images_file)\n", + " labels_file = download(directory, labels_file)\n", + "\n", + " def decode_image(image):\n", + " # Normalize from [0, 255] to [0.0, 1.0]\n", + " image = tf.decode_raw(image, tf.uint8)\n", + " image = tf.cast(image, tf.float32)\n", + " image = tf.reshape(image, [784])\n", + " return image / 255.0\n", + "\n", + " def decode_label(label):\n", + " label = tf.decode_raw(label, tf.uint8)\n", + " label = tf.reshape(label, [])\n", + " return tf.to_int32(label)\n", + "\n", + " images = tf.data.FixedLengthRecordDataset(\n", + " images_file, 28 * 28, header_bytes=16).map(decode_image)\n", + " labels = tf.data.FixedLengthRecordDataset(\n", + " labels_file, 1, header_bytes=8).map(decode_label)\n", + " return tf.data.Dataset.zip((images, labels))\n", + "\n", + "\n", + "def mnist_train(directory):\n", + " return dataset(directory, 'train-images-idx3-ubyte',\n", + " 'train-labels-idx1-ubyte')\n", + "\n", + "def mnist_test(directory):\n", + " return dataset(directory, 't10k-images-idx3-ubyte', 't10k-labels-idx1-ubyte')" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "2zu1U9Nqir6L", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "First, we'll define a small three-layer neural network using the Keras API" + ] + }, + { + "metadata": { + "id": "x_MU13boiok2", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "def mlp_model(input_shape):\n", + " model = tf.keras.Sequential([\n", + " tf.keras.layers.Dense(100, activation='relu', input_shape=input_shape),\n", + " tf.keras.layers.Dense(100, activation='relu'),\n", + " tf.keras.layers.Dense(10, activation='softmax')])\n", + " model.build()\n", + " return model" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "Wuqg3H8mi0Xj", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Let's connect the model definition (here abbreviated as `m`) to a loss function, so that we can train our model." + ] + }, + { + "metadata": { + "id": "W51sfbONiz_5", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "def predict(m, x, y):\n", + " y_p = m(x)\n", + " losses = tf.keras.losses.categorical_crossentropy(y, y_p)\n", + " l = tf.reduce_mean(losses)\n", + " accuracies = tf.keras.metrics.categorical_accuracy(y, y_p)\n", + " accuracy = tf.reduce_mean(accuracies)\n", + " return l, accuracy" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "035tNWQki9tr", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Now the final piece of the problem specification (before loading data, and clicking everything together) is backpropagating the loss through the model, and optimizing the weights using the gradient." + ] + }, + { + "metadata": { + "id": "CsAD0ajbi9iZ", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "def fit(m, x, y, opt):\n", + " l, accuracy = predict(m, x, y)\n", + " opt.minimize(l)\n", + " return l, accuracy" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "PcVRIacKjSwb", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "These are some utility functions to download data and generate batches for training" + ] + }, + { + "metadata": { + "id": "RVw57HdTjPzi", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "def setup_mnist_data(is_training, hp, batch_size):\n", + " if is_training:\n", + " ds = mnist_train('/tmp/autograph_mnist_data')\n", + " ds = ds.shuffle(batch_size * 10)\n", + " else:\n", + " ds = mnist_test('/tmp/autograph_mnist_data')\n", + " ds = ds.repeat()\n", + " ds = ds.batch(batch_size)\n", + " return ds\n", + "\n", + "def get_next_batch(ds):\n", + " itr = ds.make_one_shot_iterator()\n", + " image, label = itr.get_next()\n", + " x = tf.to_float(tf.reshape(image, (-1, 28 * 28)))\n", + " y = tf.one_hot(tf.squeeze(label), 10)\n", + " return x, y" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "2zEJH5XNjgFz", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "This function specifies the main training loop. We instantiate the model (using the code above), instantiate an optimizer (here we'll use SGD with momentum, nothing too fancy), and we'll instantiate some lists to keep track of training and test loss and accuracy over time.\n", + "\n", + "In the loop inside this function, we'll grab a batch of data, apply an update to the weights of our model to improve its performance, and then record its current training loss and accuracy. Every so often, we'll log some information about training as well." + ] + }, + { + "metadata": { + "id": "UUI0566FjZPx", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "def train(train_ds, test_ds, hp):\n", + " m = mlp_model((28 * 28,))\n", + " opt = tf.train.MomentumOptimizer(hp.learning_rate, 0.9)\n", + " train_losses = []\n", + " train_losses = autograph.utils.set_element_type(train_losses, tf.float32)\n", + " test_losses = []\n", + " test_losses = autograph.utils.set_element_type(test_losses, tf.float32)\n", + " train_accuracies = []\n", + " train_accuracies = autograph.utils.set_element_type(train_accuracies,\n", + " tf.float32)\n", + " test_accuracies = []\n", + " test_accuracies = autograph.utils.set_element_type(test_accuracies,\n", + " tf.float32)\n", + " i = tf.constant(0)\n", + " while i < hp.max_steps:\n", + " train_x, train_y = get_next_batch(train_ds)\n", + " test_x, test_y = get_next_batch(test_ds)\n", + " step_train_loss, step_train_accuracy = fit(m, train_x, train_y, opt)\n", + " step_test_loss, step_test_accuracy = predict(m, test_x, test_y)\n", + " if i % (hp.max_steps // 10) == 0:\n", + " print('Step', i, 'train loss:', step_train_loss, 'test loss:',\n", + " step_test_loss, 'train accuracy:', step_train_accuracy,\n", + " 'test accuracy:', step_test_accuracy)\n", + " train_losses.append(step_train_loss)\n", + " test_losses.append(step_test_loss)\n", + " train_accuracies.append(step_train_accuracy)\n", + " test_accuracies.append(step_test_accuracy)\n", + " i += 1\n", + " return (train_losses.stack(), test_losses.stack(), train_accuracies.stack(),\n", + " test_accuracies.stack())" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "cYiUQ1ppkHzk", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Everything is ready to go, let's train the model and plot its performance!" + ] + }, + { + "metadata": { + "id": "K1m8TwOKjdNd", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "output_extras": [ + {}, + {}, + {} + ], + "base_uri": "https://localhost:8080/", + "height": 988 + }, + "outputId": "f9d3eef3-5bea-45c1-ddf9-4edee73e4436", + "executionInfo": { + "status": "ok", + "timestamp": 1522345800262, + "user_tz": 240, + "elapsed": 52391, + "user": { + "displayName": "Dan Moldovan", + "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg", + "userId": "112023154726779574577" + } + } + }, + "cell_type": "code", + "source": [ + "with tf.Graph().as_default():\n", + " hp = tf.contrib.training.HParams(\n", + " learning_rate=0.05,\n", + " max_steps=500,\n", + " )\n", + " train_ds = setup_mnist_data(True, hp, 50)\n", + " test_ds = setup_mnist_data(False, hp, 1000)\n", + " tf_train = autograph.to_graph(train)\n", + " (train_losses, test_losses, train_accuracies,\n", + " test_accuracies) = tf_train(train_ds, test_ds, hp)\n", + "\n", + " with tf.Session() as sess:\n", + " sess.run(tf.global_variables_initializer())\n", + " (train_losses, test_losses, train_accuracies,\n", + " test_accuracies) = sess.run([train_losses, test_losses, train_accuracies,\n", + " test_accuracies])\n", + " plt.title('MNIST train/test losses')\n", + " plt.plot(train_losses, label='train loss')\n", + " plt.plot(test_losses, label='test loss')\n", + " plt.legend()\n", + " plt.xlabel('Training step')\n", + " plt.ylabel('Loss')\n", + " plt.show()\n", + " plt.title('MNIST train/test accuracies')\n", + " plt.plot(train_accuracies, label='train accuracy')\n", + " plt.plot(test_accuracies, label='test accuracy')\n", + " plt.legend(loc='lower right')\n", + " plt.xlabel('Training step')\n", + " plt.ylabel('Accuracy')\n", + " plt.show()" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Downloading https://storage.googleapis.com/cvdf-datasets/mnist/train-images-idx3-ubyte.gz to /tmp/autograph_mnist_data/train-images-idx3-ubyte.gz\n", + "Downloading https://storage.googleapis.com/cvdf-datasets/mnist/train-labels-idx1-ubyte.gz to /tmp/autograph_mnist_data/train-labels-idx1-ubyte.gz\n", + "Downloading https://storage.googleapis.com/cvdf-datasets/mnist/t10k-images-idx3-ubyte.gz to /tmp/autograph_mnist_data/t10k-images-idx3-ubyte.gz\n", + "Downloading https://storage.googleapis.com/cvdf-datasets/mnist/t10k-labels-idx1-ubyte.gz to /tmp/autograph_mnist_data/t10k-labels-idx1-ubyte.gz\n", + "Step 0 train loss: 2.244329 test loss: 2.2499208 train accuracy: 0.12 test accuracy: 0.161\n", + "Step 50 train loss: 0.64771986 test loss: 0.56013924 train accuracy: 0.82 test accuracy: 0.836\n", + "Step 100 train loss: 0.49011207 test loss: 0.42143965 train accuracy: 0.84 test accuracy: 0.879\n", + "Step 150 train loss: 0.3768609 test loss: 0.39319593 train accuracy: 0.88 test accuracy: 0.883\n", + "Step 200 train loss: 0.36007702 test loss: 0.37089333 train accuracy: 0.9 test accuracy: 0.881\n", + "Step 250 train loss: 0.182115 test loss: 0.28543878 train accuracy: 0.94 test accuracy: 0.915\n", + "Step 300 train loss: 0.2119576 test loss: 0.22305593 train accuracy: 0.92 test accuracy: 0.93\n", + "Step 350 train loss: 0.12932214 test loss: 0.29057172 train accuracy: 0.96 test accuracy: 0.906\n", + "Step 400 train loss: 0.22937602 test loss: 0.2200287 train accuracy: 0.92 test accuracy: 0.925\n", + "Step 450 train loss: 0.23444137 test loss: 0.19857481 train accuracy: 0.94 test accuracy: 0.94\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAe8AAAFnCAYAAACPasF4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3XmAFNW9Pvynlt5mYdhmQMHggnGN\nS9zCD0ElKug1edUY9ZoQTYze3GuiRk1uYjRqRHNj4n5NrhKjiUYlbihGQFRUFDSoKIvgICAO6+xL\n711V5/2jlq7qZaZnpnumZ3g+/zjTXV1dXSP91PecU+dIQggBIiIiGjLkwT4AIiIi6h2GNxER0RDD\n8CYiIhpiGN5ERERDDMObiIhoiGF4ExERDTEMb6JeOOigg3DllVdmPf6rX/0KBx10kGe766+/3rPN\ne++9h9mzZwMAtm3bhkMPPdR57osvvsCPfvQjzJw5EzNnzsTZZ5+NV199FQBw0003YdasWZg1axYO\nO+wwnHLKKc7v4XDY8x7JZBLz58/v9edavXo1Lr300oK2XbBgAebMmdPn97J19/rZs2fjhRde6PO+\niYY7hjdRL3366aee0Ewmk1izZk3WditXrsQnn3xS0D6vu+46TJs2DYsXL8bixYtxyy234LrrrsPO\nnTtxyy23YNGiRVi0aBHGjRuH3//+987vVVVVnv188sknfQrUI444Ag8//HBB2y5fvhxTpkzp83vZ\n+vt6oj0Zw5uol0444QQsWbLE+f3tt9/GV77ylaztrrnmGtx+++0F7bO+vh5HHnmk8/uRRx6JxYsX\nY/z48QUfV3NzM3784x/jo48+wkUXXQTAbAF48MEHMXPmTOi6jlWrVuHcc8/FrFmzcOaZZ2L58uUA\nzFaB0047DQBw//334ze/+Q2uuOIKfP3rX8d5552HxsZG533ee+89HHzwwVnv9cEHH+Bb3/oWTjvt\nNJx//vloaGgAAOzevRsXX3wxzjzzTJx66qm4++67cx5rPu+99x7OOecczJo1C9/+9redC6Vc++3u\ncSEE/vd//xczZ87EKaecgjlz5kDXdQDAwoULcdZZZ+GMM87AN77xDbz33nsFn3eiwcDwJuqlM844\nAy+99JLz+z//+U/MmjUr53ZCCCxatKjHfU6fPh1XXnkl/va3v2HTpk0AgHHjxkGSpIKPa+zYsbjm\nmmtw1FFH4YknnnAeF0Jg8eLFUBQFv/71r3HppZdi0aJFuPzyy3HTTTfl3NeiRYtw/fXX49VXX8WY\nMWPw7LPPAgA2bdqE2tpaTJgwwfNe4XAY//mf/4lrrrkGS5Yswfe+9z1cddVVAIBHH30Uxx13HF5+\n+WUsWLAADQ0NMAwj57FmikQiuOqqq3DDDTdg0aJF+OEPf4jrrrsOhmHk3G9jY2Pex1944QUsWrQI\nzzzzDJYsWYKGhgY8+eSTAIBbbrkFDz74IBYuXIibbroJr7/+esHnnWgwMLyJeun444/Hxo0b0dLS\nglgshlWrVmHKlCk5t73++uvxhz/8AYlEott9/v73v8d3vvMdLFiwAGeddRZmzJjhBEt/nXzyyc7P\n8+fPxxlnnAEAOOaYY5zqONOxxx6LCRMmQJIkHHLIIdi5cycAYMWKFTk/6wcffIBx48Zh6tSpAICz\nzjoLX3zxBXbs2IExY8bg7bffxvvvvw+/34+77roLdXV1BR376tWrMX78eBxzzDEAgJkzZ6KtrQ3b\nt2/Pu998jy9duhTf+ta3UF1dDVVV8e1vfxuvvPIKAGDMmDF46qmnsH37dhx77LH45S9/WdjJJRok\n6mAfANFQoygKTj/9dCxcuBCjR4/GiSeeCFXN/U/psMMOw3HHHYdHHnkERx99dN59BgIBXHrppbj0\n0kvR2dmJRYsW4fbbb8fEiRMxbdq0fh3vyJEjnZ8XLFiAv/3tb4hEIjAMA/mWNqiurnZ+VhTFaV5+\n5513cMkll2Rt39nZiYaGBk8LhN/vR2trKy655BIYhoFbbrkFjY2N+M53voOf/OQnBR17a2srRowY\nkXVsLS0tefeb7/Guri48/PDDmDdvHgBA13WMHj0aAPCnP/0Jf/rTn3Duuedir732wvXXX4/jjz++\noGMkGgwMb6I+OPPMM3H33Xdj1KhRPfbZ/vSnP8W5556LiRMn5ny+tbUV69evd6rWESNG4Pzzz8ey\nZctQX1/f7/C27d69GzfccAOefvppHHLIIfj8888xc+bMgl+vaRrWrFmT8yKkrq4O+++/P5577rmc\nr7388stx+eWXY8uWLbjsssucSronY8aMQXt7u/O7EAIdHR0YM2YMVFXNud+pU6fmfLyurg4zZszA\nd7/73az3+dKXvoTf/va3MAwD8+fPx7XXXotly5YVeGaIBh6bzYn64Oijj0ZjYyM2btzYY4VWV1eH\n73znO7j//vtzPh+Px3HllVd6wmLr1q34+OOPceyxx/bquFRVRTgczllRt7a2oqKiAvvvvz80TXMq\n0EgkUtC+V69ejYMOOgh+vz/rvY488kg0NTXh448/BgA0NDTgZz/7GYQQ+PWvf4133nkHgBmSY8eO\nhSRJ3R6r7YgjjkBzczNWrVoFwBxfMH78eEycODHvfvM9/vWvfx0vvPACYrEYAOCpp57C888/j9bW\nVnz/+99HOByGLMs48sgjezXWgGgwsPIm6gNJknDaaachFotBlnu+Bv7BD36Ap59+Oudze++9N/70\npz/hvvvuw5w5cyCEQFVVFX75y196RqAX4phjjsEf/vAHTJs2DW+++abnuYMPPhjTp0/HzJkzMWbM\nGPziF7/Ahx9+iNmzZ+O///u/e9y3fYtYvve67777cOuttyISicDn8+Gqq66CJEm48MIL8etf/xq3\n3norhBCYMWMGpkyZgh07dnheryhK1ntWVFTgnnvuwa233opoNIrRo0fjrrvu6na/I0eOzPk4AGzc\nuBHnnHMOADPYb7vtNowePRrTpk3Dt771LSiKAp/Ph9tuu61X551ooElcz5uIiGhoYbM5ERHREMPw\nJiIiGmIY3kREREMMw5uIiGiIYXgTERENMUPmVrGmpq6i7m/UqAq0tUWLus89Ec9j//Ec9h/PYXHw\nPPZfsc9hbW11zsf32MpbVbPvKaXe43nsP57D/uM5LA6ex/4bqHO4x4Y3ERHRUMXwJiIiGmIY3kRE\nREMMw5uIiGiIYXgTERENMQxvIiKiIYbhTURENMQwvImIaNh6443XCt723nvvxI4d23vc7sMP38cN\nN/y8P4fVbwxvIiIalnbu3IFXX11c8PZXXXUt9t57QgmPqHiGzPSoREREvXHXXb/D+vXr8Mgjc2EY\nBnbs2I6dO3fgnnv+iN/+9jdoampELBbDD35wOaZOnYYf//hyXHPNz7F06WuIRML44out2L59G668\n8lpMmTI153u89toSzJv3dyiKgoMOOgS33XYL6us34M47fwefzwe/349bbvktdu7cnvVYdXXuqU8L\nsceGd0c4gfc3NOLYg+sG+1CIiIa9f7z+GVZuaCzqPo87uA7nz5ic9/l///fZeO65f+D7378MDz/8\nIDQthT/+8c9oa2vF8cd/DWeccRa2b9+GG2/8BaZOneZ5bWPjbvzhD/fh3XeX44UXns0Z3tFoFA89\n9AAeeeQJVFRU4Oc//yneffddvPzyyzjnnPMwa9a/4YMPVqK1tQUvv7wg6zGGdx9ceecbaO2M46ZL\njsOk8X0/gURENDQccshhAIDq6hFYv34dXnzxOUiSjM7OjqxtjzjiKABAXV0dwuFwzv01NHyBiRO/\nhIqKCgDA0Ucfg/Xr1+PEE0/CH/7wP2ho+AJf//ppmDRp35yP9cceGd5b23YiPOFNSMnD0dwRZ3gT\nEZXY+TMmd1slDwSfzwcAWLJkETo7O/HAA39GZ2cnfvjD2VnbKkp6gREhRM79SZL3OU1LQZJCOPbY\n4/HnP/8Ny5cvw5w5N+PHP74652Nf/eqxff4se2R4f7ztCyjVbTBG70RLZ3ywD4eIiEpAlmXoup71\neHt7O/baa2/Isow333wdqVSqT/vfZ59J2LbtC0SjEVRUVGLVqg9x1VU/xrPPzsOUKSfi9NPPgBAC\n9fUbsGXLpqzHGN69dPykA7G4CZArO9DSwfAmIhqOJk3aD59+ugH33XcnKiurnMdPPnkGfvGLa/DJ\nJ2vxb//2TdTV1eGRR+b2ev+hUAhXXHEVrr32J5AkGUcccRSOPfZY7NzZghtv/AWqqqrg8/lw/fU3\nob7+06zH+kMS+doDykxTU1dR93fjit+ipTOCQyLn4yfnHlHUfe9Jamuri/632dPwHPYfz2Fx8Dz2\nX7HPYW1t7m7dPfY+7y+P2Q+SL4mmcOtgHwoREVGv7LHhPbFmPACgLdk2yEdCRETUO3tseI8JjQIA\nxBFGPKkN8tEQEREVbs8N74rRAADJH+egNSIiGlL22PAeW2FW3pI/xtvFiIhoSNljw3uME96svImI\naGjZY8M75AvCLwcg+eNoZuVNRDQs9WZJUNtHH32ItjbvnUjlsAyo2x4b3gAwMlDDypuIaJjq7ZKg\ntn/+88Ws8C43e+QMa7a6ijFojDWiqSt7UnoiIhra3EuCXnDBRbj99lvQ1dUFXddx9dU/w+TJB+Lx\nxx/Fm28uhSzLmDp1Gg455FAsW/YGtmzZjDlz7sD48eOz9pu5DOjVV1/nLANaWRkCIJdkGVC3PTy8\nxwItQKfWPtiHQkQ0rD332UtY1bimqPs8uu4rOHfyWXmfdy8J+uijf8YJJ/w/fOMbZ2PLls24994/\n4J57/oinnnoc8+cvgqIomD//WRx33NcwefKXcc01P88Z3LmWAf3ww/fx1ltLcc4552H27AuxaNHr\nJVkG1G2PDu/a0FgAQAysvImIhrM1a1ajvb0Nixe/DABIJMzu0pNP/jquvvq/cNpps3D66bN63E+u\nZUDr6zc4S362tOzClCknlWQZULc9OrzrKszwTildMISALEmDfERERMPTuZPP6rZKLjWfT8VPf/oz\nHH64dy2L6677JbZu/Ryvv74EP/nJf+Chh/7a7X5yLQMaCAScJT/XrFlZsmVA3fboAWt25Y1gFNE4\nZ1kjIhpO3EuCHnro4XjrrTcAAFu2bMZTTz2OcDiMRx6Zi0mT9sX3v38ZqqtrEI1G8i4lCniXAQWA\nVas+xEEHHYpnn52Hzs4OfPOb38QFF1yE+voNzmOnn36G81ix7NGV96hgDSQhQw5EEYmnUBXyDfYh\nERFRkbiXBP3hD3+E2267Gf/1Xz+EYRi4+urrUFVVhfb2Nlx22fcQClXg8MOPwIgRNTjqqK/ihhv+\nG7/97Z3Yf/8DPPvMtQzokUcehVgsihtv/AVGjaoBIJdkGVC3PXZJUHvZtp++fgvicQM/O+pa7L/3\niKK+x56ASwj2H89h//EcFgfPY/9xSdABEpCCkNQUIvHUYB8KERFRQfb48A4qIUiqhs4oJ2ohIqKh\nYY8P7wo1BABoj4YH+UiIiIgKs8eHd6XPvFevIxEZ5CMhIiIqzB4f3iMClQCAjjjDm4iIhoY9PrxH\nVZgj+Xa1tw3ykRARERVmjw/v0RXm7WE7OjrQHk4M8tEQERH1bI8P70qfOWBNUpNYvallkI+GiIio\nZwxvn9nnDSWFpvbY4B4MERFRAUo6Peodd9yBDz74AJqm4T/+4z9w+umnO88tX74cd911FxRFwfTp\n03HFFVeU8lDysm8Vk9QUWjvZbE5EROWvZOH97rvvYuPGjZg3bx7a2tpwzjnneMJ7zpw5ePjhhzFu\n3Dh897vfxcyZMzF58uRSHU5eITVo/qBoaOviRC1ERFT+Shbexx13HI44wlx6bcSIEYjFYtB1HYqi\noKGhATU1Ndhrr70AACeddBJWrFgxKOHtV/wAAJ9foK2NlTcREZW/koW3oijOYuXPPPMMpk+fDkVR\nAABNTU0YPXq0s+3o0aPR0NDQ7f5GjaqAqipFPcba2mqM1M3K2+8XaI8kMXZsFSSu690r+SbOp8Lx\nHPYfz2Fx8Dz230Ccw5IvCfrqq6/imWeewV/+8pd+7aetLVqkIzLZK78IISBLMiTFQCKpY+u2NlQG\nuTRoobgKUf/xHPYfz2Fx8Dz237BYVWzZsmX4v//7P8ydOxfV1ekDqKurQ3Nzs/P77t27UVdXV8pD\nyUuSJPhlP2TVXHi9jYPWiIiozJUsvLu6unDHHXfgwQcfxMiRIz3PTZw4EeFwGNu2bYOmaVi6dCmm\nTp1aqkPpkV/xAbIZ3h2R5KAdBxERUSFK1mz+8ssvo62tDVdffbXz2AknnICDDjoIp512Gm6++WZc\ne+21AIAzzzwT++23X6kOpUd+xY9kyhxpHo5xXW8iIipvJQvvCy64ABdccEHe54877jjMmzevVG/f\nKwHFjw6YS4IyvImIqNzt8TOsAYBf9kMXZmhHGN5ERFTmGN4w+7wNGIBksPImIqKyx/BGeqIWyDrC\ncYY3ERGVN4Y3zD5vAGZ4s/ImIqIyx/CG2ecNAKrPYJ83ERGVPYY3rPu8AYRCEitvIiIqewxvpPu8\nQyEgHNMG+WiIiIi6x/BGus87GABiCQ26YQzyEREREeXH8Ea68g5YS3tH46y+iYiofDG8Afhls89b\nVc2KO57UB/NwiIiIusXwRrryllUBwGw6JyIiKlcMbwABJQAAzrKgrLyJiKicMbwBhFQzvCXVrLjj\nSVbeRERUvhjeAIKKNVJNNu/xjiVYeRMRUflieAMIWpW3IZkVd4yVNxERlTGGN4CQGgIAGJJZecdZ\neRMRURljeAMIWgPWdCQBsM+biIjKG8MbgCqrUCQFmhXe7PMmIqJyxvAGIEkSgmoAKWGFNytvIiIq\nYwxvS1AJImkkAABxTtJCRERljOFtCaoBJHQrvDlJCxERlTGGtyWkBpHQk1BkNpsTEVF5Y3hbgkoQ\nAgKBoOCtYkREVNYY3hZ7opZgCIiyz5uIiMoYw9sSVM0pUitCQCSWGuSjISIiyo/hbQlZ85sHQwJJ\nzUAixaZzIiIqTwxvi115B4IGAFbfRERUvhjeFrvP2+c3wzvM8CYiojLF8LbYzeYqw5uIiMocw9ti\nV96yz+zrZngTEVG5YnhbglblLavmbWIMbyIiKlcMb4tdeUNheBMRUXljeFtC1mhzQzJDm+FNRETl\niuFtCTK8iYhoiGB4W+w+b3tNb85vTkRE5YrhbfHJKmRJdtb0TunGIB8RERFRbgxviyRJCClBZ01v\nneFNRERliuHtElQDiGlxKLLEypuIiMoWw9slqAYR1xJQFRmaJgb7cIiIiHJieLsErWZzRQE0Vt5E\nRFSmGN4uITUAAQHVLxjeRERUthjeLj7Fb/5XNRjeRERUthjeLn7ZBwCQVYGUzj5vIiIqTwxvF5+s\nAgAU1YCm9b/ybutK4MEX16G5I9bvfREREdkY3i4+xay8FaU4fd5PvFqP9z7Zjb8u3NDvfREREdkY\n3i4+u9ncZ0ArQrN5PKl7/ktERFQMDG8Xu89bUQwYQsAw2O9NRETlh+HtYjebS4rZZM5Z1oiIqBwx\nvF2c0eayGdq8XYyIiMoRw9vF7vO2K+9i9HsTEREVG8PbxWk2tyvvItwuRkREVGwlDe/6+nqceuqp\nePzxx7OemzFjBi666CLMnj0bs2fPxu7du0t5KAWxK2/I5ujwfjebC1buRERUfGqpdhyNRnHrrbdi\nypQpebeZO3cuKisrS3UIvebPCG8OWCMionJUssrb7/dj7ty5qKurK9VbFF1Ws3mxwlsqzm6IiIiA\nElbeqqpCVbvf/U033YTt27fjmGOOwbXXXgtJGtyUs6dHFZLdbM5mbyIiKj8lC++eXHnllZg2bRpq\nampwxRVXYPHixZg1a1be7UeNqoCqKkU9htraas/vcf9IAIBqLi6Gqqpg1ja94fObp9enKv3aT7kb\nzp9toPAc9h/PYXHwPPbfQJzDQQvvs88+2/l5+vTpqK+v7za829qiRX3/2tpqNDV1eR4Lx1IAgJSW\nBAA0t4TRVBPo83ukkpq1Pz3rvYaLXOeReofnsP94DouD57H/in0O810IDMqtYl1dXbj00kuRTJoh\nuXLlShx44IGDcSge9mhzQ+KANSIiKl8lq7zXrl2L3/3ud9i+fTtUVcXixYsxY8YMTJw4Eaeddhqm\nT5+OCy64AIFAAIceemi3VfdA8St2n7dZMevs8yYiojJUsvA+/PDD8dhjj+V9/uKLL8bFF19cqrfv\nE6fyBitvIiIqX5xhzUWRFEiQYMCsvDnDGhERlSOGt4skSfApPqfy7uk+7x3hXXjsk38grsUH4vCI\niIgADOJo83Lll33QhTVKvIc+7/s+eghdyTDGVdTi9H1PGYjDIyIiYuWdKagEkDQSAAC9m8p7W2MY\nXckwACBpJAfk2IiIiACGd5bairGIGREEv/oqtic3593ulfcbnJ8lzn9KREQDiOGdYXyFORe7pGpY\nrb2af0N3i/ogT+tKRER7FoZ3hnGV6YVUVPjzbifAe8CJiGhwMLwzjK+oTf8iCquoZTabExHRAGJ4\nZxhfOc75OYEINEPLvaGn8GZ4ExHRwGF4Z6j2V+EHX/4h9I4xgCTQGm/r8TXs8iYiooHE8M5h/5pJ\nMLpGAQCaYq05t/GMV8tTebNXnIiISoHhnYOqSBApc7BaLJV7KVLhSmbeKkZERAOJ4Z2DqsiAYU4+\nl8g7AYsnvYmIiAYMwzsHVZEhDAUAkNBzh3chzeZERESlwPDOQVUkQDfDO5knvHuD4U5ERMXE8M5B\nkiQo1pot21o6cm8kvNsTERENFIZ3Hgp8AICV9TuwsyWS9TxHkhMR0WBheOdhhzdkHZ2R7pvO2SxO\nREQDieGdhyqlwzsX4bpXzBD5lw4lIiIqNoZ3HnZ4S0qe6VFd3EFORERUagzvPHyKz5yIRdaR1Lqv\nrA2w8iYiooHD8M5DlRXAUCApOpKp7KZzd7HNZnMiIhpIDO88fKp1r7esI5nqofJmszkREQ0ghnce\n5ixrKiRFQ0LLUXm7f2blTUREA4jhnYeqyAVX3nqe8GZBTkREpcDwzkOWJXN+c1lHIpljxLn7VjEO\nWCMiogHE8M7DMIQ5YE0WSGiprOe9zeY9lNicw4WIiIqI4Z2HYQhAN+c3j2mJ7rdlnzcREQ0ghnce\nuiGcZUFjqXj2Bp5bxdi5TUREA6eg8F67di2WLl0KALj77rtx8cUX4/333y/pgQ023RAQyQAAIKKH\nu92Wfd5ERDSQCgrvOXPmYL/99sP777+PNWvW4MYbb8R9991X6mMbVIYhIBIhAEBMdGU9z1vFiIho\nsBQU3oFAAPvuuy9ee+01nH/++Zg8eTJkeXi3uJuVdzfh7VmYhM3mREQ0cApK4FgshoULF+LVV1/F\niSeeiPb2dnR2dpb62AaVIQREMggASCLXet7pwK7f1pZzxDkXLCEiolIoKLyvueYaLFiwAD/96U9R\nVVWFxx57DJdcckmJD21w6a5m86Scq8873VS+uy2CpvZY9hZ2djPDiYioiNRCNvra176Gww8/HFVV\nVWhubsaUKVPw1a9+tdTHNqgMwwAMFUJToSvRrOfdlTckkQ5q9zZW5c0KnIiIiqmgyvvWW2/FwoUL\n0d7ejgsvvBCPP/44br755hIf2uD60rhqAIBIhKCrkawAzry3O3ezub1taY6RiIj2TAWF9yeffIJv\nf/vbWLhwIc455xzcc8892Lp1a6mPbVBdcsbB+N7Mg+DTqwFZR0cyo49fSieyxMqbiIgGUEHhbYfP\nG2+8gRkzZgAAkslk6Y6qDFQGfTj56AkIiBEAgMZok+d54b63WxI5VyGxA53ZTURExVRQeO+33344\n88wzEYlEcMghh2D+/Pmoqakp9bGVhZAwP+fOsDe8vROziJwBzcqbiIhKoaABa3PmzEF9fT0OOOAA\nAMDkyZNxxx13lPTAykW1MgotALZ37fY8nll557rXO+lrARSJfd5ERFRUBYV3PB7H66+/jnvvvReS\nJOGoo47C5MmTS31sZWGkbzSAXM3mmaPNvQm9qf1ztO31OvwVYyBaTy71YRIR0R6koGbzG2+8EeFw\nGBdeeCHOP/98NDc344Ybbij1sZWFmmAVhACi1uIkH3zaiBfe3gJkNJvruje869s2AQCUmhb2eRMR\nUVEVVHk3Nzfjrrvucn4/5ZRTMHv27JIdVDmpCKpAVIJm6ACAB55fCwA4cLLrukcS6EqGcdt7D+Oc\nyf+GQ8cchNZ4KwBApHzs8yYioqIqeHrUWCw9g1g0GkUi0f0a18NFZVAFhAxN1z2Pp9y/S8DqjlXY\nEdmFBz5+GADQEm8DAIhkiH3eRERUVAVV3hdccAHOOOMMHH744QCAdevW4aqrrirpgZWLiqAPEBI0\n4Q3vpK65fhMQGQndaoe3prLyJiKioioovM877zxMnToV69atgyRJuPHGG/HYY4+V+tjKgll5S9AN\n74xqKS0d3lLGgDUhhFN5QzYY3kREVFQFhTcA7LXXXthrr72c31evXl2SAyo3duWti+6azYUnoBN6\n0pk+VZJ1DlgjIqKi6vOi3HtKNVkZVCGEbC5U4pIy3GEunAFtABDX4+mnFH2POVdERDQw+hzekiQV\n8zjKVkVQBSBBhze8tYzKO2GkB/DFtHR4S7LOAWtERFRU3Tabn3TSSTlDWgiBtra2kh1UOamw+ryF\n8PZdp3QNAdd2yTzhzT5vIiIqtm7D+4knnhio4yhbiixDEjIMpKC5J2KR3TOsGXkrb7DPm4iIiqzb\n8J4wYcJAHUdZkyUJAgZSWrqpXJJdt4pJQMod3qmoazsDBpjeRERUPH3u8y5EfX09Tj31VDz++ONZ\nzy1fvhznnXceLrjgAjzwwAOlPIx+kyADEEhprn5v1R3eAkmRDu+2RIfn9ULSQEREVCwlC+9oNIpb\nb70VU6ZMyfn8nDlzcP/99+PJJ5/EO++8g88++6xUh9JviiRDSAaSrvD2VN4QSBnp9c2d8DbM0ysk\n721mRERE/VGy8Pb7/Zg7dy7q6uqynmtoaEBNTQ322msvyLKMk046CStWrCjVofSbLCkABOJJVwgr\n3klaUsIV3vF28wctCAAQYOVNRETFU7LwVlUVwWAw53NNTU0YPXq08/vo0aPR1NSUc9tyoMgyJFmg\nI5JuGpdUb+Wtwd1sboV3yhwf+2w3AAAgAElEQVSPzsqbiIiKqeAZ1gbbqFEVUFWlqPusra0uaDtV\nMU+TUFzXOlblLTQVkj/puQu8PWk1m1vhDVkv+L2GouH82QYKz2H/8RwWB89j/w3EORyU8K6rq0Nz\nc7Pz++7du3M2r7u1tUW7fb63amur0dTUVdC2spABCdi2M31vu2SHt+5zqvB9qvZGQ3gHuhJh87lU\nABIAQ9IKfq+hpjfnkXLjOew/nsPi4Hnsv2Kfw3wXAiUdbZ7PxIkTEQ6HsW3bNmiahqVLl2Lq1KmD\ncSgFUWTzNHVE0/3akDXz/m3NvP6pwlhMm+AdnCecZnP2eRMRUfGUrPJeu3Ytfve732H79u1QVRWL\nFy/GjBkzMHHiRJx22mm4+eabce211wIAzjzzTOy3336lOpR+UxUF0IHOqGvaU1UDdBWQzHu4fQhA\nldOn0y/7ENet5nb2eRMRURGVLLwPP/zwbpcNPe644zBv3rxSvX1RqbIV3rH0oDQoGoSuOn3fighA\nkdN98j7Fh6iumE0bDG8iIiqiQWk2H2pUK5S7oq7R5opZeUtOePuhSunwViU1fZ+3zGZzIiIqHoZ3\nAXyKGcrhuN3nLZzK2x6spghvs7kqqxCaFeYyK28iIioehncB/NatYl0xK7xlA5IkzD5v2A/5PM3m\nqqxA6NbvisaVxYiIqGgY3gXwWfeX68KqoJ3bxNzh7Tebyi2qpMIwrN9lnUuTEBFR0TC8C1Dh91k/\nmRHszGvuCm9J+Jy+cQBmFW5V3pJVeXdGk7j/2dXY1hgekOMmIqLhieFdAL/PCm/JmkdNsSpwwzXj\nm65mNJur6cpcMdf0/ufyrVi1sRn3Pbt6AI6aiIiGK4Z3ARTJOk2SgCSlK293szkMNavZ3A53STYr\nb3s98GSKA9iIiKjvGN4FUOxbwCSByqAPvoDVg+2qvCXd22yuuprNoegw2OlNRERFMmQWJhlMslV5\nS5JAZciHsGrAACB0BYmNR0EZ2QRZqvbcKqZIKgAZQpedypuIiKgYWHkXIN1sbuDEr4yHL2D1fRsq\njLbxSG35CoRhB7b9GsXZxu7zdkjSwBw4ERENSwzvAshWEP/7qZNx5tcmweczk9i5jxuArouMZnPV\n2UbKvM+bVTgREfUDw7sAduU9fmwIkiRB8dmVtyu8hchoNndV3jL7vImIqHgY3gWQrSVBDWGGtqxa\no8Vdo811XXjmNreb0IWuAIoGwzDSO2SzORER9QPDuwB2Fa1b4S1Z93m7m80NQzgD2wCkg1xXIUlA\nyuDiJEREVBwcbV4AO5S/6NyGz9o3A0rKfMJwVd6GAclVUctOs7n537iWXguceuetj3dgQm0lDti7\nZrAPhYioLDC8C2D3eS/e+joAQLZWGfMMWDPSk7CYr/Fuc+fqe3AELhqQ4x1OYgkNjy7cAAD4yy9m\nDPLREBGVBzabF0B29WUDgJCyJ2nRDYHHXql3frfDW1LNKj2hJyBghntnJIkHnlsDg6POe6TpRs8b\nERHtYRjeBVAk72kSMMy7vQxvn/fGhnbXa8zntN2T0tsgXZl/UN+Enc2REh0xERENZwzvAmSGNwAr\nuNN93PGk7unztkebG51jobWMN3+Gd05znfeP9YhniIgoG8O7ALKsZD+oe4cLRGIpz+8KXK+xKnQD\n3hHnDO+esWeBiCgbw7sAco7KWxgZ/eAwB1c5r5HdK45Z94lL3srbYHj3iOdoePvX+t247I6l2N0a\nHexDIRpSGN4FyN9s7hV2Vd+K69TaQS/YbN5rDO/h7c8vfQLdEFi2eudgHwrRkMLwLkDmaHMATjXt\n5g7j5vZk+glhbtssbYLkT1cYKY6k7hFH5A9v/PMS9Q3DuwC5Ku9RVaFuX7P4vW3pX6yg362uQ/Co\nt5yHUymGd08Y3kRE2RjeBcjV5z1+VBWqQj4AQCjQw1w3OZrYAVbehWCzORFRNoZ3AZQczeaqrDrL\nfFZX+Lp9vcjRxA4AyZSe83FKY3jvGbhWD1HvMLwLIOf4ZnEv/1kdyg7vQyeNxrUXHoWvHTouu/KW\nNQACb3e8jHe2vwcAWPDOFsxd8ElRj3s4YHYTEWVjeBcgksq+jcW9/Gd1hT/r+ZHVfhy272jzOeE9\nzZI/DskfxxfJT/HEp88CAJ5ftgUr1u0q8pEPnLWbW7BibfGPn5U3EVE2hncBJo3YBwDw5VGTncfM\nZnPz5xGV6fAWmlmFj6kYCQCQ5exmcykQg+RPrzJmrxMOwGmKz2f+ss34+LPmPnyK0rrrHx9j7kvF\nbznggDUiomwM7wJU+6vwwIw7cOa+pzqPqa5Z1/yq7Axei6+ZisTGo3DUhP0BABUBNavZXArEIAVi\nzu+NkXQYdxdWndEkXnznc9z7zOr+faAS6unio7fKObxffGeLs+IZEdFAYnj3guIKbFVWPfNu71NX\nBQCoCYzAdbPOwKTx1QCAiqAv655wSUlB8qfD+4GP/+KsEa7p3YR3JJn3uXKR1Io7gr6cm83nL9uC\ntz7eMdiHMaSV8bUZUVljePeCu59blVQ4y2ZIwMGTRgEAamtCzs8AUBlSs/q8IQlP5d2aaIW692YA\ngN7N7WPhaCrvc+Wi2CPoyzm8iYgGC8O7F7Iqbye7JZxxwpfwzan74rJvHOp5TWXQlzUPOmTDCe+v\n7zPd3F9tAyBrWZV3Uk9i4ZZX0Z7oQGe0/CvvRLHD23U6/ufvH6KxPZZ/40HCC4y+4y1iRH3D8O4F\n9/3e7j5vSQJURcbZ0/ZH7UjvzGuVOZrNIRmQ/HEoIoBzDzwLB4a+AknVIPnj0DIq73/Uv4CXtryC\nFzctQke4/MM7WeRZ49x93vUN7Zj32sai7r8YONlO37HZnKhvGN69oHbT551PZUjN7vOWDEi+JFQj\nCAAwdOt5SUBzVXHtiQ6s2LnS+b0jo897xY6VWLBpUS8/RWkVu/IWGVVtOS7mknnBRURUagzvXvBU\n3pKCQtK7MuiDENnN5lBSkI0AAEDT0o+7+7zvWzU3/RJJdgashQLm/h7f8DQWbX0dulE+M7UVu887\nM6zLsYk6VeRBekREPWF490L2aHMzSLrrtzNvFcucpCUBSQIk3QzvlDUOTZIM6Fafd2ckieZYi/Oa\nqBZzKu+qjBndolr59AMnSthsnuv3cqAxvPuNfd9EvcPw7gXPaHO5h8VILLIsZd/n7TMnaJE0c3IX\n3S5WJQOaYQbBtX98C7rQceDIAwAAsVQMHZEEAMCvKp77qbuS4d5/mCJyH0vxR5tn/l4e4a27Dox9\n3n0nCup8IqJMDO9eUFyBrcqq606xHsoG4X1e8pshbM/GJgzreUkgkdTxxqrt0GWzyq5UK+BX/Ihq\nMcQTZjDqhkBcT8/QFklF+vyZisHdtF30Pu/MyrtMwlvT0sfR3b35ROXglZUNWL+1bbAPg4qI4d0L\n7nW9PQPWemzyywhvnxnMwqq8Dd16Xjbw7Fub8bfFn0JSzI5wvxxEhRpCTIs5FZ4hBLqS6cAO55h7\nfSC5w9tdeacMDXd/+Ces2LEy18sKkt1s3uddFZW72uaANSpniZSOp17biN8/uWqwD4WKiOHdC1kD\n1iw9ZXfdqFDOx42kWXk74S0Z2N5kNoFLqtkRHrDCO6rFnYFRhiEQTrnDe5Arb1d4ufu8t3Y24LP2\nLXh8w9N933eZjjZ3BzYHrFE5K5fWKiouhncvSK5RNYprkpae3Pz94zC+60QkPj3G83gqYTbD233e\nkiTgXApY06X65QBCaghxLY6UZm5oCOFpKh/sZnMtT+WtGVquzXsl84unXAasuQepsc+7H6w/Z+bY\nBiLqHsO7j3yyAvf0qN0J+lVMCnwZRkcthKv/OxaVYRgCuqvytkmqGXw+KYAKXxACAklh9pUbRmaz\n+WBX3q4+by0d3rmWUu2tzLDOvO97sLgvWDjavP/K5aJsOOK5HZ4Y3n2UOT1qTxTFOtVGeluR8iMS\nT0HX0n3e6RdYlbcUQIVaYT1vPmYIIJxKjzAPJ7NDMvMfbCKl4911u5zqvZjczebJZPrnLtcxLnrv\niz7tO/N7Rx/gLyJNN7BkZQOice+88u7AZp93/7Fpt3TKpauJiovh3UfmwiSmQu5R9dnh7VqkROgq\nWjsTcPLUGm0OpPu8VRFASA1ab2pW45l93pnN5l/s7sIPf7cUb3603XnsuTc346EFn2D+si0FfT63\ndVta8doH2/I+7xlt7ro4CLtuYVv4Xu/fF8jRbD7AX0TPv7UZT762EX9fUu953N1Uzmbz/mN4l065\ntFZRcTG8+6jQ+7xtimIlvHuFMV3BLY+uRDhid3obTsVsh7cCPypUc8CbZFXjhiE8TdI7Irs8s6wt\nX7sLAPDU6585jzU0dgEANu3o7NVxA8Cd8z7C35fU521+y9fn7b7/3JD7tiJa1gxrA/w9tHFbBwCg\ntTPheZwD1orD/nOyabd0mN3DE8O7j1RZ6dWiCnblLQz7vxKc028FuiS5m83NKlsRfgTUgPVYesBa\nXDPv8z669itoT3Rgbct656V2FSP7EqhvMwPcp5qj4/vTbJ6vOvKMNk+6wtvVIiAUb/gV/J5Z93kP\nbFC2dZnHPbI64Hnc22wuEI2nsO7z1gE9tuGEAVM6bNUYnhjefaRIhU2PalNVO6itjQ1X5S7Sk7TY\n7CpbNgLpJnopfatYzArv0yedAgB4a9sK57VOv/A+a3DvqofwcdNa+K33T/ajSszXt5tvkhZ35d3X\n8M5s8hvoUcntYfO4R1T4PY+nXIP0NM3AnfM+xp1PfYT6hvYBPb6hzv6nM9AXZW5CCCz9cBt2tQ7u\nfAmlwlaN4Ynh3UeqrOLkoycAAA760qgCtvc2m8vCHd7Wn8E1YE3yJyAMCbLwwWc10UtyepKWmBaD\nLBQ89sIuHDhyf2xo24hdkd3m7uzAC5lBMn/Ty/D5zPdI9WPu8XwDX9yjzd39v+5BdYbct+VMM9/S\nEAKabmTNvFYq9mfOfD8tY5KWLTvN7oimMlxvvJw5zeaD2POweWcnHnulHr+a++7gHUQJsfIenhje\nfeSTFXzntC/jjh9NwWH7ju5xe6fytprNZeSqvN3hHYNIhqDrrv512Wo2N4CYFofQfdi8vRMnjDfv\nH/+0bZP5vN1vnjJHqTdGmyGpZgWZ7EezuZ5jGlAhhGeeb/e0oRHXKHhD6Wt4Zw9Yu/z3b2DO3z7o\n0/56w90FkDkoTcszYI0LbPRNb6vD9zc04sEX1xWlqozEzC6q4VqgsvIenhjefaTKKmRJwtiRuWdP\ny9o+Y7S5e7S63Q9uN5srqg7Jn4RIhKDpBnyKtYqY5K6844BuTtEaUioBAM+/vRHN7bF0FaOkB4nF\nVXOFMvfgqriW7hMvRGbl3RZvx/ee+ylWtb0PqGY4u0MtYbgCW+1bs3n2gDXzd7vSLaXWrvT88cmM\nFgv3eXT/LGekdyyhIZbo/2Q1A03TDWzd1TVg79fb6vCP89fivU92Y3cBTd2vf7itV5/FMATunPeR\n526NoYyV9/DE8O4j91SphVCt0eb2JC2q5FrW0x6wZjWLT5xo7lskQkhpRlazOWCGt6GZj8swt4+m\n4nhx+efpK20lHRqblXcANenp835iwzO4d9VD+KhpbUGfQc+oPjd1fI6ElsCyliUIHvkG4Is74W0I\nA5qhQYHVV9zHyjuzz3sgFwGx108Huq+8NU/l7Q3vK+5+C1fc/VaJjrB0HlrwCW55dOWA9eH3tTp0\n5k/Io7E9hsdfqcctj+afXz+ztaSxPYZ1W1rx10Wf9umYyg2ze3gqaXjffvvtuOCCC3DhhRdi9erV\nnudmzJiBiy66CLNnz8bs2bOxe/fuUh5K0Vz+le/hrP1metb2LoRdeUtWda1KKn71PWu61IxmcyVo\n9puKRAU03Ug3m9vN6rIBXegwUnZ4p5/fvKPTudIWcgp1FWMBAElEoY773FMl2iPUN1rN7T3pbrIH\nSTGgjGx0giypm8EXRJV1Avp2q1jSSHpaEIq95Gh3uqLp982cRU3zDFhzDTTsY7N5S6wVN7xzOza0\nbuzbDors/Q2NAFBQZVsMfa0OMy8oMxXy/0vmn2y4dX2w8h6eShbe//rXv7B161bMmzcPt912G267\n7basbebOnYvHHnsMjz32GMaNG1eqQymqI2sPxxn7fb3Xr3Oaza3wliDjgL1roMiS0w/ujDb3m1+Y\nduWdsjPErrytMBO6VZFb64VLio4dzRGzipEMQNYxOjAKFx30LfP5jACt9JnN7YVOr6plfAnYI95t\nysgmZxR2QrdmiDMqrffuW+W9UnseoWNegz20qbezRdW3fYZH1j2BVB/mWe+KuirvjLECqTxzm/e1\ngnz1izfRlmjH3DWPFbS9EAJPLKnHui2lvT2tKuTreaMi6Gu+9HSPfUE5LHX765DHPu/hqWThvWLF\nCpx66qkAgAMOOAAdHR0Ih8M9vGr4UuzR5s7tZVbftyJD2KPN7T5t1aq8k0GkdAML3ramFrUGrNnL\nhcIKbwjF83wkrjkBH1KDOGj0gZ7nbVU+c0BbOJk7vNvi7Xh03ZOQrIuJzCrHvtf8lJFnw4hXQK5u\ncyrUlNXfrYgghC47y6D2VhhmOMk1zX16/b2rHsL7uz/CxwV2Dbh1uirvzJDwDNJzN6FrfWz+tVpy\nNFHYRcb2pghe/WAb7pz3UZ/erzvukfWZF2yl4q4OOyNJrN3ckndbz/H11I3ShzJ6uGXdnlh5G0Lg\nd3//EP9c8flgH0rJ9G6asF5obm7GYYcd5vw+evRoNDU1oaqqynnspptuwvbt23HMMcfg2muvzeov\ndBs1qgKq2rum6p7U1lYXdX/dGdlsNT/azeaKitraavhUGYmUt887FJKBlFlZ+/wqWtpTwCjXJC5W\neAvdrIpGjrA+hxXOmiGchU1GVY/AXnXWrWzW/u3PXREIAl1AzIjmPBfLPnkbK3evQuAIGfH3T0f1\niJBnu+QXZrhVh6ogkgHIwSg0w0BtbTVi7eaAMkX2QST9gJrM+R7vbVuFUcEafHns/p7Ho/EUKoLp\nqk8Zux1GR61nm978/Xyh3v+9U64vPUOSPK/3B9LHJrv6XYMVfmc7dytBT+8dCJj/FDVDK+g4O+Lp\nC7Fi/3/c1pluUQm5Pk8pqT7FeZ9fPLQEja1R3Hftydhv75qsbSOx9EVVVXXQeV2u44y7rrnyfY6a\n1phnm5he+N9tKGgKpy+cC/k8w+Ezh6NJfNrQjk8b2nHJN78y4O8/IP9mSv4Olsz7ZK+88kpMmzYN\nNTU1uOKKK7B48WLMmjUr7+vb2orb91ZbW42mpoEbTdvVZX1BWAEqdKCpqQuyLOWYpMX6YhYyOrvi\nUCUFCddrneZva8BaW6s5ktsO/65Iup9Y0hR0tVnPWzO0NTZ2QpIkdMbMintXuAlf7GyCX/Z5+vI7\nwzFnv1IwjJaWCJpC5nsmkjpefOdTqOOARFQ4rQApI4mmpi7s6jAHOukpCdD8kIKRrPNtCAN3vvMQ\nAOCBGXc4j2/Y2oY7nlyF807e32yokAClphkpyfBML9ubv19LR5dn+22NYUgSMKG2yrPdZ9s7cO/T\nH+Pq849EY0u6RSIWT3le3+EKuIireb2tPepsF0+mq+jujvWtxmVY9Nkbzu/23ydTMqXj3U9247iD\n69DWnv73UKz/j1es24VdLVEctl/61sfWtuiA/DuJu85vo9XPvnFLC6p82Y2Dja576Ztawmiq9uf9\n99zSkm7ty/c5OjLOZVNzz68ZSlpb0/8f9/R5Bvp7sVQiroWEBvrzFPsc5rsQKFmzeV1dHZqb002d\njY2NqK1NV05nn302xowZA1VVMX36dNTX1+fazbDh3ELk6vMGkNHnbQ1YU+1FjmVougG/Yo3Ylg0E\nfIrTbG73ecPwNpvHk5qzTYUagk/2eZ5/7q3NeGfNTqfPOqEncd1bv8b1Cx/CZ9s7nGN2z58uBSOe\npuKuWNJpAZAMn3MsQtagG4bTbA5dgdB8kBQdCc3bdJ7Qc98+ttIaLLVw5RanA1JSNchVfR/5HE15\nJ0/59V/+hRsf/lfWds8s/QyRuIZnlm5yms1HVPg8zea6YeCL3el/nO4+b/e98O6R/d01Xc5bu8Dz\ne0TLfaG6YPnneHThBjz56saSNO3OXfAJFiz/HM0d6XM1UPO25+qXzddk7668e1qOtZAxEpnvM9xW\n4ervx+mIJPHBp03FOZgBMtz+hrmULLynTp2KxYsXAwDWrVuHuro6p8m8q6sLl156KZJJ88t85cqV\nOPDAA0t1KGUhff+vNe847D5vKV1NWuEtK1Z1bshIaQYCavo+74BPTt8CZjWb6xrM6t0K51hCd6rz\nkBqCIiuQhAzJev6fK7bi4X+uzxpwFg5twd3/SPehulcFU0Y1oiWRHhxlGMK5QJCFz6m8JUWDpgkk\nrNHmwlAgUubFR1vcezUaTaXf372wiv1FLqvmY0I3L07kEd5+UPMiQcsK5lw6k7mvhA1hYHc0/cVk\nT6aj6Qa6oklUhXwI+BVPiK1YuxtrXQPFtDyD19yz2el5phAzRPbjLbHcg9B2tZih/vmu0t7j3tKR\n/ruUMrzdrXG5Lm7yjST3hHcPo80LGayV+d7D7YvffQ76MjPh//z9Qzzw/JohNfVvrgmlhpuShfdX\nv/pVHHbYYbjwwgsxZ84c3HTTTXjuueewZMkSVFdXY/r06c5tZKNHj+62yXw4kK0Ba3a/tV15T6yt\nAmA1nctWVW6FNwwFKV0gqKbv8w74FSek7cldNN2AJBSn2Tye0JyAt5cTlaB41wuHQEJPoNJeKxyA\n0FTPVbq78lZrt+PvDQ8imdJR39BuTlpih7er8oaiIaUbSOr2iHgZ0Mzwrm/x3pIW1dKh61772/4y\ntS8OjC6zGVcOeQc8aprAw2sfw8+W3YRIKuoJccMQePHtLc5kOJ3J3IG3cMur+M27v3fudbfvCtB0\nga5oCtUVPvhUb3hv3tHh2UfKM2DNtba5PUJd0pHQcg9Ei2vZrQ/5Rv/b/w/phijpl1OLq0uglMud\nunMkZ3jnCdGwK7x7Or5CgjgrvHvY519eXo///r/lPe63XLg/X19Gntu3Cw6lqX/zXSwPJyXt877u\nuus8vx988MHOzxdffDEuvvjiUr59WXEqbzugrdHml5xxMPbfewdeiSsw7AFp9n+FDE0zYOj23Oe6\n2Wwup8MdsKoj4Qp1pG/NspcTlYXqHW2uaBAQ2K9mknO/t4hXpudgR+4Q+cvL6/Gv9Y046/9NgqRo\nELoC3YCn8tZ1A0nDWr5UV2AkzGOYt+kZHD/hSAStVdJirvDuSHRiZMAcnGR/v8jWoDsRr4DQFUhB\n7/GkdANrms1j//mymwEAPzjsIhwz7ij8a/1uzH97C0LHCEABOhPp4HdXa+/sMCfvWLV7DY6qPdwJ\n70RKRziWwoSxlYgndU94B/2utdxlAy0j/gUlPgJSMIIW3Q/AHHyXTBmAZCB45FuYv6kT3z3sW1nn\nM7P1AzAHreVi37FgGKLHirM/8lXeiZSOrmgSY2sKm1WwJ+4gyZWx+YI3Ek+fn54uYgoZaZ35Pj0F\n/turd1rbGVDk8p/nyhPehkAP89rkNZRaJIbSsfZV+f+fN0yMqQlaP3mbzasr/Pi3KftClc1wkkc2\nIpwKQ7Kq8ZRuIJGy/keUzD5vJ9ytyjulGea93q5wloLm1fLY0Bjzd6E4zeZAetWyoBLElV/5sfmg\nrHtmrAqnIhgd9C668q/1Zn/0xoYOc1CcrkI3BISRWXlbzea6DL3xSxBJM7Cjrv5cd3gv3rrUqdad\nudntixFdhYhXWp/JfZtQdoDtipjHZ37BC2cZ1Q5X5e2e6tReS317s9msbs+E12atJmZW3rInxGLW\nQLTbLjsBoTFtSFR/Dv8Bq+GbsAmrxItOU3hS0yH545D8CWzsyD0NbVw3gzKoBHDKPicCQM570qOp\nmPP31Q0BrciVhbs5tdm1drm7JeGOJ1bh539a4Zl5rj/0HirCfBcoUdd0sz1V3oWFd+ZtgIV98Wd2\nKQgh8PFnzWU3Ha773PYn1IbSLWdsNqeiGVUdwG2XneBUywq8k18okgJJ1RD48ofYEdllzaomIaUZ\nSKYEhCFDkg34fa6QFq5lPo2McA5EASFhTMhscpZyVN4AsGZjJ3738GdQjRAgG051J4RAOBVBlTWR\nS6akpluVt2p++en2RDEaNF044W1oMiBk6O3mYEU7oAHvILKPm9bipS3mGIms6V11FUas0hz17k+/\nRtMMyJL3f2FP8Lmmh+1IdDrv51621J4AJ2m9zl533V6UpLrSD58qw3AtwBK3ngv61Zwz7dmfMakZ\nkHxmOLfEWz2f3WZX3idNnIqJVXtnfwaYs9XNee9ObAq+CkBYK6sV98vJHUSeytsVjvZ88u5m6/5w\nh0GuUMn3GfU83RQ5t+1L5a27jyv//jOPb/naXbj3mdX466INPb7nQHJ/hP5c8w2lanYoXWj0FcN7\nAO01phIHpk6D3joOU+r+n+e5zBHGftkHVTFHmyeSulllywZURXaazYWr2VwYCiRfylkARA5GIeuh\n9LzoIqMyt5qk7XlzzIsD3ak8E3oSmqGhUq1AcrN5n6Q9hzoAJDTdDEddhaYJT5/3X15ej664GZS6\nZq+mZr7WDnXAW3kDwEeNa8xNnT5vb+UNAFIo3XSe0DSnYjyq1jzGlGEHp56ezAaAgMD7uz92nks/\nYU9ba430z2hTHFFhhjeQDji7sgr6FShq9rdh0khCNwxsbwxD8iec998VzZ4C2J7oJqQGnb9VKiPk\nP2hcjY5kJ8LyLsjVbVafd3Er77hrBTXPimnWZ3avsFasL3F3tZ85h735Prk/o2dq2j40my98dytW\nb2rJu02+VfIyZVbe9iDGTdtLv2hOb3gGBvbjNoVi/z9XSkPpQqOvGN4D7IpZU3H1cT/AtMO+1O12\nqqzCp5qVdyJlhndNtYq6kaH0wDO72Vw3zIFhAEJfXQqoCUj+BJRUFYQQ+OPzaxCNCUiygNPsbM8X\n7p5iVU734dn93RVqJTsCVIYAACAASURBVPTmCdA7R8OADsBuEk5Bks3Qjic1T5/3Z9s6sGqTGVS6\nZlXyVmVu94UDQDSjv7cl3obGaHO6/9OpvBWIlNns7p5mNZyMQkDgyNrD8c39Z5rnwqpaw7GU83q9\nrQ5CAG82LEdjW8QTRPY99kKyBt9l3F49wmo2B9Jf1vGEBglAwK84I+LdknoKTy/dhKde/wzwpZug\nd4R3ZW1rV95BNQjVuqVPM7zhvXLXh87PytjtVp934V9OWzq+wJ/XPo5wMpJ3tHE8zxzg9mduaDSv\n8tQJG/HytpcKfu/u9NRsnm+kuztce2w2z9hvNK7h6Tc24Z6nP05v002fd+b+3ecvc8rcZmtA11in\ni6w86D20cBS8nyE09Vyxu5XKEcN7gPl9Cg6eNKrb2eQAwCer8Cmy1WyuQ5FVCDkJQ04BkvWl4fR5\n604VDgDKSPP+eilZiVhCw/ufNmXdCy75zdCwQ1FYfeZ25b0zYgbNCL81QYDzeiu8EXFeH4lrnsob\nSFfYuqZ4Xp9wVd6fN5mVynXH/Bhn7GtOpdueaE9/Qcqu+9l17/EDQJc1rWuVr8IJPrvyjsRS6dHq\nsSroLXthV2wXfvXss3jh7S3pE23tLwnzizfzy7raVXl/vH0zVu/YjFhSRzCgQJYkyEqu8E5imTWo\nya68ge7DO6QE0pV3RrN5S7wN1b4qyEKFXNmRNWBtV2Q33tuZf33zf9TPx6rG1bjx5b/i9sdzbxfP\n009rn4+GRnNMgG/CJqxu/xDhWApPLKnvVxO6O0dyZUrmMqw276IwvWs2D8ey++u7azbPvIBwH1Pm\nc01Wd8OoEYFuj2mgGT3cklfwfoZQNbsn9HkP2Axr1DuqYt5fHEtqSGoGAlAQTnXhXflvgGwu4iKE\nq9lcl5wFFeRqs0lQSoUQtkfmWkHv+9IGpD4/DFLAXrnMHDls6GZzvH070spdq8ztIxMAtDmVM2Qd\nMFTEpU4oMEeoR42Uq/K2mrqtCwwtZVW2OZrN12zdBXUsMMJf5dzSFtPi6S8J2T52NT0JnSss7TnZ\nK32V8Cne4AvHNE+fubbjAKhjd0Ie0YpVG9OTB9n3w8eMMO54/37ElNEA9nKe19ROrK94BsrY/fBk\nwyLz/RJnOyPOJVflbUSrIFeEkdCTqAgoiCU0p88bAHZEssM77qq884V3OBnBmNAoSMlKdIR2Q0fK\nEzi3vncnAKAmMAL71UxCwJ7Uxz4uawBdomIbNm34ctYxAN5mczc7HKMJLf33ADD3pbVYs6kNmiHw\nvZkH5XxtT/L1ecuSBEOIvCuCefq8ezlgzT1ffa73BrxVW+b+3bPmuS/0hBDOQL5yC7nM0eZ9NZQC\nkc3mNGh8soqAT0VXxPyySfc3Cydw7C/7ZMqAcC2bKVeYVZIwZIStLys7PNW6bZBrmiFb4W3Ezfu8\nDatvWlENGMLA6uZPEMIIvPCKNWGIYc+/bo149pnNqCJe4am81boGyKN2Oc3Qeko2mxFzhLd7xLs7\nvJ2Kxj2TnD0Lnavytu9Dr/RVOLPI2f3F4VjK2b/QVAgtPdGNh7WNhhS2djag0f+x5+k2YxeSUgT+\n/dMLm8STOkKBdDcBACTWHwe9dbzzGYP281blXaFU5q68dbvPO2Teiw/vrWIpQ0Ncj6PaV4VqqRaS\nBBiBzpxNyvd/NBd3rLwPQgi8/uE2ayY2gdZ4m3ksqubchZApka/Z3AooTRee127aaf5/YfSjedId\nJO4+b7v1J6nluaDoplk7U+aXeFeOkfLdNptrmeGt53zOfftavhYDIQSefXOTZxbDgeAZbd6Ppu+B\nWqSmGPaE+7wZ3mXi9Emn4OBRBzr3OvtkFUG/4vzDE3L6S8eerGREyAy8aELzNM/KlWZ4G7qcbtbU\nXaOiJQEpEDOraWsCFfteclk2oBk6UkYKWjQEZ35Su9ncqnxl64vciFeiPZxIr3AGwDfhMwgrZFMp\nCaOqA1AlMzyT1rSpumFO8iKEOUNb0BXedsUl5HSfd2azPwBENKvyViucCxk7+CKxFKCmK3e4lk1N\nnwcjPSFOPkp2c3IslUDQnx5dDwBC8zvvsaO1AxV2ePviEJoPtYFx6Eh2eia+AbwD1pZ+YIb79tb0\ngCd7lrsqfyWqYN72JwKdeQcP7Yo2YmP7Jjz+Sj2WvN+AjmSXZzIcuTJ7MFVcS2Bly/KsVeeAdEBp\nugHZdZ99NGVdlAT7vmSonmcglWKHd54Q9FbehQ9YE0KgM9q7ZvPsyts1sM89IY/r4iffRcfnu7rw\nzxVbcftj+bs4SsGd17kGBvbEPb/AUFGs1oZyxvAuE//fAWfgJ0dfZt0iBqiy2WxuS0npL2A7qGsq\nrfCOa5B82TN1CUN2+vjcfeKSrEMKRK0mc8nZFgBk1XACUNcl1768fd72hCkiXoH2cDIdrjCb0u3K\nW+gKVEXGiKDZPG/fLhWOpszPofmh6cKpvONaHAnNACCQ8rUDwgxGu9nefTucHUqVvgrzVjtIzoC4\ncCzlDG4TKX+Oyl3Af9BKz2fPJQVr/vdPj4HeYYYnanYh4LcvqlyD6qxz8MTrG8zKXElBCkZhRKsw\n2mfeKpdZfcdc4b16o1khN7anJ5SxZ56r9lVBMsygFJLebRW0rvlT5+ftHebAwUlV5gBJKZQ9Teyz\nG1/EB13L4Nvn06zn7PBKaYZ5+6HFvmjpzz3NIk+zuT1oMpmnP7uvfd66Yc6alykz4LuvvHM3m7tb\nLqJGFz5sXJ31Pok8XROl1t8Ba3Z4r9ncUvKpeYuluwuw7vx10QZnEp5yx/AuM4p137JPVhH0eatl\nN2HIGGmFdyyhQW8dl7Uvs/K2vmxEOoilQBSSqkEkXTNluSpbu8/VHinuft4OTykQMydesSZnCfpV\nnDzCmkFMNiAk3ZqaVIJPlVFTYb5XJGmGVWs4BikQgxGvQDJleJrNkykdck0z9EA7KhP7mHO4Z1T+\nABC1Ku8qfyUkSYJPVhFJxLHgnS1WeNvN5n4AMoQhpcPfl4AywgxLvWW8s09ZT48U3m+vaucWPpEM\nQsTMufn9B6xG20irerIH1bmqe8jmjGxyVbvZzN01GmN85t9nc8fnnr+RHd6bGiLOHPbuPm+7X7/K\nX+lZtz39hZT9ZRxOpPvZd0fN/v0vjzBnN5QrO7NGnO+KmhPb5Ap2O7xSuuFtclfSa8c/99YmvPbB\ntqzX9iTfaPN05e0Nu2ff3ISXln+OsNTsdH/0ps9b13NX3lpGuOVbqx0AYnmazd2tBLvHvYSH1z6O\n19au97z2b1/8H/wHZy+GUyhDCM/FQ8Gv62cVav89GhrD+M2j72PTjg7c98xqRON9v3ArplxjI/py\nwZJM6Xjzox34y8vre964DDC8y4w9baoqKZ7KO4sho6bKbPKOJjSkPj8MX459A0YsPamK0CWn8naW\nEQUgWc3u9qxn9v4As9nZvlXJXXlnjVZXXCPMAVQEVewd2MfaRoMOzemHVhUZFX4zFCNWsOzobIYk\nCYh4BZKajpCSEd6VZr9gZXQ/81hzNJt3psyFEsYEzYrYp/iwszWM55dtMf/B+lyVN2BeaNjH75rn\nXa4I43jpAnNbpB8/cOJIp5lbaH7rIsDU5WswH5fTg+Ls1gF17834rKUBcrV5cWCER2Kczzw3G9q8\nM63t6mqF0GX88dkNCPnM/bv7vO1b9qp9Va7WAyNdfarZYbR5d5vzc7u1GEyNOhpGIgg51JX1ZaZI\n9oWZAXWfDVDqvjB/l1zN5prhad2RrM8djafw0vKt+PuSeuyM7Ma8T5/POV97Lkae+7ztqYTXbmnF\nR5+lBxf+c8VWvLBuGT6vfhnq3uY8+T32eXtmFzN6rLx1XXQ72txTeWu5K2+7p+mJ1z51LpQMYaAj\n1QZlRO5FZwpx3zOr8V93vdVta4emG57lMM337t993plTwP7+yVX46LNmvPnR9l7vq9iefXMTfnTn\nm9jZ4p06ubtBh24bt7Wjrcv8/zVfyLd0xEs6HXFfMbzLjF15GxCe8E6PJbcYCmoqrfCOpwChoEau\n9TRf64bkDFjzfPFat4l5mrqtn9uqV6fvv3Y1J2eFp6x7Xl8RUOFTFXMOckWH4QlvCRX/P3vfGW9H\nVa/9TN/19H5OzknvIR0SEjpEulIFiShYLyI2BEQR9PpD5aJX5d5XQbHAtYAIypULWABpIXRIg5De\nc0pO3XXKej+sMmv2npOQkJAE5vlAOHvKXrNm9jzr356/wfTMWax0xyDt5EUKSdiOhxjTYM+5eRSY\nJjgdgxEYq+w2H3D7YGkmKkxqERuqESB33eR9z/k51OD4GZztI2F5FehIjxDu/mRMxylzRvgxascA\nsf34rgGLndIG8VhnODZGNZaFPvlpkZvgZSphKnG0pVqwrm+DiGl7xENPoQskT5vT8LwDuXXqoBTz\nlhdQPO6rsAXKmMqRmBc/EwCwTYqZ9xeY7CuJg2QroJhF9GT7AxKnQo42MQijeQN0Rt5xU4fteMg5\nebylPwa10idSYXnnfCL58Su348mtS/D0tucwHAghIg8j2DDD30f+/Cf3Bd3PWgO18DU2Fsfx8Pra\nnmFL1uRzOR4JlXYtzXrfXZ33cAlroXFuhYhEtqCG/b4RAReWkRvHlOJbv34Bn//RUwGyGS488Xah\nlogfcC/Du9Uudnd4aMlGAMCK9cFFUVAlL/yaO3uz+O7/vIyb734RQDjJ9/Tn8dWfPov/vPe1sm0H\nGxF5H2Lgcp8e8QJu85OSl+CyKR8RWeeEqEjFDWiqItxXhq4G4rfE8RPWvLxvkYsabznWy+uwY9vx\n5zUPBT4D4Mufqi4AAqjB2vKERevS4eo0EU3xyds0NCTMIHl35ujLl1reXiDmXXRcEVsnpCRhTvXd\nxUNuH+pitejpz+P7v30Zjh20qDXTptYwczcTT2rqwv51drbD3dXC+qYbgOohldBx2xePRW1lDBk7\nA0MxAaIGLG9DYeSt+AI1gfkC/GQ3x4DjemhPt8EhDr551xMAgK5cD4jiwcumEDM1DGR4jbwjXLfC\n8jZTUtzft7x5XH989RjUaC1iO8cga4X6qwfXw8vSmv2/L1+BL972NJ5bSePv/YX+wHGinaylwXE9\nPLVlCfr1jZClCbjl3ZcpAIoL64h/iYXGa10rUIqubA8Gi0P4zSNv4KofP4Wt3ZlhNbdLX7YD2SJW\n71oLrWETVOba91gI47W1PfjRH1/Dd38d7o4W51IdvLTzFQxk/UUst4qD3+1hlf009DYa/y+zvAsS\nebthbnP/M0X1hFUnt9flHqF9xe6M561d9HmRPQHDLYzeLrRS5aJDEKW6GYFF2zBW86ad9J70MC3/\nsORH3tt+1cbesm0HGxF5H2KQyduSyLs53YA5jTNgKiwm66mIWzoqkqZI7DE0FfUVPkm7riLI29ky\nDsUNkwFIwiEy2Uj//0bvWwDoAsHfTv/fHLNMxHJlyzwRM6DrKrW8VQdEcaEpPB6uIcXc5isHX8fD\n6/+B3iJzKRcSKNgutnXmoCoqNnX3omh7Qq5UfAdhMWtOiEYBLhxk+k1c87MleHNzHwaGXMhtTxW9\n6LvMAboA4W5zVodOHJal7hIYGvMU6P6POGNnYaksN0Amb7Yw8VAU4QNSQt6KXqTKdVDgekR0U4Pq\nghCC7Sx5jeTSKBRdZLJ+jTx/6QvL20j6fd+lmDe3vFNGCiopDy2IVquOKch7xY4NAIA7HlwJQgj6\nCiWlSxqXf6WWt0tCrErNRUXSxMBQEUosCzVGX3KqomJd/4aApekRD//x4m345Yrf4cnXaDLQ+m0D\nw8a8S8l73dYB/PjV22GOXClkfVHiiXpdcq/L4C9xo/0N3Lf+TxhM+fFM/rIujXFvJstgtKwXf3O8\n2rUcAwU/L6DUba5YGcSP/Jv/5ap/H4ekKgO5MmRf8HZ6cpcuSDj25DbfuGMA/3X/soDrfTjyPpRy\nuEuH+HZi3p0lLU7DKjhMQyv77FBBRN6HGHi3MZd4Abd5OkGJQybvmKkFpBh1XUVrbYX4m3gaeofY\nKp9ocDtHiAYn/Bz+viGPQpjbHIA1eSn9n5KYNz1GBzQXRHVE85WYqSNp+eP86/q/+brmtonnV+7E\nt3/zIlxbw2CBveTYGF23xDvAPuelal2d0o+LaDSBTKUdxVy1ECBcriInn58vWlzPg6XSfXVDJu8M\nYiodu6gVB+ApLC9AsUXSXqAcDzSWbjHCdlxPiKcomgvXI9gyRInMy6XYi1AR96efuXeHbE7eKSGB\nC9XzrT5meadNSu6EoKScboiGDYgKkqXPhpyYtq2/F45EzqpnsnI6D6ahwnY9IfIiQzc8NFbHA+1n\nTxt5MmY3TAfgl8ABtAFNxsliTd86keCn6wrk05JhyAYA1m4rr4tWNV8NcHcQOvkshGEnt4ltnHxl\nK01uHSvvs7p3LX6+7C68YD8ItXon9Kb1cFwPHgsDFG0XetPGkkF6tIwS/n0EEBDuKRsv8fZIzm8n\nbC1n4e+N5X3lfzyOl1d34bkV5Tr85eMoP9ef1/zfbtX+9hV7Gnep5e0GLO/wY3mf8mSMl5mW73co\nl5lF5H2IgVvepJS848wFzcmbqIgZQfI2NEVYhAAATxUPKIXix39RalmHrDBD3OoyFATd5lTpTYei\nuVAU1gwFQNzUUBEP9oC2XZ6lreI1Fssjju5b1oxcbVt+80iWM3P9J9W0fz2uCkUliM/5BxQzD6J4\nAVc37bxGaDy9pDOb6xKYjFw1g373uv4NsD0HMS3BxufPnY0C8k4BLmxhvYfNkaHQc7oegcoFDVUX\nRdtFX44nDkpa2GyBMsAWXZt6ekA8BU7RD4koiivkTLmLO2kkqTEqhwYA5NwMVI/OPfdCFIlv+e0c\npLHCZHYU8svno4KwzHvdEfK8A5LL18vTcyUTCpK8xpuPQU+IOZRlcLn17xFPJPFpqor/emCZf97d\nSHgOZotlOR+q7ore67sDf4nzygoS8/MBuFXtegSKlYU17Um83hOMsfMXf3+BHpdVemGNewVG+5so\n2g4efHo9rvrxU1i5cRfUip7AsUrA8vYTqoazvPNOAf/+3K349crf7/aa9qTbrVZ24U3mPQPefsxb\nnvdk3F+YD7eYKP246Nr4+6YncNeqe3Y7vr2F63n42h1LcO9j4W11AZQ6YgJW9HCaCNt76LuxtoL+\n/sLc6283Uc0jBKs29r6jxi97i4i8DzHwhDW3JObNLW9D5a5XD5apo7bSJ0VdV8vIuxQyAQXIhoQ8\nCnsgd/m7EjFK3rL1yWO0MVNDMhaU7LSJLb6DJxEpngHD8jCiIeWXAknhQSK3PWX/5vNAR2MaJ8xs\nDYxXYdnqPMnMMjV/PjSnrDOb43rCbc47hf34lTsAUPUzOugY7E1UCtT2CljeQ12w3mBN4FwyTGbN\nP7NsOx54YpMYe9HxROa9fJ9UaIDioj9TxKadg+jNDQKOibVbB0RCG1RPlCxx8t64Nc+6z0neBcVD\nkRRgEDZ+fq3En1SejY5CEqZdA5PF8hXNgc403Tlx0fmk280YEd4WbnnHtLjwLshKev1539LnBNfd\nnwsmj9VtwD82/QuATzAXnzyOnstxxaLW3jyOnch5W+QtkvGYkp6iEtF5T7a89Za1UONZPLL1kcDx\nolQupMd63inikefpPX3hra1Q48GMZyhyzFsi72Es78c2P4nOXDde3PkqVvS8OSxp7l6m1IM14SX8\nz9q7sXGAVkS83WzzjTv9+/R2Er5K8wGyUmfEMG/NviKTc9DVl8fGnYPIOTlsGiwvS1R3Y3kPN/4u\nFs/mW12XAEYe8SMfwWObnsTTr2/H7//xVuixpXjxjU6ahf9WePjmQCAi70MMgZh3wG1OicVQfOvN\nKnWba6qwfADfsm6o9gl+WPIOURIjw7jNOUbUVfqHqwomtlcFysc42cRMXciJcnDxEz6GuKVjYmsD\nHGLjcxeNRnUFHWdBTiJ2Zbc3V3BTkU4YaKyOB65HJOUxy7syYYpriM96TKjQnbuQkoHjEphsMdLX\n9Dh6cr2iZGt2zZHivM6OUXD7a2ATGy/upPrvXBa11G0OACZbbK3fPuhnzGsOirYrVMrkudVVHVA9\nPL9qJ2761QtQDBq394h0P1TXLxdi5L1lexH5okv34d4JVmGQGeT3UQFxNbjwJ7UvT4nZKRiIW5oY\nLzQbpk7H1S/FefkCSTdcn7wNej5L8cm74PrWZU/Od3trjLxL1dOMjlV4YM1DcDwXhAAT26tw1GRa\nG19winCJC7evDs72MaKigTeMCUPfUAGPv7JVkJDcjc4cuRJqRXfA8pZDQNLFis5hgYQzBqphH1zA\naH3t0gLDldzmEnlb4eQtJ/r9v9fuxLLulaH7DVceV3RtaHV+WODxzc8AKPdqEELws9d/jf9dG1yo\n9PTnWRc8EiDm4cgvb7t4+LmNWL2ZlmxmbT+GvCvfF3rMvoCX5xUdF//96p34/gs/KRM7KvXWBGr3\nh1ns8NACfw4c1xNVDH9a81f86onnsKnTv++7s8K3sERBfr/fDUTkfYhBVcOzzbmVoTGZUUV1ETM0\n1ErkPba1UsiE0pPQ40c2paXPJHeYbJmXan5Lx7O9yzaPb6kVC4yBrI3KlIXjjmgX23lTkpipiZcc\nh6uxlxnLJm+pTeDoFkqSd6+6V4xHdpvLMWthgbsa0gkDDdWJwPWUlsNVJM3A9ei1NN5cnaIuccfz\nhCeBqDbuXf0AvcaqMWhPjQheuEv3W9e3EaYSA8nR+f33y+eXzRG3vAH4Cxtmeedt+sKvTib8/TUD\niurhjU19gOJC0VwQx4TrefA8iJg4J29uUaowaRmT7DYXjVmkBZur0wx5Bp4QZ+cNxC1dkLeiOeLe\n9hcly7tAnzdV84TbnBOXqcZD3ea9WXo8IUy6Vy+KEkYK/9nryXK3ugKTkXPe4wsxQ1wDNCcQ81ZT\nvbjnzT8LBb8f3PMq7n70TSxdxWK3hpSAVbMT1sQXBQm6HvGrGmRIuQWDdjl5F11byMNyD4ilxv3K\nDtUT3gWZvLmGQSl6MoMwSAIN8ToAwxPgcKpyj21+MqDBz/NKZPL9xV9XYe2urVjWvRKPbHxMfH7/\nk2tx9zPPIT7zceitawLqdjIxquke4bnY3p3BH59Yi+/9lraslWV4d2a7Qse4L8ixDP9C0cP6Aerp\nKG3y44sJ2VjbtwHLi0/AGE1DII7r4anXt+FP/1obOIaHRGQJYPkdEZu6BErCf/YzuxGl6WFW/HCS\nvgcCEXkfYjihbSEA4JSO42GZ5daAcFUzy7u1jr4oJnVUY1RzhbAeAQh3LHe5A74rm26XasI7R8Dp\nbIPT2eZvl9zQJJeCN1SJRNHvuGVqJs4/kVoZU0ZS17HIqAZQLPjkHbf0gIAMjAIjW7pPU20Ccxpn\nYFrdJKzr34AhbTs7h3TxcsyaK615GtIJk1qBsopcSUb9rPH1qEun/HMxxboYUzVzXSL01wFfxtXQ\nDJhG8GfCSSTjZGEp/uKptT6FUpiaLITj66sXbQ95FhNorvGTDGO6IVnOvshMNu/QlzBbwMiWNyGA\n6rG+6l65d0K27ImniVp2wE+kKuboPbKYWA50h1U7EAzZQ1DsONA5GvYW1pVMc0SiD0+aMxCDxa5X\nJm/umvcGqJiOVrsNg3JrTql0atsQJVtVVYVlXfAYKTAvCl3EObAMSU9/1HI8ufVZPLLhn3A9V5RM\n9bA2nUqImI3sNlfCyrcUV7yMB/dgeXMPSEz1PUCK6olqD368O1gFNZ4RLm0ZWTuHQk7DB0efAcDv\nA5ArOPjt31eL/Rw3PKntjV1BFy8PXcge7KGcjd89/1TZsX99diNyBiVEo3VtoFc5J38lNgRr0guw\nJtM6/lK1uqyUUf+/ax8WvyEZL+54Bc9L/elLUXSL5fr/kuXN8asVv8OPX75dhMe4Vfzwhn/ihy//\nP2zxVkKv2waoDhzPw6/+7w08tGRjwAshpH+55e2RMg+kKpP3btrfdrPnbLgGPwcCEXkfYphcOwE/\nOf67mNVwBCyj/PYYnGBUDzFTQ1XKwq1XHI0vf5hm+fK4LQBBvgH3ouwelC1vosHeMBVeVs5WD24v\nrJyPKUnfhWxqBj588nh89zPzMGMctRZiElnlmfEbs6jbvLBsIYobJ4rtinR+njRycvvx9OsUjyXE\nlMfdrSnPCsubeNTytgzNT3aDH1fk1xC3NMwd7y88eDcxUzOggCa1aFKHXMI8Dbyvugw5NGCqscC2\ns0afiqq+2eLvQHvOgHyqi6JXBCEKWup8z0jcNH0vCCccx0Qmb9MsbE+lMe+Cr3QH1wAhCLjNdU3y\nTngl91+aJy7/6hQMxE0NMS3OzmvT5iu6DZe4UPKViO+a5hOo6kiWN53LJ1/y+8bLMW+e8ObsGAli\nGzDa30Rfwbcqq6sly3Dlb2BNfRquPgRNVaEqCoqk3PImqitCSfLcPrrxMVz1xNegN9FSL9cjwoPh\nZYOLq0DCWgi5Q/WEBSqTN/GYfKtr+6ED9jwljARkHf3BnA3Xc7FhYBNMLyUWMLe8eBs2D/ou7oJt\nU8liR0c2xyxCRn6PLN0kyc8SbM9tx5cfvwlPrH8hMNwEy81wulqhQRM6/4E4t+KhR6WJX2rp619a\n5PXYdBHVP1TwNQXYgpiXBZaSmWx5bx7ahqU7yrPOf7Xy9/jNyj+Iv4u2G9B8v/Wl/8Y1T92EXfle\n/HrFH7Az0yme9VIZ1NV9a6HX00UQJ+A1fesD+6iJwYDbnH+XJ2nYc0+G63riPnLIev6lynUyIvKO\nAADQVPYjUspdedzyVlRPuNJrKmJCwjBgeTOrNpDMEaKqxlGZNANxW0teCDA0VfrkbmoGFEVBY7Xv\n9pXJm7+EYqbGXKBKILNazlavTtPjWpK+znhpkhx3hauJId8t7lLL2zTUACnpsWLgHKauBRc2jNhM\nzYSmqXC8oOXNyVtX9fLYqpQ3YKlWYNOpI09ERW6cv13zr5d7PbTa7XijawOKjg14KtobZcvbpN4F\nkDLL2/OISNrjpMFnvQAAIABJREFULwlFt0EcAx4jb3gaFIVlC3P3eWkSIRfaAc1GB2huQNzS/fun\nOYiZmp87UIhTS5yoILaBIjJSzLsI4mpYsqwbdz9MXZM524/r8nixl6mE09kORSEYcCh5X/Ghqaiu\nCU6vmhhCwaRuV8NQy8ibXoODdFJ+PoOWqNEuNVlhiwsu7AJQFz63vF3J8k5qKcQd2kRGUT3YbJ4H\n7SHEtBgaN58HZ/toAIBDbL8Gmn1Hykj4vyvFQ6Ho4q3e9cg5eaSdVnj9dWIMXUO+KtiWHuZKdw0M\nZei4RJMdRhp6y1rE5vwdD+/6LYrI4c8rngxc85Cdpde1fip01RALKNntrbe+BcegnhAufyyseOn3\ns9T5E17dvAFf+q9nxCLHigWJqbQ3Oq9l93rpb3ht34bAdtlb4HgObNfG13/+HP7th/8Sn29l5ZM3\nPPtdvLDzZfxr6xLkmOVdCHNJMw8cH2NNrDqwWUkMBkrF1vZuxuceuwbLu94Qn8ltb6EHr4mrJAI0\ncY4QgnvefACvdvnhCdvx0McSE4frQX8gEJH3IYzm2gROmNWKL15whPgskE0eAqOEcDVVCcgbkuEs\nbwC1lbEAoafiQWICgNYaP0lNjudyWJLb3M821/06TEk0hYu4AEBVih4XsFRLM+Cl97OaYGVWsuWt\ny+QddJsbuio0vGWYqgFNU6h2t7SY4Mk3pmqUkbec9CeTsxibNN+xEMtbjWXxt/7f04Q1TxPudtNQ\nYeq+Z0V0RXNMZPIOtSCY5e1fqA04BmzH893mAGJxlLnNP3bqBEHufFvOy9JnytMRs3RhvampPjqn\njLy9giU8EKQYR8YblFzGRX9O2Hf15XwrLONkaEzZMcR+3EqLWRo8zd83zix/T6X3z9BUOGD3UnwH\n/d5kXAqTlHTVcwerpG1sIWeb8DIVYpz8he95HqAX4RXi+FjHlTAddqzqsg531PJOm0nYDoSHpugV\nxQKAex8qrKT4XXHPx7Iu6vKOF5vhDVXD3joGAPDcm742+NZeupghjo6BQWZpMsubX6Wa3hUoAyxk\ng7+/jJ1hc6RAgyFCF778bT/05vVAMQEvkxZiQaVqfRxLNvtlc+NHVOH8U/zcDzW1C/IP8ub/eQld\ng9TFbO8YgYQex/qSJjy25xPj7ct+gy8/eQN2ubQpztbuTKjVammmkKQNI0au9Oc4dCxyxjtA3d6y\nbsCDq/8BAPjjW38Wn/FjHdcLvEMAQElI5J230Vvow5Nbl+Dny+4Sn+8azIuZiCzvCACoxfzRRRNw\nxBh/tT4hdQTcwSoU3pwdekwpucdMLaiQJJM3CZKZkDdlkBOpONpqq6T9yxcSlaZvRfKXWMyULT//\n/LpE3tzy1lTNJ9mSxUVx3RHwWMIUb0nKY96moQlXOOCX9nDi0jU1IBwiX4OuKtjUOYTfPbpOfM7L\no/RQ8vYXKLy0SoYWIG95MVOSw6C6UIkuLNiKhCnlNDjCCiCOQd3mhJM3LwVzoai0tj5XcFDgbnMA\nlim7zTUoACxDCyTNAUCB5JDQaC5CwtIRN+j86rU70KmsgWKypKd8TMwDKcThEgdEp5nJil70xXDY\ngi3LuscRQtDv7PLbz7Lvz7t0e6ezCTuTVNr0kxMvw1ktF9Ahs/71pqHCUYKVA2JRKB5PAhhFjEx3\n4OjmuXRqpC588iKosHIezGKNyDsAaLKiYhQB2/QXSACgUMvbIx6G7AzSZgq27Sc2Op4jkTf9jsp4\nWlLCo9v6cixbv0jnmeTpwAekBc6OPhZGcA08vIS6yGWyo99B5X7zrx1L50hx8M+XttA+5ZkiuocG\nxBzpii5i5tzw1iq7oShAYeN4EMeEogDZYkHEkkvj/tttP8FLU5WA0Iw1+XmYE18AJ/A1W/rx6rrt\nbJ4NjKrsQHd+FwaKfqWCHMte2fMmrftnv+MbfrEUP/uLb81yFN2i0DRwPSJCFv7AWNWJ68sJK0RF\n7oVF9JqsbMBtvnEbnfNdhV6YE16A3voWc6F71I3O3iFpvQLENpnbnC0M8g5eWeu3C/WIhy2dQ3h+\nVac/3ihhLcJwiGtxFFfNg9dfH7q9lLwtUyuxvOWENXr7501uxPc/O5+2/pMs7/GtJf5MMMuCn1sr\nt7wbEv5Cg1tIPGv5e5+Zh7PmjQ0dK7e8AYiM5dIMYJJP4SOTzg1+oWR5u92tZePxX8R+9q0MQzWk\nemH/+3gs2ND0snri0fX+NRoh5C3PtxJI6C8JA6gudMUQZXQVSRNtKRqX16q6AuSbZZa3r89OAuSe\nLTjCbQ4Ahkl8kvc0aBpLAJOS5gACm+TAeSNmakjr/uIrhz5R1uQWLDEPXoH1ZlcGac9yzRPhEL5Y\n4q1fu3O7YKMgLF5ueXsKJYo3B/3yqE1bbdz10Dq2ncdXM7Br3gKgCNLjC4BYjL2U9SIUBUjoSVw0\n4TwoTgyKbqOphu4vXP/FGEBUmAodP7f+O6uepIsgT4Preb4YDot5d2W74REP9fE62K4nFp02sSWl\nO2Z5mwkpt4Fu4yEEl1Vf8DnK2QUUbRe/fGgVVm1hjXpcXWyX8wb4dRLHFGI7ikoT2VZv7sMdf10B\nWymI+VWhS25ztsBgdegkmxZz2DkwhKLtQW9eC60mqKrW7/o1y6qqBKRhAdAOaZK1nnPZ78s10Jyg\nZX49OT80kA35/cmu+tfWlau6FdyicJtD8VhIyQfPc+GLqEwxA89mioKuCkVzgqV10m9Qq+wRTXgc\nhzDLm97Hj7Z/Bl6mIuClsl0Pv3/CL9+7c/n/4DtLbsMDT/niMZHlHWFYqHu4Y2aJNWwZWlD3N1Aq\nRh/kptoE6qvi0FQ1QO6TO2rF/08fU4svXzjdj8ejNL5OURvzCZ94Kl08MJd5Q3UCR030CVa4iAGk\nErIrmrfwLL/YMXV+TJx386pImNA1Bc7WsdQqIeUxfuIRVFoVpaeDqRmi5EgJqXU3VCMgvfjZD07B\n5R+YIf7Ww8hb2t+TpEcntNUFd9RtqIqOuGR5z2+eCxAFesNmP6Pe1ZHJ2zR2yaw6a+ozfptXx0Au\nHyTvbGIjI2h6nzVNoeTLXtqJlEvdpooHz6afxS0dST2Jwlv0+lzFFpa3V4gL0RauVpbxBkUSk8hl\nYN+/dscu/Owvy/HEm5ScSYaFW3jZGrdwTN+789yrg4J8XOY2R7IXUF2MVuaCFBOB73hsgJYUcosx\nriawoycLt6hDt1xRiREgb0BkxOftIjziIR9jFmM+IfIKAFoOt2pjL/64lNbzt6aaqQyqwlu32uVu\n81hSkD/XyOdVBbativtJPy/gsZe34ull27F5Fy2Rq02m/aQ/j7vNFYgcCFsqeWT3dzBrY8122mKX\ne0Dyeep2J4SAe43VWAbEU0AKcXGN37l7KdZvH4AxgvUzcHTkXliEpNMIBzZ4GZ+mKgErmkP+zdhM\nuY84hig5zEreroxdImID3+1N57A8abDgFkTCGkqSyVJGUhCrIyzvrK+q6OqA7gT7juvhSWe268F1\nCfNuqMgXJE8Zu8ai7QbG8GrXcmjpXvEbEfu8S4jI+z2GcLe5VPIVEvMOWJYSudek/Bfr/KlNmDra\nJ3MAAUEYDpnc4WkBlzk9p2+5VyV88RiZ8MQChBGVSGarS6IuLnkDPA26ptDEKkUBoIAUEtAhuarZ\nS8ojBIs6TkBV/wwa72OQyZk37pARqJsHVZKrkhYBCb085i27zT2pfj5VojKnKABcFQ3VCUwbXYu5\nExtQHauC5VZBiQ8FMuppqZhfh6omhkQmLHENDOVtFGzfbd6XXI7p09l99VToqsK6zrH5GPMMVNZb\nmj8TBduFoijwhmhoxEYOipmHQhTAlmLezPLut/tgxIPkzc/Vn83h+VWd+PsKSt6lljePLTpMMCb/\n+kLs7CmIuHavthHfff5HUHU6B5br51qIlynJQzFzIt5tKQlk8g6Ia8BVCjBYtUYpefNcjbybD1iD\nzrYxrByPC9FQ8n19K81gbk01U8ubu80JJe+EpQuXdtKK+d4PI0jeXPdAdPDzCsiKen3672lzx4rf\nnS2XWqksROKYoGI7qng+sgUnQJwAkMl6ICBwPMePeceyIIUEAFXyDri49wnfclR0ByAqFM8MzLWm\nKqHlcmD3UYkPQq3sogtqT4NC6Dhkb1fGLre8lVjWJz+jnFjzbkGUivE5cnc14gOpy2CqlvjMcT04\nnoO8mxeqisTVoaiOmGN6fSXfwd3ujkcXAJoNuAZts+zySgJ/n7LjpTkChkmqO0CIyPsww5566IZa\n3oGYd3mdNycbz/MCCWvyQqBUfrB0eygUglhJrbpsrVt6+PG8QQh/iR49tQkfP20irr5oBkzNpCtu\nNv50wixrSiCTN38RVqUs6KqO6sKkQHtUUzVEOdCYmjZ888jrMK5q9LDXaGhqoJZ9XHMdPnDkCNx0\n2Vz/slUFhVVzoearsKDZF27hgh4yKhMJ6JqKL104HfOnUq+CiQR9YRh+0h0tFSOB8h7enAWOgf6h\nYHY9APSxzm10kaMGLG8A0OtYwhT7bOG0ZurZYQRAyTEPnSQAKNA1BQumNWFEFQ3Z7Mr3wUqweHKJ\n5a0YBVimImWrJ6Brip/YptlQFF+qlQghGVVoxW8Z2ibmwHN8qV23zw8ZKUZRWN4WErSch32HbrJY\nrsVkMJnHIGXRf9/Y0oVfPkL7NDudbSDFOAtNcPJmI0pQi7Ml2URj3sxtTsnbRdzSka70UBmjrV1L\nyZ+7r4u8RxD7DRb1XRhwugHFhcZEgyqsBGrScRAiuc0VOW4vJe0x0ti4Y1C4r0lJ7kHRsyl560W6\nwGChB+Fh01ykkiE04AYXWSqzvHUvjuKGSbC3jaLbmSWqN9MFDsnR3vSKS8chk3e2pH4bAPSGLYjN\noNnmhuWTYFKpggIFBafot2FlY/EKcShOnC7CJLc5j6kHLG/NoUTMUGrdK6oHKJS4HZewcj0ahvIX\nOPQ7MnlbkLfT3QJnRzubA7o9bmmR5R1heBT3QN56iaVoGcGENeLJ2xXpv1wmskSqkyGsLWCY5Q0A\nY6voD5vYFsa0BF3VMtEamoFvf+JI3HrF0SXnpeTI5V1jpoZjp7eIuHiVVcmuRRMNW2QYEnlfcfZ0\nfP7caRjTSo/RNZW9YPh1aSJO1VSTQGOqBhWmbJkH57M0/p00E/jwiePQ3ugfo6kKvMFaJDedgOqY\n/7nc7IGjpabc2rcU+oJVOem4GnIFF7miCy3mZ1VziyWQxyBZ+l051vCFuc0NPRgWEYlVnobPnD0F\nNRUxen+IBuJqKHg5KLotXsS6ruITZ0zGNefT+9WT3wU9zsnbEucCAK1yF5qmv+W77l0dLbVJsVDQ\n67Yj1rLJF3MJkZYFAM9gCnBF+twkYjq83iZUDbIKDL0o9NKTajWyeceP+zJLTjGZNeZpuOXf5iPN\nyHv11h68vtFPsgJo8ppX4vZWrBwsNYZ7/rYJBP4C1CU0YU2zCsg4Q+iobGGeJhWEKFA1Rt6eDVM1\nYLOsZrHAqejBC7gPWsNmaJXsGow4aiuo0EvOLhey4fFuIkkFr9vZi9gUKpzCFy5y3NzziEgMEwtX\nISTjoChJ2Y6qYhnlJeENVSXoLw4grqThdnb4izUtaBUX3jgyMA5ZMjUjZYIHcmMAQPEEeRc3TMIM\n71xYmomiWxAxb1GD7eooFF0YqinKHh2X+Cp2IrFRh6J5yBSkeWTkrQ42wB1gZWWqC9vxqKdDs0Fc\ng3lwuOVNv38wa4v5cLvaUB1jybvsGY9behTzjjA89mR5lwovWKaGie30ITthVmvoS5KngHgEQQlR\nibiUEPIu7fTEccX0T6Bi2wkgmUqcefTIYcdqqDra6lOoqYiVfS6j1Hr33dYkKNTBj5dUz2pSScwc\n71tqhqYG6n0BX7iBZ30nDD9cUGZ5l2Sex8JKxbgbnhDELR1nzO/AvMmNWDituWxfSyuPmXPyVrhl\nzd2sRRfElPpCM3KXQx2Vdf6LWGQrexp0VaVubzlhx2KuVtfXntfZfSaOgSFniMqzshc5d5vH9TgS\nehy7cr2iJI+/zD+4YIw4f6eyRri947qJz35oqug0BwBoXYmCW4ACJZDMJ5frOBq1evM5Rt5snKpD\nCXj2tCTMup3wCnGkvUYqYcleuq/iL4DqUPJm46urjPueE83P6OcvfM8jovWqxvu6aw5yWQVLWJtM\ngy1aHTh0MZ2gNdod6XamSgfAU6FoLJud2DA1U2RNBxfQ/iKNz21N2gI8Ddtz23HHsrsgMvqlccLT\nxMKoM+tnO8ulcIBP3rwlqli4Sm5z3oa3Wm3CN46/KuCh4ffC1bLwiIdxDS04YVYrxrXUse22P5eA\nOG57Fx1vgLyZZXzVjE/jqhmfDswBvRd8gRLDUMaFpZk0YU3EvNn8cfJWTJFQ5rgeduWpp8kX86H/\nDhX8MSi6DS+TRmbVLH8Bwo7f5qyHogDeUGXAbS5yC3K2mA/iGJg7voWek2kimHpkeUfYDfbU67fU\nhZyMGWitT+G2Lx6DxaeMLy9XAkRrPyrmIFnGEonK33v6yJNRH68NxH5lWJqJq886ATd8bI7I+JXB\nm6+UeglKt/NyH8sILjgqmeWt6DbSyZBac0XOXA9+h6YpActbBpf7TOp+LH5PlndY9yQeo+eqcecd\nNwafPnsKmmuTqDQqA/uGldvFWekWfzEeN82vr1WkF7/O483SgqxdnVZ2PuL6lrdcIy7I3/W158e3\nV2H+lEY0pqtEaRBPaNOlhUtNrBo9+V7U1zOyZy/CU+YEdeCJ4oB4KmaOa0RTTQLfuuyowPa8W4Cl\nmaiuKF8EAUBRoyV7OUbeosENK9dzk53wFAfurkbs7M0hm7eF29RGHlrtdroAkcSBYixPQdHcMnf0\n8nW70DfAXMUa70jmBBZIwuOkuFTVLk5Jo6OizV9oen5M2iU2DCk8M5yXAQDqE3WoqYgJ1/1rXcuR\nJf2wJlBJUd/y1oU7l3sv7O0j4Q0wi5aR8zPbn4dLiBAb4QtX/syYY19FhtDx1+ktSFspGLoKj7e5\nZZamrdHjm5J1+OiiCWirpgaBYmVhjnsZWsUudk56n555lWaqb+rpFdfG3ea5IR1JI/heUHQbnsEs\nZ9tAf6YIS7NQCIl5wzGQLzq+qJLqoug6uH/NX+k1MhU7fo0i1q7QOm6xuJcaBdmOh60OFW5xu1tD\nLe+hrB2o8EjH6Hvig8eOwHc/PQ+WoUUx7wjDY97kJswaX4+vLZ4Vup1nVNdZ9Zg2uhZnL6Qu7GSM\nJWaFkTezvUvbBcqiJvKmM0Yvwk3zrw0mp5WgtjKGUc3h5M7JebiYOd8u9MdLkt7SJn0BKbqDdLyc\nvGXLu1RIxtBUv+SoBJwYZMtbLyFXTmAfnXQhxlaNwsiKkqYlAM6Y34HT5rXjU2dNLtv2hWlXBWr0\nwzL2E5rk1lc0jGzyCb+m+xhfI54n+kj3tEFvxw+O/ffgCT1VinlL99RgbnfPz3jXVBWfOmsK6lL+\nvXOKLAFLWrjUxqphezZ67R7qvuS92y3//GkjhYq0CsXTce6xNI9A04KLy135PliaiY+dOhFnzO8o\nmwvCwgCdPVRHnN8jTmI7MszqtC1s685Qy1tWA+WhBdvCSbOobn/CYG1Nx7wu+otzwn9pdZcIJ2ga\nK8nTnMACSSgPcnI26QJjRLoVlsmS+jwNHmhfe9uz0dMnJToNoxxYvXURLM2kSoeyVCk2+vMhW95C\n598RcyD2Y8f/c9OT6NPXQIlTD0ap5a1oHvQxNJuee5Fk8halWCo9vi5OibE6Sc9jtKyHVs3ugfQc\ncm8N11bo7suhM0sJ/Sf3vFn221fTPSC1G0CKFrxsBdZvH0BPn4OcUxAiLVzbgdgx5G0XGgwxxgIZ\nQme2G9PrpooWvdzyzhTZ74Qt1OIaj/v7mgeO6yGDXpCiBZJL0wx1fs/ZImkoZ0uuewOVTGggFgcq\nUxZMQ0XRdvdoYO0vhJs+EQ5ZWKaGK88tt644UkYS35p/HSrMVHhMOqS1J3/Zlbb+k6340pZ77wS+\n5R1O/pogb/riLiXvuJThHeY214gpvqd0gUHJV8XE+Cx0NFQFtnELf3duc+5Wntc8B/Oa54SOP27p\nuOD4saHbUrFYwPIPu0dJPQk4/vbqtH+9llcJe/0UWJOfB1GZFSBZhZahIaZbSOoJP8bo6dBUBTFL\ng9vTgqJuw+zw5SHh6mVd32TLyCkyYpeItyZO44V9hX5U6JXg7RsURUFh9UxY41+B7dlIWAZqrKQI\njWglnouck0M6UYdpo2sxbXQtHlqyEYWVR6FuyhoMEhazJ4Bjq+hoTQjPByfvXqaRbqoxbO/JImbq\nouc44MtbLpzSjounUNnadExanNWzpL1Aq1z6Ha45CKj1rCpAmmON11mzeD57oSeNhL/wJCo8uEhY\nOlzVLbG2gwsYTt5xRp7phBH4neaJn+XtDXBi8suYeLlVIJ9B+v+COgjVGoKXjwsPguyB4z9z/rsy\nNBW9/QRWo+/9Kap0DPUsVl2TKM/VCHw/I/8iyeP1tT348f8+g9gR6+EO1ACuIQiZQ6vugqIAxS3j\nAU9HvujCLChQzaLoC6ym+kEIdWsXii7i4HF5FzZT4RvoKxeEGsxnAZjQ0rS6okKvQhcQ0DywHQ8O\n8ZUCs3lb/K54eCJrboNVxWrfPRWVcbqIzjssVBUbgDbiDeSKp5XNzYFAZHm/B1EXrxk2mexblx9V\n9hmnZbIbgi61yt8JRFx+mFOqSnncXkZCcmvL7U55rJlnm4dZtfzlf3TNSTh7zKmh35PYjdv8ncLU\ntYALN8z7kNb9a7JUU7jhAZoMN29C0NqX3eomW4BUxZi1ThSAKNBUBcmYgfqqONydI6ES2UrSAhYz\nAD+jHxAvYpng5Xr+ilgSs8bXi0XloglzUUmakXcLyDm5gJiPripCHpQjVhL394aqMdM8xf/A1QEo\naK5Jipp8txict7pUBTp7cxjMFuF2t+KktuMB+PKWDekKUXWRNMt/G7LkbYJtH4qvg9FOFzky2QkJ\nYJ6Mp9iIaVbwuXV05L0cYpVDVMSmpLJDBifvGHvuUnED8o8jD2r1FlbPFORbEaf7ajXbpQ575RoO\nAPWsKUYRhCWr3fjxuThnQfnikqvrmYYq7jl3mxdUujyrZ5Z3Q0U5eZeqNxJPRc7JY9naHiEA43bS\nZ3XZup7gHHDPgOwV83QoCi+1I1CT/SC5FFRCyZ9b3takpbAVapWv3iDVkrt8AcF6rTdSQZZRFvOI\nSZZ10XHhoCg8BnLuBPds8GeBjRhxg3fQo+SdSa6F0bwBm3qD7UoPFCLyfp9hREMKN867BjcvuEF8\nxt08Lvt3XP+5uOGoqwPH7U/y5pa1S8KTO9QSy7s05t2UpOpN9bHaQO05fzlzyzuMGPnLP6xjG/+e\nZIjlffHJ4zB+RFVACW5fQL9fETHN/kJ5b2ceFgAASw+St6oquPC4oDu+QrIkeQ9s/pKloQefMK5f\nPBvzpjQibUrk7Oplcyxn3E8f2YxPnDEJx83wBXZ4xj9A5+vKc6dhFksMvPCEsRjZQL8/5+QDSXma\npsLZOg7FdVP9awxJ2otp/vg4cTbX+Za36ypI6v51N1VVwiME67cPQFVUnNi+AIBfTiff0zE1I/zs\neAauugbQOefQG1g3L4mYDFWjiyJmeXuqLeLoHPa2MSDwUGxaRj/gmvNmubdJMQsgRKEd5cDIW/N/\nG0VlqGwMHfXU82GOXAU11ReYJ/n7AKDIiJfY1LXb0ZTG+NagZgPgaxYQ4ru9eYJWERkYqiEWdfXp\n8pBYaSIeHAMFL49ETIfCeoDzkM+ytUHyVpmSn+w14Za8Yuap7oHmwstUwjI1arm7vuVcTG1mx0jN\nhQIxawIt3Y8RqTbUWLWB8Sqai6ydp78VtmjpzxQDx1tSCaCzg4Z3+D1/bPNTeHzz037mPQn3KO5v\nROT9PkRDog6VVvnKmbvGLSWFpmRDYFtIXtY+QxXkHX5SlcfaWcy71FoZXdmBzx7xcXxp9hXB47ik\nNCPv0pp3gFs1VIq0FMJtHmJ5nzJnBK67ZFawZn4foCgKrr5oBmbWURWz0sQdAEiYMVHrbGkmkjFd\nkLKmBUkLAJKmFONn19DMFjgcfOlVmbLw6bOmoDImddjytLJEx3qplOeoiW1YMK1ZzB2AwPOTCLsG\naQ5ly5vfS/klHUbeimv6nhNO3jVJcbzreUhLY2itoZ6ATN5BIqajwkoHqiHkMVZYaehvLkL+9YXi\nM/k+xIzdh5scj8BQLKiJQSjxAXhKOXl7fY1I6Wm4Zl/g+NLKCQFXF2JKybgRVC5TWaxXImdu9QF+\nXH+4RLiiTscwsq4ON1xKQz1hiZIJk96zwWxRsjqZ2xw5VJgp8ZzIz5x8DYCfsEk8DXllANvct0SN\nNo9Db9hRrtYG0JJD/qyLOTviaRH+IPkkYqaGfNHBUM6fI0/3NQ9Kx2N0rGJzRFATr0KMe5mE5e0K\nsR5O/rmCFPPWHDom1QOxTdibJtFxSc/tfW89KMJYils+twcCEXm/j/GlC6ejrT6JY46gJQ+cvGWC\nGsvqoxtq4uUn2EfwOHRYpjbgW+Ya++2kQmq5p9VNLluAcLe5RuiPKszyPml2G7568UyMaPDJ6+On\nTUR7QwpjWqk1EbS8939ayOSRNbjsiPNxycTzsajjxLLtluRaNzUqQiMatygKNFULvDiC5E3nroy8\nSxwnSSNoeZeiPu6Tt0zEHBVSA5rQBUiAvP2xcs+HHDoI08h3XCI8LPwl3taQFDK6iZiBasn676j3\n3fiJmA5VUZHQ/WssXfAYugqST6Hw5mwU101FXaU/3ngIecvEWSi6mKgfDUVzoTdtgIci4iELkLRR\nCaIEO7uVhif880slmpoaIG9P9ZOkOJRA1QDvXS/FsaUua45OiW/e+A7RwS6szDNlMNlbqVaeyt8S\n2MghLXljShd79BqY5rzJa8jpta8k//TVANl5t3aXS6USxwCIhoZqdi8kmeOWDno9Zx45HjGTajP0\nZf2ySRKKQnq/AAAbb0lEQVQbCJxfHo+i29CbN7BrTIjx8Xtijl6OdblV9CBXFzkAckKbodN7Egif\nlNxzy6I/suaaYEXJgUJE3u9jTBtdi29/4ihhhXLXuKym9qULp+Nri2dhTMv+eyC55T0cefPt6YSO\nb19+JCrfpqv6iDHUHdZWzVyKIdZFzNQxqaM68PI5dnoLbrr8SBh6iOUdco79AV3VcXTLkaFjNAzV\n713NSJeTN19Y8aoCUzUQtyTVOmF5S33R6ZkCf6UMyfIOJW/frZowysm7UnqRlxJj6THyNXLrkkus\nAggo1vE6esfxfCEPRkS1FTGcMa8DC6Y14XPnTEW15Sccjm32a/m5cE9ausbSaxC1+P31cLvbxLMD\nhJO3vMAp2C7aY+MB0HI7opAyyxsAKqQmL9yKa2sIL1NEqbWmlv825PvUW/Sbhvglf/52N6RxkRyO\naU+34YNjTgsQZMqS480avEwaWsUuaPVbQBQvcDwAjKroQEqpFp2+xjbV4rR57SK0YW8eL/bV0n1C\nOnU48FBGQzUTKUr6IaW8QRu3jKqvQ8LSkc07yG0ZAQyy6xS96/05mDLC9x7yTPWkkfS9H9K+y7LP\n0jE4BkawBQ4kt7mha9QL4egY1ZzGly+cXrbodPUcTM1EOvHOQmtvFxF5RxDglrfspo5bOsa1VQ13\nyD5B3UPMW1jm8IZ/2YXg46dNxFXnHYGFU6hs4R7lW4eBoRnCZbuv53gnMHVNvMy5vCTPOOfZ2jUx\npg6lKIhJ8WruNi9VsCq1vNvSkmBMyAtVJtQwy1te1ISRe3wYy1tkrEtWolySyMvRbMcT18jjoYqi\nIBEz8IkzJqO5NonqmL+gbK3zn9EjxtJrr5LIPVGywCgNxUwf689XmEuYuJqwVfNFF0nTAnE1EVMP\nI++URN7cyjv32NE4//gxZfsSV99ziZFENos6ThL/z/UQZOudZCqRe/HkQGy/QiJfRVGwqOMEJGzf\nQ5OWyRsKiutoAqLesLnseAC4es7nME+7UCwARjdX4YLjx4pyPrenFflXj5PGT3uNDwfujeGWt73N\nn6ch0Bh52kxhVEsFXI+gp9dD88CxwXOwRe8lp4zHF844DpW5CQAANUkt85SRRJznHYQtJFzd98rx\nksGqbrg1a2jioWvgklMmYOro2jLvQ09uV6gH5kAhIu8IAsfPpAlJs8aHtxvdXxhTORJAmHVIMb1u\nCh1P24K9Om/M1DFjXB3SZhKWZgaSrvYWPEZ6INzme4IpWd5claqmgvc7py8MTmxFtwhLiqNazHug\nqzqumvFptPfRspVSWhhd6ddUf2NxeQWCjHgIecsItbyHiXnLXp20Qq1dXv8L+Ja37bpoSNDnkBBg\nzsRgDgYAVPMFDILiOVzJri3V4o+x1PKWyPu4GS1oqfWvIWUl8K3510IfkhY4ro6PnEItyUVzR9De\n6LYpuqrFJaW9tnrqrm9IBpvoANSDcvq8Dnz9yC/jxBHHBM4vo3YoqONAXE2QCQBMrh+Lr5TkfJSF\nPzxdlNQBCP09aNLzLSc+AsBXPngs4GqC+NIhxxMoYlw8h+Wy0yYKWWRSjAnLXHZpA0DlllNwztgz\n/HOxPIiKBPME9jYF8hIA6k2Z2O7f95baCmiuf2/5d+SLDlRFRYfH+ruzkreUZHmHClY5BkYIqWMF\nbj99RovJbfQjVy+rfvHnItwDc6AQ1XlHEDh9XgcWTmt+227qfcWF4z+ICdVjMatxeuj2SbXj8b2F\n3wyWK+0FNFXDNXM+H3AN7y0Sehx9hf6DYnkbuubXm7JabRHzZqRTKxGX/DIxpSz6CTVjYbkZAD1l\n7D0i5WeOj24O96x8ceZnsHFwS5m7tBRhZYmyNR6WkAYA7eZErCg8g+aUb/0J8nY8zG2cgXV9GxDL\njMTZx00qO77GCo77psvmYiBbFHM1qmoEsCV8DHweZ42vx8dOnRgcu6WjLl6LOSNH47lupn3u6Zg/\npQknzaZCL6+s7mJSpdQzInsqvrZ4Nrr6cuhVN4nvT5oWvvrJo0TYoCXVhIUtR+GxzU/R87s6IHHC\nCGUatrxYg/icf9APSohGVYKJi6RE2lh87vj3Jox8PdZ61elpQoKFX266bC5Wb+7D5JE1UFdUwovv\nYseXPwfHTm/GP5dyOWBK3o01CXz90jlYuWEX/vPe10DsGHXtl1xDtVGHk9tnYkzlKNz94t+xsZMu\ndqaMqsH0TbUY0ZjCX5esByGK8C6kzBTGj/AXXifPbsPmt6rQyYVYuDgMlzw2LZCiJRZZKTOJmDK8\n5U1cHUdPbcIf/klbpBbfnIPYnL/BNQbE9pgxvOt/uGf9QCCyvCMIKIpywIkboC7Go5pn79aqTUuZ\nrfuCpmQjUua+kT9A3c5pI1VWc/5uwNJV1roRqGSJYaUxb9EUAcGyt9KSLz6DpIS9Dc3AhOqxGF9V\n7sLlGFc9Bie3Hzfsdo6w8IdcBx6WkAYAU5Nz8NkjPo6zR/v19kdNpkQ+sb0auqrjkknn47w5c0Q+\nggzZbQ4A7Y1pTB3lx65HVUqysiXPkio66ZW7qrkVP76+zf/Q1f0sZQCmqQUy5mXhoLilo70xHcgb\nOG56G1rqgs9jXPYGuDpOnOV/XzpBVdZ4YlmYlRhIOvR8adLW+iTOOYY1B5LqpsPCHzWDM+H2NqCt\nOE/McXtjGiczmVut6Lv+wyz3uso4KhL02r2S52DyyBpceMJYv1lKyTV091PCHVXZjqnG8aLne1NN\nAl+4YDqV2iWqyI8wVB2WZiIVN/CJMybhyxdOR3tjGpMa/C6AHz91Mlrrk2KR1TdUCCRHUstbCx0P\nAMAxkIobuP6js3Hy7DYACohtwVN5A52g5X3JxPOFpxAID58cKESWd4QIIfjY5ItQcIvvaAGxr9B1\nFW5nG2yjgCvPOx8AUJP21a+AYDKWXH5klpK3SJ0t/56rZn66/MO9wHnjzsIDax7CxJpxZdsaE37o\nZTjytkwd0+qCNevnHDMacyc2BKoBhgOPaZdm1nPwpD7e31lGW30SmzuHUFc1/MtWvi7iagGXv6Vr\nAZd02Eu7OdmI5mQjtmd2oqmy3LshW84nzhyB8SP8fToa0wAUmEocBZIVVutpR7ULyeOEEYcChS7M\nJCI6YWYrTpzVhhNnt+Hz/1WEmuqDSozQZ/mCo+bi0aWNWPyhCaFzoBerwIVd08N4ssZVjcZLna/5\n1QESqtIWyFZekkUt81HNVP50guT+lhdn3PuSZImH3kAN1FgWtudn4C+QmvzMbZyJf215BgBNPj12\nuh8uaapJYHl3HGqKJr8ljSRivN+BY+L85o/h3rcegJryLWuAVtmMba3EkZMace+W5diapS4U4hgB\nsaKjW47ElNqJeK2b9q1/N2PeEXlHiBACUzOHVak70DCYhKuzdRzqE9R6a29M4ZxjRokOaTweXB+v\nDVjbZoj4DDCsmN07wokjjgnGbSXIRCFaNZYgbKyqqgTaq+4Opmbg5gXfGHZxAAC5l06iL9sPBD//\n6AcmoK0hxayr0rHTfyulxjtfuXB2YB/TUAPkHQ/pLqcoCq6Z83m81rUC0+unlm3XVA03zbsWf1n3\nMOY1B88/aSQlNiPXgEJsA5QEJRdNU8X9VhUVcT2GrJMrkTv1O7DpJIbCiqNRlQ4nlTEtlbjinOHl\nluOZDgwZW2FU9pZpP3AsnnQBptVNxuyQMFgypkvtR6llfvyMFpxz7KhABYvrlmfX88WS21/ni+WE\nYGTFCDQmGtCSKs+h+dAxo5B5eRJeGKKqZykjAVPKjxhd3Q4vlxbkXZr1P7atEvW91YK8DcUs03pI\nmynoqg7HcyLLO0KE9zOSMQOXfmBCwPpUFAVnLRgl/q6NV+PauVehxqoWtdMAy1QPwbvUKyGARR0n\n4G8bH0d7upwggXIvwb6gcpjOdhy3f+kUhDlPYqaO044qb4RSio9Nvggv7HwFExpaA5/HTC0QTx7u\npW1qJuY2zRz2/PWJWnxy6uKyz6tSFlrrkujcUgN97Aa/R3XJjeTiIh111Th/8Ww8+vwmHD2Fkpii\nKHBcD4CCUU27n6fhoCkGiqtnY+KY6mFzH3Z3jRPaq1D7Vgp96BYKZJapBcIbANA7SGPSFSG9Cnij\nkXFVo8u2AfQ6bzjqK6GeBUPXcMmcU/DCE4/T79aswH6WqQXaKB83pTyMJHdPTJrhXRJrrCp05roD\nuQ8HGhF5R4hwCIJn/u8OYaSol3Tt8t9T7z57nz36VBzVNCvUnQr4mfEHEqX913eHay6eiQeeWheY\n+yObZuHIpvIOfg3VCZw4bQyeHqB61/EDYHGdd/wY/PTPWRQ3TII3SC3x0kXYpJrxWLVrNU4ffRLG\n1ldibFu4FT1hxL6Ve/L5U/YxPUpTVZw//Rj8YvkGuF10XsMWmJUpujiZPKqmbBscE1+c/CWMqKsu\n38awu/CWoer4ztHXI+fky/bTVQWktxVesg+jnWNw6YfKEyPlBWKlVU7eAK3+6Mx1v6sJaxF5R4jw\nHkLpy4n/fRAMbyiKMixxA8O7+A8WJnZU42sds/e8I8MHpx8JrO2GSzyMZuWP+xMzxtZhbGsVVm30\nPQSliYeXT/kICm4xkMAYhpHDtOfdEy47bRLufvRNXHRSeV7D28XMhmn45lFfxdeefx1AeF+Bs44e\nicqkhWOOaA58fvnpk/Dy6i6MaWh6R9LE1bEqhFG/rqtQMrUoLF+A6qnhYYFKSU2wKhHufeClm2Hh\nkwOFiLwjRHg/4GCw9x5Qmhl/uCFhxHHxxPMO6Hc01iSwamOv+LvU8k4YiVBteY5vXDoHb27uxbi2\nfVNIbKlL4tpLyj0Pe4vGZD14NnxYuMTQNZEhLmPhEc1YWELo+xO6poqSjPgwuvNT6yahVZuADVuK\n+MAJ4eWtnLwjt3mECBH2Cj/43AK4XnnSz26SzQ869kfM+72OxupgedecCeHW4XAY3VKB0S37ZnUf\nKBxKizZFAbhBP5znPWkkcN2xl9Me4lY4ZY6pot6RltSBW2iUIiLvCBHeA6geJpt4yqgavPRmF2aN\nrwvdfjBxqLnND0U0VvtW9e1XH79XMfxDFeYwCmXvJj555iS8sbGPlX3tObSkKsqwxA0A46vH4gfH\n/ntkeUeIEGH/4NjpLRjVVPG26qbfbZjvASI60JgyqhqTOqpx9NSm9wRxA76lezBx9NRmHD2VWsn7\nK6fz3SRuICLvCBHe01AVBR1N+67xfiDwqbMmY/32gVDVtAhBGLqGr148fKnZ4YTT53Xg2eXbUfUu\nqDjuDfzQ0qEYXBoeB3Qpd/PNN+PDH/4wLrroIrz++uuBbc8++yzOP/98fPjDH8Z///d/H8hhRIgQ\n4RDC/ClN+MjJ4/e8Y4T3FM4/fgx+eOXCQBOZQwEXn0wz6WVltsMBB8zyfv7557Fx40bcc889WLt2\nLa6//nrcc889Yvt3vvMd3HnnnWhsbMTixYvxgQ98AGPHjj1Qw4kQIUKECBHKILvQDyccsCXQkiVL\ncPLJJwMAxowZg/7+fgwNDQEANm/ejMrKSjQ3N0NVVRx33HFYsmTJgRpKhAgRIkSI8J7CAbO8u7u7\nMWWK322lpqYGXV1dSKVS6OrqQk1NTWDb5s2bd3u+6uoE9P0cI6uvP7RigYcronl854jm8J0jmsP9\ng2ge3znejTl81xLWSjV59xa9vdn9NBKK+vo0uroG9+s534+I5vGdI5rDd45oDvcPonl859jfczjc\nQuCAuc0bGhrQ3d0t/u7s7ER9fX3otp07d6KhYe/EByJEiBAhQoT3Kw4YeS9YsACPPvooAGDFihVo\naGhAKkVrTdva2jA0NIQtW7bAcRw8/vjjWLBgwYEaSoQIESJEiPCewgFzm8+aNQtTpkzBRRddBEVR\ncOONN+L+++9HOp3GKaecgptuuglf+cpXAACnn346Ro0atYczRogQIUKECBEAQCHvNBj9LmF/x2Gi\n2M7+QTSP7xzRHL5zRHO4fxDN4zvHYR/zjhAhQoQIESIcGETkHSFChAgRIhxmiMg7QoQIESJEOMwQ\nkXeECBEiRIhwmCEi7wgRIkSIEOEww2GTbR4hQoQIESJEoIgs7wgRIkSIEOEwQ0TeESJEiBAhwmGG\niLwjRIgQIUKEwwwReUeIECFChAiHGSLyjhAhQoQIEQ4zROQdIUKECBEiHGY4YF3FDmXcfPPNeO21\n16AoCq6//nocccQRB3tIhzRWr16NK664Ah//+MexePFibN++Hddccw1c10V9fT3+4z/+A6Zp4sEH\nH8RvfvMbqKqKCy+8EBdccMHBHvohg1tuuQUvvfQSHMfBZz7zGUybNi2aw71ALpfDddddh56eHhQK\nBVxxxRWYOHFiNIf7iHw+jzPPPBNXXHEF5s+fH83jXmDp0qX4whe+gHHjxgEAxo8fj09+8pPv/hyS\n9xmWLl1KPv3pTxNCCFmzZg258MILD/KIDm1kMhmyePFi8o1vfIPcfffdhBBCrrvuOvJ///d/hBBC\nfvCDH5Df/va3JJPJkEWLFpGBgQGSy+XIGWecQXp7ew/m0A8ZLFmyhHzyk58khBCya9cuctxxx0Vz\nuJd46KGHyB133EEIIWTLli1k0aJF0Ry+A/zwhz8k5557LvnTn/4UzeNe4rnnniOf//znA58djDl8\n37nNlyxZgpNPPhkAMGbMGPT392NoaOggj+rQhWma+PnPf46Ghgbx2dKlS3HSSScBAE444QQsWbIE\nr732GqZNm4Z0Oo1YLIZZs2bh5ZdfPljDPqQwd+5c/PjHPwYAVFRUIJfLRXO4lzj99NPxqU99CgCw\nfft2NDY2RnO4j1i7di3WrFmD448/HkD0e94fOBhz+L4j7+7ublRXV4u/a2pq0NXVdRBHdGhD13XE\nYrHAZ7lcDqZpAgBqa2vR1dWF7u5u1NTUiH2iefWhaRoSiQQA4L777sOxxx4bzeE+4qKLLsLVV1+N\n66+/PprDfcT3v/99XHfddeLvaB73HmvWrMFnP/tZXHzxxXjmmWcOyhy+L2PeMkikDvuOMNz8RfNa\njn/84x+477778Mtf/hKLFi0Sn0dz+Pbxhz/8AatWrcJXv/rVwPxEc/j28Oc//xkzZszAiBEjQrdH\n87hnjBw5EldeeSVOO+00bN68GZdeeilc1xXb3605fN+Rd0NDA7q7u8XfnZ2dqK+vP4gjOvyQSCSQ\nz+cRi8Wwc+dONDQ0hM7rjBkzDuIoDy089dRT+NnPfoZf/OIXSKfT0RzuJZYvX47a2lo0Nzdj0qRJ\ncF0XyWQymsO9xBNPPIHNmzfjiSeewI4dO2CaZvQs7iUaGxtx+umnAwDa29tRV1eHZcuWvetz+L5z\nmy9YsACPPvooAGDFihVoaGhAKpU6yKM6vHD00UeLOfzb3/6GY445BtOnT8eyZcswMDCATCaDl19+\nGXPmzDnIIz00MDg4iFtuuQW33347qqqqAERzuLd48cUX8ctf/hIADX1ls9loDvcBP/rRj/CnP/0J\n9957Ly644AJcccUV0TzuJR588EHceeedAICuri709PTg3HPPfdfn8H3ZVezWW2/Fiy++CEVRcOON\nN2LixIkHe0iHLJYvX47vf//72Lp1K3RdR2NjI2699VZcd911KBQKaGlpwXe/+10YhoFHHnkEd955\nJxRFweLFi3H22Wcf7OEfErjnnntw2223YdSoUeKz733ve/jGN74RzeHbRD6fx9e//nVs374d+Xwe\nV155JaZOnYprr702msN9xG233YbW1lYsXLgwmse9wNDQEK6++moMDAzAtm1ceeWVmDRp0rs+h+9L\n8o4QIUKECBEOZ7zv3OYRIkSIECHC4Y6IvCNEiBAhQoTDDBF5R4gQIUKECIcZIvKOECFChAgRDjNE\n5B0hQoQIESIcZnjfibREiHC44ZZbbsGyZctQKBSwcuVKzJw5EwBw3nnn4UMf+tDbOscdd9yB8ePH\nCz3rMHz0ox/Fr3/9a2iatj+GHcDOnTuxbt06zJ8/f7+fO0KE9yOiUrEIEQ4TbNmyBR/5yEfw5JNP\nHuyh7DUefPBBrF27Fl/60pcO9lAiRHhPILK8I0Q4jHHbbbdhy5Yt2LZtG6699lrk83nceuutME0T\n+XweN954I6ZMmYLrrrsOs2fPxvz58/Fv//ZvWLhwIV5//XVkMhncfvvtaGxsxIQJE7BixQr89Kc/\nRV9fH3bs2IGNGzfiqKOOwg033IBCoYBrr70WW7duRVNTEzRNw4IFCwI9ijOZDL7yla9gYGAAjuPg\nhBNOwJlnnokf/ehHIISgqqoKl1xyCb797W9j48aNyGQyOPPMM3H55Zfj/vvvx9///ncoioKdO3di\n9OjRuPnmm2EYxkGc4QgRDk1EMe8IEQ5zbNmyBXfddRemTp2Kvr4+3HTTTbjrrrtw6aWX4vbbby/b\nf+3atTj33HPx29/+FpMmTcLDDz9cts/KlSvxk5/8BPfddx/uv/9+9Pf348EHH4TjOPjjH/+Ib37z\nm3jmmWfKjnv22WfhOA5+97vf4Q9/+AMSiQRaW1txzjnn4Oyzz8Zll12Gu+66Cw0NDbj77rvxxz/+\nEQ899BDeeOMNAMCyZctw66234r777sO2bdsOSy9DhAjvBiLLO0KEwxzTp0+HoigAgLq6Otxyyy0o\nFAoYHBxEZWVl2f7V1dUYN24cAKClpQV9fX1l+8yePRuapkHTNFRXV6O/vx+rVq3CkUceCQCor6/H\n7Nmzy46bNWsWfvKTn+ALX/gCjjvuOFxwwQVQ1aCNsHTpUuzYsQMvvPACAKBYLGLTpk3ieN4+debM\nmVi7dq3okxwhQgQfEXlHiHCYQ3YrX3PNNfjWt76F+fPn4/HHHxfNPGSUJqSFpb2E7eN5XoCIS0kZ\noL2M//KXv+CVV17BP//5T5x33nl44IEHAvuYponPfe5zOPXUUwOf33///fA8b7fjihAhAkXkNo8Q\n4T2E7u5ujBs3Dq7r4pFHHkGxWNxv5x49ejReeeUVAEBPTw9eeun/t3eHOAoDYRTHHyGYJlwAMAjg\nAFROSC0STCWCIJCYBhwOwxEqegIkuqLBbRN0LQaBxkBZsdkaDJutmeb/05PJ517eZCbz9bYmSRLF\ncazhcKggCOQ4jm63m2q1mh6Ph6SfVv97VJ/nuXa7XdH+z+ez7ve7Xq+X0jTVYDAobX6gSmjeQIUs\nFgvNZjO1Wi3N53MFQaAoikrZezqdKo5j+b6vTqcj13XfGnq329V6vVYYhqrX6zLGqN1uy3VdrVYr\nNRoNLZdLZVkm3/f1fD7leV7xVWq/39dms9HlclGv15MxppTZgarhqRiAj1yvV6VpqvF4rDzPNZlM\ntN1ui3fn/3U4HHQ6nbTf70vZD6gymjeAjzSbTR2Px+J/4tFoVFpwA/gbmjcAAJbhwhoAAJYhvAEA\nsAzhDQCAZQhvAAAsQ3gDAGAZwhsAAMt8AxJ5C+54P8QOAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAe8AAAFnCAYAAACPasF4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzsvXe8XVWZ///e5dTba3pCQiAJCSWE\nIJGmoSSgjsg4gmCb4Tf+dCwURUdEQXGs41gYFQvDiIyIiKIIJIAgEBJCgJBKertpt59z76m7fv9Y\nu55zboiQBCL783rllXt2WXvttfden6et55Fs27aJECFChAgRIhw1kF/vDkSIECFChAgR/jZE5B0h\nQoQIESIcZYjIO0KECBEiRDjKEJF3hAgRIkSIcJQhIu8IESJEiBDhKENE3hEiRIgQIcJRhoi8I7yp\nMW3aND796U9Xbf/iF7/ItGnTQsfdcMMNoWOWL1/OBz/4QQB2797NCSec4O3btWsXH/vYx1iwYAEL\nFizgkksu4bHHHgPgpptuYuHChSxcuJCZM2fy9re/3fudy+VC19A0jfvvv/9vvq/Vq1dz1VVXHdSx\nDzzwAF/72tde9bVcvNbz3wi46667+P73v/96dyNChFeE+np3IEKE1xsbN24kl8tRX18PCBJas2ZN\n1XErVqxg/fr1IZIeCZ/97Gd597vfzW233QbAqlWr+PCHP8zDDz/MV77yFe+4+fPn8+1vf5vTTjut\nZjvr16/n/vvv55JLLvmb7umkk07i9ttvP6hjly5dyvnnn/+qr+XitZ7/RsAHPvCB17sLESIcFCLN\nO8KbHm95y1t49NFHvd9LlizhxBNPrDruuuuu4+tf//pBtblp0yZOPvlk7/fJJ5/M4sWLGT169EH3\nq6+vj09+8pO89NJLXHHFFYCwAPz0pz9lwYIFmKbJypUrufTSS1m4cCEXX3wxS5cuBYRV4IILLgDg\n1ltv5atf/Sqf+MQnOO+883jve99LT0+Pd53ly5czffr0qmu98MIL/OM//iMXXHAB73vf++jq6gKg\nu7ubD3/4w1x88cWcf/75fO9736vZ18p7ueqqq1i4cCHz58/njjvu8PatXbuWSy+9lAULFvCBD3zA\nu85I26dNm8b+/fu9893fy5cv5/LLL+fqq6/mM5/5DAD33nsvF110ERdeeCFXXnkle/bsAcC2bb7x\njW8wf/58FixYwC9+8QtvrL74xS8CsH///pD15MknnwTAMAy++MUvsmDBAi644AI++clPVllMIkQ4\n3IjIO8KbHhdddBF//vOfvd8PPvggCxcurHmcbdssWrToFds855xz+PSnP82dd97J1q1bARg1ahSS\nJB10v9rb27nuuus45ZRT+PWvf+1tt22bxYsXoygKX/7yl7nqqqtYtGgRH/3oR7nppptqtrVo0SJu\nuOEGHnvsMdra2rjvvvsA2Lp1Kx0dHYwbNy50rVwux8c//nGuu+46Hn30UT70oQ9x9dVXA/C///u/\nzJ07l4ceeogHHniArq4uLMuq2VcXP/nJTxg/fjyLFi3il7/8Jd/97nfZt28fIISiq6++msWLF3P+\n+edzyy23HHD7gbB+/Xouv/xyvvvd79Lf389Xv/pV7rjjDh555BEmTpzIj3/8YwD+9Kc/sXr1ahYv\nXsx9993HXXfdxerVq0Ntff7zn2f69OksXryYn/3sZ3zuc59jcHCQJUuWsHv3bhYtWsQjjzzC1KlT\nWbly5Sv2LUKEQ4mIvCO86XH66aezefNm+vv7KRaLrFy5knnz5tU89oYbbuA///M/KZfLB2zzO9/5\nDldeeSUPPPAA73znO5k/fz533333Ienv2972Nu/v+++/n4suugiAOXPmeNppJU477TTGjRuHJEnM\nmDHDI85ly5bVvNcXXniBUaNGceaZZwLwzne+k127drF3717a2tpYsmQJzz//PPF4nP/6r/+is7Pz\ngH2+8cYb+dKXvgTAhAkT6OjoYPfu3Wzfvp3BwUHOPfdcQJitb7311hG3vxKSyaR3P21tbbzwwgue\nteO0007zxuepp55iwYIFxGIx6uvreeihh0LWlkKhwPLly/nIRz4CwKRJk5gzZw5PPvkkra2tbN26\nlUcffZRiscg111zD2Wef/Yp9ixDhUCLyeUd400NRFC688EIefvhhWltbOeuss1DV2p/GzJkzmTt3\nLnfccQezZ88esc1EIsFVV13FVVddxdDQEIsWLeLrX/8648ePf80TfXNzs/f3Aw88wJ133kk+n8ey\nLEYqVdDQ0OD9rSgKpmkC8Mwzz3gEFcTQ0BBdXV0hC0Q8HmdgYICPfOQjWJbFV77yFXp6erjyyiv5\n1Kc+dcA+r1mzxtO2ZVmmt7cXy7IYHBwM9U1VVVRVHXH7K6Gpqcn72zRNfvjDH/L4449jmib5fJ7J\nkycDMDg4SGNjo3dsOp0OtTM8PIxt21x++eXetkKhwBlnnMFJJ53EjTfeyK9+9Ss+//nPM3/+fG66\n6aZQexEiHG5E5B0hAnDxxRfzve99j5aWlpo+2yCuvfZaLr30UsaPH19z/8DAAC+//LKntTY2NvK+\n972Pp59+mk2bNh0yLa27u5sbb7yRe++9lxkzZrBjxw4WLFhw0OcbhsGaNWtqCiGdnZ1MmTKF3//+\n9zXP/ehHP8pHP/pRtm/fzr/+678yZ86cA17r+uuv58Mf/jDvf//7kSTJG4OWlhYymQyWZSHLMrqu\n093dPeL28ePHI8uyJ3xks9kRr/nQQw/x+OOPc9ddd9Ha2spvf/tbHnjgAe+6g4OD3rF9fX0kk0nv\nd1tbG4qicN9991FXV1fVtrs6IJPJcMMNN3D77bdz7bXXHnAMIkQ4lIjM5hEiALNnz6anp4fNmzdz\n+umnH/DYzs5OrrzyyhHNuKVSiU9/+tM8/fTT3radO3eyatWqEaPKR4KqquRyuZoa9cDAAOl0milT\npmAYBvfccw8A+Xz+oNpevXo106ZNIx6PV13r5JNPpre3l1WrVgHQ1dXF9ddfj23bfPnLX+aZZ54B\nYOLEibS3tyNJ0gH72t/fz6xZs5AkiT/84Q8Ui0UKhQLHHHMMo0eP5pFHHgHgd7/7HV/+8pdH3A7Q\n0dHBhg0bALjvvvuQ5drTWH9/P+PGjaO1tZXBwUEefvhhb2zmz5/Pgw8+iKZpFAoFrrjiCjZt2hQa\n93PPPZff/OY3ABSLRb7whS+wb98+7rvvPn70ox8BwgoyZcqUgxrvCBEOJSLyjhABkCSJCy64gLe+\n9a0jkkEQ//Iv/4Ku6zX3jR07lp/85CdeVPiFF17Itddeyxe+8IVQBPrBYM6cOfT09HD22Wd72qaL\n6dOnc84557BgwQIuu+wy5s+fzymnnOKtPX8lLF26NOTvDl4rFovxwx/+kFtuuYWLLrqIT3ziEyxc\nuBBJkrj88sv53ve+50W4z549m3nz5h2wr1dffTWf+MQneNe73kWhUOCyyy7jS1/6El1dXfzgBz/g\ntttu48ILL+TPf/4zN998M5Ik1dwOwvJx88038+53v5tUKuUt8avEO9/5TjKZDBdccAGf+cxnuOaa\na9i/fz/f/OY3ufjiiznrrLO48MILec973sN73/teTj311ND5N998MytWrGDhwoW85z3vYcKECYwZ\nM4bzzjuPdevWceGFF3LRRRexZcsW/vmf//mgxjxChEMFKarnHSFChAgRIhxdiDTvCBEiRIgQ4ShD\nRN4RIkSIECHCUYaIvCNEiBAhQoSjDBF5R4gQIUKECEcZIvKOECFChAgRjjIcNUlaenuHD2l7LS1p\nBgcLh7TNNyOicXztiMbwtSMaw0ODaBxfOw71GHZ0NNTc/qbVvFVVeb278HeBaBxfO6IxfO2IxvDQ\nIBrH144jNYZvWvKOECFChAgRjlZE5B0hQoQIESIcZYjIO0KECBEiRDjKEJF3hAgRIkSIcJQhIu8I\nESJEiBDhKENE3hEiRIgQIcJRhoi8I0SIECFChKMMEXlHiBAhQoQIRxkOK3lv2rSJ888/n7vuuqtq\n39KlS3nve9/LZZddxo9+9KPD2Y0IESJEiBDh7wqHjbwLhQK33HIL8+bNq7n/a1/7Grfeeit33303\nzzzzDFu2bDlcXYkQIUKECBH+rnDYyDsej/Pzn/+czs7Oqn1dXV00NTUxZswYZFnm3HPPZdmyZYer\nKxEivGmhGxZL1+6jWDZe76542NuXZ822/te7G0cNXtjYy879wyxduw/Lsl/v7rxq9GWKrN8x8Hp3\nA4D9AwVWbekDoKyZPPdyN7Y98tjmSzovbOw54DFHGoetMImqqqhq7eZ7e3tpbW31fre2ttLV1XXA\n9lpa0oc8Z+xICd8j/G2IxvG143CN4d2PbOTXizdw3twc11x+6mG5xt+Kf/nm4wDc/+13oSiHTn/4\ne3wP9/Tm+NEf1ni/48k4F8075rBe83CNo/vcf3XzQpobEoflGn9rX+79+jv4+d0vsmzNPmRV4aK3\nTq55/I9/8SzPv9zNdVecytvnTHjF9o/Eu3jUVBU71JVuOjoaDnmlsjcjonF87TicY7hhu9BwN+wY\neMM9p737syTjh2YK+nt9D7dWaKobt/dz2tS2w3a9IzGOXXsz6K3pw3qNg0V3zzArN/YAsGnnAKcd\n117zuA3Oc3hh/X5mTWw+YJuHegzfUFXFOjs76evr8353d3fXNK9HiBDhtcE180lIr3NPqqEZ1uvd\nhTc8SroZ+m2aR/+YvZFcOJZtY5jiG1EPYAVqrheWgsHh8hHp18HgdSHv8ePHk8vl2L17N4Zh8MQT\nT3DmmWe+Hl2JEOHvGq6LTnrjcTdGRN6viHIFeRtHsc/bRb6kv95d8GBaticQqcrIH0mLY+bP5N44\n5H3YzOZr167lW9/6Fnv27EFVVRYvXsz8+fMZP348F1xwATfffDOf+cxnALj44ouZPLm2ryFChAiv\nHW9E8tYj8n5FaHp4jEzz6CfvQun11byDQWeWZeP+UuSRddn6VAyAzAE072x5iKZE4yHp48HgsJH3\nrFmz+NWvfjXi/rlz53LPPfccrstHiPCGwf6BAo3pOOmk+Nx6MkXSCdWbEGqhe6BAQzpGOukf0z1Y\noLk+QSJWHbiZzZUxLZvWxmRou+Wazd+A7H0kzOYDQyUUWaKp/rUHSFm2TVd3jgmj6pEliZ7BAk11\nCRLx8PMoayZ9QyXGtde9pusVSjq7e3OhbYPDJbJ5jaa6uLetN1MkGVdoSMcrm6BYNtiyJ8u49rqq\ndwOEANWXLTKmrbqvA0Ml4jGF/mzJu+dK2LZNV0+Ose11ntnZtm329OUZ116H5IxTXeBdz5cM9vbl\n6WxJeedYts2W3VniMZljRjfSkynSmI6FYiJ2dQ8zpq2OmFqbZGudUwslzbdmmJZV9bdl2WzenSGV\nUEknVOJxxfuOhgo6fZkidakYqYR/naV7V/B/G+7lIye8n4s7zjng9Q8VjpqAtQgRjkaUNZMbfvYs\njXVxvv+pswD499uWIQG3//v8mufohsXNd6xg9nHtfPQfZgLQny1x48+X8455k7jk7ClV51z7388A\n8D8VbbpKhvw6cLdpmWzL7mBq85SawsOR0Lw/++OlQPW4vBosfm4X9z6xlcvPO45Tj2vn33/6LBOm\nZ7A7N3LdnH+jOdEEwDf/70V2dg/z7Y/No7059aqvd/MdK+jLlkLbNuzKcO2tS0L3c8Mf7sYup/nF\nxy6vauOexzezZNdKmuoVvnvlZVX7n1i5h3v+spmvXHU64zvqve2mZXljB3DlBcdz3pzxVeev2TbA\n9+9dxZknjuaqd5wAwOMv7uH/Ht3E+88/jtOmdfLvP32W9iZfcFi5uZdfLd7I208dxwcvnAbAqi19\n3HqfiKq//v2z+c7dK5k+sZnPXSFWSGzcNci3fr2SudM7+fgls6r6kc2V+ffbljF1XBM3fHBOjdH0\nEdT8g0vvXFJ/YVMvP7l/rbddSuZomLUSuXkaVqaTz922jHHtddzy/73FO+avu5cAsKJ7JRefeGTI\nO0qPGiHCYYRmiAlhKK8BYDj+tQMZP4tlg7Juhvxre/pymJb9N/vcfBPhkWfvezf/ie+v/CkrulfW\n3K8bZs3tbyT8cevDfGHJLWimxsrNIsh21ZY+9vYXIFair/FZ+kuDdA3v8c7Z2S0ijQcOMrhJt2qb\nkSuJuxZ2De0hPmkDieNfrLm/qzdH4riXKI15ofY1MkVsoKsnrOFXmrbXjrAuf9veLADPrNnvbXtx\nUy8AK17uYSivIaWHyE+7H7mpx2lLRG4/8aI/Zv2Be3VzAGzYlfHb3LsRKV5gxYaemv3oHiwCsGVP\ntuZ+0zIxLfG+BX3uZoC8NSe+oC9bDJ0rN/Wjy3kxxoo4d09fPnRM2RDPOqkcuSVwEXlHiHAYURlf\ndDDapghSssnGtzGsiUm1NyMmt6DPc1XvWnoLB0524pL366F5P71HJF7al+/2tlkBf+PR4PN+ZOcT\nDGnD9BbD45zJlVEa/WVcRaOaaJUDBEC52DW8m2v+egNL9jxb+wDJIjZlNXJzd2iz+1yf6FpywPZ7\nC/6qHsuuHm83IK43EyasSvIeyVRdy6Li3rdhWpiWTWzsVtHGpA0j9jN4vf4KoaW/OMCSwu+Jz3hu\nxPMrz6nEf734E7723HeBcLR7Tisi1WWIH/8Cw8ZwVV8AJFXz/pbragsHZVMck1CqXReHCxF5R4hw\nGFG5tEc/iKU+Zd1Ead9LpvU5/mfdrwF/cnU1hf35Hn625k7+47n/8jQGoCoDl6d315hkNw5sYU3f\n+oO+l1eLxri/TjVI2G/0pWLuhAygW+EI6d5MESnuE0ZBD5MfgHIQEtOjO/8KwIPbH625X2ndj9q+\nl8TxYeuFu7xpW3YnALZR7QEtaQZFxSfvWn0cibzzAQKT6rIMpNbXJH+5xj2qTuCXYdqifcVpq0Yf\na12vp6Ivq3qFCVtOjEzQwf6XtDD5dg3vYcfQLnoKfWim7l9L0fn+y98mOfNZlOZetiUfreoLgBQr\nB/7WqIWyeeSj0CPyjvC64I2UZvBwwqwgU10/OPKWG4RWtze3D/AnJ3epUG9RTMq6pYcmm0rh4EBW\n8x++9DNuW/2/r9ifkfBiz2q+8dz3KRrVpBCc6IPEFyTv16p5W7bFrSt/7hFg9X4bpW0vcsv+mvtf\nCbuGdnt/l4zw5NybKSLFAuRtVCeRMi2bIW2Y32/+MzktX7U/eI2JDeOq+g4gN4nnbNvhB1jWTWzb\nJlN2TMuKUUWufZkScr2vKeZr9NGNZu/LhImx4JiWpbosyZnL2Bd/gd3De6vOryWfuEuuTMuirJtI\nqmjLNqsDNF3BsxAwZe9zTNKphAgEfMkhb9saWRjakFvtPefKe1m+z3cZ5PScZzavJGJNyZLT86G+\nACEhDTV8zu83/5lfrL0LzXnHa1lgDhci8n6TwPUvWrZdMYGaNY97pW2vBat61/LJJz7PtuyOmvst\n2+Kl3rUU9dIBr23VEAAOVV9/uf433LT0m6/6fLcfQfI2TCtEriMJMJpmIsUFIbYmW4Cg2VycP1jy\n/YHByaaSED2zeeC3bdvkdJ9MKv3ouiGIwbKtmtqWi9vX3sXu3F5W9673zgHxXILm/JAGG+hfppzh\nzvX30F3oxbQsBofL2LZd9QxFX6rHqrfYz4bBzdy/9aGq4yzbxjBMYsesIzaxtrm2ss18SQ+ZTHcN\n++RdOSn3ZkpIcX/cCjUEGNO0+c7z/81fup7imb3Lvevphskftz7MZ578Mn0lIaTJkh+xXiwb7O4f\nRG7sQ2l0yLscDnzTdJMhLYdhi/5KEhR1v49lzWT7/iGkhE/Y+RoCxLDdR+yYtfQMD4W254o6iZlL\nSc70a04MlAZDxzyzdzm7zZeRm3tInb6IPbl92LbtRZAXk7tZMfCMr3lXQB2zleuXfImCXgwJoK5F\nJp2IYds2e/Ou8CXh2pJ0w2JgqESuqDNYyrIz/gyJ414C/JgDwzK4e+PveaFntX9fWt5/xnJ1v7Kl\noWqzeby25m1ZNn/peoqVgfaPJHlH0eZvAnQPFvjCT5/l4jMm8fLOQbbvG+J//n0+Dy7bwX1PbuPm\nf57LxFEN/PWlPdy5aCPXv382MyYJ0vjNXzbzyIouvvWxeXS8hsjZIO7fIibbv3Y9w5SmY6r2L937\nHHdv/D2N+jF0r5zOj649J7QsA2BTV4Zv/t+LfPySWcydLrLzPfp8F3c/tpkbPjgHPdVNS6KZ0XWv\nLnPfc/tFAJBhGajy3/aZvLxzkO/cvZIPLZzGceP9VIr5khEycRumTUyt1iZKuomUEGSwfVeZNW39\nXhCNaxbvCfgyQ5p3gBwf2LaY4eQQ0IYkSZR1k49/90nOOGEU557lB9Zc96On+MAFM5h/6niG8hrX\n3LqEc04eiz3xRV7u38R/nHUjsQOMgaZb/P//+SRnnTiGf3nHDD73k6VkpN0kRCBxyKToE7PFnwZv\nB6A50cSG5Z1s2JUhEVcoayafuewUZk5u5cWe1WzdrPDw091V0dvd+XDw0n/d8xLb9g3xb+85ke/+\n5iWuuGgikmKCbGFaJorsE+SqLX384Herue59JzNrShvb9w1xyy+fB+CbH5tHZ3OKVV27vON/8dBq\nxqnT/WsnXkJp9f3QtUzSmfKgR3j3P72de34Dn79iNt/69UpSpz8ROvalbfvZ2TlMc32cz922DOnY\n5SSm+wKQVKHxLVu3n98/v5LkTH9btpynLp6mpBlc/+Ol5EsGiVk+mQxr1Zp3X/ol1NR+8oqBbvhR\n0kPlAnJdmNAHy2F/76833AdAfLLQqH+y5EFSPafQ0SKeUXncc7yUAzlZ+x5iEzZj2LBpcAu5cnXf\nDNPi9kWrKDrmckm2QBZC4pduX06PE6TWcEwXeJ+5ze0PvsykUQ3stzZXxRIM6znyJdFfqYZQ8Z3f\nPUeb7FtB5MZ+5PQw2BJIdugeilr1+bWEuMOFSPN+E2CjE7X50LM72b5PfJCWbXPfk9sAPzr0waXC\nf7Z07T7v3EdWiIIxm7p8Te+1wtXmRlp77EbuZhFmuv6hamn2ryvFMfc+4ZeSve+vIjBm+cbd/PdL\nv+CW5f/5mvsa1BoPFktWi34/tGxnyOddKOkhzbsye5aLkmYguf492eLhZ3d6y1hcTb7HMZvHlXhI\nU3DJ0bAMFu34CwPNKwDxvIediPdn13fTlfMjfVEML3p2l6O1PLVqN893v0TeKJAp1Q7ScbEvI/Yv\nWbMPy7YZGCqHTI1lI0jezrNP+pO1bulelHDZuc+la/ezYWAzt6+9i8cHBUm8vCus+e2vIO91OwYp\nlk1+/dgG1NHbWbRGmFslSZivg1i0XBDzn5buAMS6eq/d/gLL973Attxmb5tml0NLBOzOTeJ/UwgE\ntczmPaVe729TEtaRxc+NUIBJ0dm0O0NXTw7dsFCawgFykmqA5L87f1ixKqQVA2RLeedehCY7arRN\nLOW/vzm9uo+u8UFp2093xo84HypWa+lBzTtoNZJi4t4yWYOd3TnvGVYimbb454unM2/mKN4+2yfI\nn6/9FXvG/L7KvVHWTZZt3hHaJsU0hgs6PYNFL9+BlvYtJA2NYoy27xtClqvzIeS0PANDzvtYg7yF\nZu5bshLTxfcjmXHn+v67nCtWZ4orRWbzCIcSlZGiUrzAQCHrmbfcCdUNsKn000LtwJRXCzenkSzV\nfv08U63j56t15ZST8CS47MM1t9nqa5N+g6biVxOIogdyJQfHslAyQj5vbQTyHtZySJK7QNsMCS8e\neRd8Yqg1BtlymKzcyF8Xe3P+RCkpJprmulWcyzb4wlqmXE3ewTEKEpebgSpE3gEBSDMsUDXkQKT2\nYDHnBWC5SMQVT4iT04JUKpPT7A1EsQ+X/D70JlcTm7iR4lh/nfJAKSx8uglz+lIr+dnqX4aEqoHC\nEHe+fA92zH+PJMW3mtiYge2Oz7aW5q0NBI4TRDE40lI/xaA3U/RiG1yhIISA1hcb5QsBVkEEBA47\nhNubKSI39DM0cTGmFCDvcjUhm5Lfn64B35ozXK6+n6DmXWt5m6aJ96w0AnnbisbZJ43lX981k7NO\nGlO1X2kJC2NlKUd8imOSdueCWJk9TuKaGcc0ITf1hPz6l54nhILebBHd9L8LN2ZgWM95Yyyp1fcg\nqToZR8gNCktSsQXbkpDrhjyXVqYQHs/GeEOkeUc4tIhVJNxPnvIUNy3/ukfq7oTvEnStmsG1siu9\nWnjBOCO8fqZDDF6QTsW1C3qRVervUNp3UyxXTxSGUjs46GAR/AC1V6F5uzm7K8k7XzIOSvMeChCv\npBj0Z/0J1jQtbNv2JlLN1MgXq33Kg+UwWelGONZhMKhNK4bXF89H3uATT7YGebtL2AAKZoA43Ykx\noKGEzeYWyROXED/Gj3LPFKsrMCVisqfpuYFKlQUt9hd88t7W7U/87uQaxEDRHw/TMulveB65foBy\n0xZW9a1DM/yJfqhUYwJWDIYdTasQ8C3bloRqJ8g774zhPV+bQS2gPTvrg0dapy8pOn2ZkhfbYBuB\n4C6XuFS/j5Ll77cKIrmKaxbvzRRDhKZaooJXvkLz3jS4FSvhH7c34z/zXMDEbmbbkGyZTDDOooal\nwRVkhgvV38yY5Hh0S/e+p2DSFu/8eFhrjU3Y4AluFB33k6qxs1tsax6TJTFNuLdkW4yHkhTj25sp\nUTQDz0lLOPeV9yPTlWrNWVI133IQ0MylvTPBjCHFyyROehqAgYL/DZzVeiGtyRaKRumIBeNG5P1m\nQ0CajKsVmrdyZDXvkczmXiCRM2lVLrfaNbybItmQ9haEJudqbg9ix9CuUDRxELkAMb0as7k7gcdU\nudpsHiDQkTSUIT3Qf8UMBVaZlk1eL2AENJ+hgJbktl+pLZtWOFguUwoICLJBWQ8njwlOpBkt7PuE\nsHBQNH1hyU0sEgzyqQxYq4zyHa6hESZiCrudSHsMYbKsDCQKmnF39PmWCKxqrTWoea8f2EivuoHE\nCc+BLO47GImdr0HekmJ4VoVcYLy1DXNRiFN0iNHVzmPHrmJDYZV/vqPlDeVqvE+2BIpBT7ZAr5sg\nJBCZHdNF/Elw3EzE3+ZQC9aQKBE6rPmad5D8k5YgviB59xT6+MHKn3r3D9CdC0SmOwKKOdSKtuUU\nVCsVGsOagVmOcDGU10LzjLZsY2voAAAgAElEQVRtFi0J0Qc3ULJWamC5gryDEfbGUKM3Bjv2i/dR\nSfnHj5VEPIIuFVBkib5MMWzCdsZzqJwj4zyDWj5vggKSs9/oHYdeSHrjLzlj1p0V42V0T6TdmEZK\nTWLaZkjjP5yIyPvvEAOlQTQzaEoNkETg5Yx55C32u2ZzzSpXSemHMsmHa3IdSZu3bLe/Yn/l8ifX\nZOx+XJU+tqLtE9NI0dI/X/Mrfvly7dz6w4Go3FdjNvfIW5FCVox8yXCehY0yagd7ctVLbwAKhk/e\nUkVErGnZVcQ8FCC/2uQttO6gmX446ANWzCrNO6g51zKbBzX3vOFf39e8S9imgm0qlMxqn3cQtZaa\nxWOyPz6KDtgVS+L00Du6dzAgyFnV01qwv7XKoxYD1oOc5vfX1WpRDE+wcTXvuvxUrFwrshX3rDWu\nEKS2+W4J28bT8mwIERtA0m5Ckm36snl6B4uOUO2/N0nTIe9gwJfzHWtbT8Z2hJu8JvrQmy2FiClh\nC7N60KKU06sF3IGCP0Yll7wHRoMZQzHTDGnDXpayWm4C1zIwVNA9rdUcGIXZN576mMidviWzXRxb\n49sXAl8wsMA/xsoJ8pcSBU/zjiXEOOp7jmVS8ngAslqWtqYkvZli2IK2XUT2DRQDgmhgjLRts5x7\nEGOcSqj+flMV30dIKLTodSL0bSNGb6ZIWhWBevkaY3M4EEWbH4XY1T3ML/68nk9eeiKdLeGi9nm9\nwJeWfoPx9WP5wunX8H+PbOIvL/oaZnACiFVq3g5Db2m+l889bdO89VLv2B/9YS3nzxnPFRccf1B9\nfOCZ7by8c5Dr3z/b+1DveXwz2ZxGLqWBAiDxg3tXsdkpSPCpfzyJyWMasQhr3pWlI7tdf6/zcXUP\nFkKBa3nTn4SeXtPFI8v3ceOHTvMi1otGkUw5S4MzET350h4ef3EPLQ0Jpk9soXNyteb98PKdrHi5\nhxs/dNorWiFcYUOpMJs//uJu9vUXkFI54pM2cHfXBo4ZfQ0dHdNC5xfMvC9WKz7hqopMMbGXb6y4\nN3R8vkLzfu7lbh7ZvAncVNWySV+2xH/+5iVQyySmr6Bo+WSlxk1PAHIzuAXJ+4k1W2nMdHHh3An8\n/qmtDA6VGTfL13SDxPenZ3aI8+MlbC2JpOrsGxxC003iMSUsSOL4CZ3+j++oZ3dvDqV1L08Ul1G2\nXQ1JRBkPFzRu+eXzDAyVuPyiseLWJBnLtugeGgScSHTJH3NbjyPFNJ7esI1ZiX5mTWmrGVQUFCCW\nb9hLYgZItoK2+VSSJz8VIkPN0kgACScVpmzF0S2DsqlVuUJsSwJLDWt5FRpfPpNAaRVBcXv6ZEa1\npukPHJM0WxlmK/Gpqyi91Iytpfz2jJj4B+zoHeCWX66gN1MiMcrC/WpiOLWoy4M8u24/v35sM23j\nstDqdlJEUvfkMlzzvb/S2ZRkz2CWeDNgim9GNtPY2GTKQ7SlWnjmZT8S37+voNbqru0W53em2wG4\nc/09nNwxq3a8i2yKNtzgMMdaUVpzJraewDZU1NE76Fk3FmjwrmFmOmlKNEFJBLt2NI9l3fYBVm7d\nBzEorT4Lu1SHjEJ/UVhrmuvj5J0xHNf/DrYMlGHKWi/4rqkuTlmvuIf++fR0OMl0VIP+vAZpMUZ9\nmRId44UrIK8XSODniT9ciDTvoxD//fs17O7Nc/+S7VX7smUhDe52tJYgcYP/UUE1eXuk5Ex++/rD\n2vdjL4i2Hti6iJuWfjNkuq3EH57ezoZdmdBktvi5Lp5d3+1V7zEsg1Vb+ymUDTI5jXXbhfZkOaTq\nkXeFGd9dJuVOYNv2DbFuh29CzZm+VnnnY2vZ11/w2ga8NchlS5DDLxdtpKsnx+qt/fz2iS0hs7mr\nzdz7xFZ27B9mYFhM/I/tepIvLf1GTbO6YYj+xlQ51Hd3PIPEuLXGWveiJTRZ24gJE51kkUooJOMK\nQx3LveMkXZBVIUBGmmFy2x/XMRQ0dct+pLrascf3IzqIxy3vOXmacUwTpk5bwlSKvLBR+JT/vHQn\nz6zd7yWPUWWVklXh/5QNpJiOrSWxTRXd0ti0W5hcS3p4vNpTbRhogM2oVnE/8amrKdhhbV+KldnX\nX2D7viGyeY2N+4VmO8FJbtJfEM9/zrQOkulAycfhZmxLwlIK/OB3Ivgp6Au1ikIjdCO1g+M1Sj8Z\nu5wS5tsa5OvmsVY1oRWu69/gCUFuxrPyunnYhhr2V1eQt2vilhQD07LpaEqScuSQs8fOo8mY6B3b\n0C76Kak6tiWDrXiad1e+i+37hsgVdU8rbUk0M8Y6EXO4ha58F89u3k6uqLN70DeB1xtCECrbBbbu\nzrJsXbfXx6ljBOm675rrLtm63/fne/0P3CMO8br7zh53BhMbxmNjM1QerhKgZEMoIak5j3uBeZKq\nYVsSdrEejDj67uORZBu5LksqoVK2xHc0a2In582ayrFNk9kwuJlp08XzH8yL91wkh5EYk5jAgN6L\nlMgzujXtPceGRJpPvWc2tiV5yk1bU9IXnB3yTlvtTFBO8PqWLfrfaaGkM7vzJE5sn0FnXTtHAhF5\nH4UYcgJC6pPVfiP7gCUvCJnN3UxIru9VqTRlSbVNzot2Pk5faaAqorkWatbudYSDsiHuo9NZu+ul\nAK2INh9Z8xb34i6BclG2AxOxc7/BW3Ozk2mmVm1Wl02yAZPycCk8ybhc/IctDzJQGqyZdcowLZAN\nCsmuqr7Hj3+exPTnvd+1AuLKtiBDu5T2+pSIKSiyhGwEAn1KQrovB8hINyykZA65MbBGOKC9h9Jo\n6qItJRYgb9MCbKRYmeZEI7aeQIqXqtJn7sntJ6HEmdQwAc0uhd6Vjsni2tZwC5gKyCb5ongPKrN8\n1cXS4n1QjFCZSxcJSbwbUkwjmw8kRTHFxDyrbTqqpNDPDuIxiX+7ZBbHT/K1HqtUJywA8ZJnBXH9\ntbGhiRyfOA2A/ny1sCNZCiCBWaE5O/uTagJFlogPC3J9dt/zvrAqmyT0Nuxio/C31iB/M9sqtErX\nv+1sb29OolllpjQdw+XT30NCrqO8+RQAzjhFVC5D1T2N2y6lMTPtKI0DyM3i25Ad8rzm1I+RUJKY\nvULI2Ws6Firn2zH6RzPJnOe0GXgXnb5ceubxpBMqaOJdcZMDlZx3Ttt6EqUX345VrAuR97hRTh4B\nh/iS8RjHNYtqeHkjXxWVPaV9lD+8DQNCaFUMMGOeddF2+iCpGsm44rXxrxefQjKhcsGkc0UDDb1M\n7KzHkp3+OO/8xLiwcClt+xjVmvb6m5QTzD6uQ5i9nW1j2tJV1gNVkZkxfpTTB92L1bDNGLppM731\nOD520j8TV0Yu9XsoEZH3UQg3pWFDuka6wVcIsAp+YO5yD9eXqCgyECAb+cDZygyrOjBDMzVRLEEO\ntx3KmuWQd8kh77FO3WOXICqXihkBn7duGV6gkrf8JlS9yaZsB5f4iD4G/es9gexfwSUvUqJA6rRH\nWbTjL962XLmCvKtyh9fI8mZaxI9byZ66p9haeDm0T2nuC/2u5VPXKGDbYDlZtSTFEOStSEiaX3fZ\ndLRGzfKfeX+5j8TMZUiq4S83CvrNAwFKdllMikrM94drugmqjiTbNMQbsEsppHiJTD6Q7U6y6Cn2\nMrZuNC1JQSZBa4LWuB3bkjB6JmBbigjGGhSknXeimG1TYczwOYK8Ee9lMmVWvXMtsrOkKFYmGxDS\nipYg77H1Yzix/QSM2BAtHWUkSfKIxb1HW0tCrOwJGG4msobisXTUif5nQwF8jvbs+DhtUw2Rr/ve\nJeQE8ZiCVaynM93OjuwuQd6ShSTbWIZ/vhCgrND5Vq4Fu9jgkYvSIN7rlqYYNjZJ1dHsFckjLtcq\nIyl6IChNwth/DBAonOFcI6UmUWQJa0jYyPP0e+MNYPaOp0FtAFsKPUM1bnrnx2Iylkve5QyWbVGS\nRTu2HgdkQdJObAKAGjP8sUMQn/usl+xZ7uVkd3FSu59tJnHcS6RmPYuk6khmjNaGROBaQEwnHlMo\nODEPKUX0rTMlNN5sOYuqytiyLtwWtqC5MbFjkWwFdew28vWbkRQD25KIqWIcG2PNSIkCclOvqG8e\n8Hm7z8H1a8tNfdhjnRUThloVVHskEJH3UYxaUeFBM26tJQth8hbHFsoOwcmSZ+6CEaIxAyjVIJ4l\ne5fzu81/Iu6UKHQTHlQm+we8oLr6VIzm+rgXqeznwq4OWOst9PmEqRiA7ZVelOoyJE56GjsogDj3\nIAX81K7mDWHylOurE9G4ZnMXlZp0Lf+pYfpJNoaMAye3qWV216WiiLB2J+eA5m1LgQxteYe8bf8e\n9pZ3ICkmetfxGN2TgLDmHXz+linGRFZNz/pSMjSSJz0FQL1aj1VOIUli+ZUXSZ7MY9kWY+pGe+lb\ng9HphprHLtWBkRBaqwQ9Q4Jsi6azpGr/MSQK47wJXU6UeEL/XxLTnwtZB1plx7edyntCK0DRsa40\nJxqZ2ijiMFIteeceAuRddDRvyRcw3ICidCzFqAZB3iUr8Jwd8rZ0550xYk6kcfC9EwlyknGFsm7S\nnmwjbxTIlYre+YYhuwMi/ne/rQpSsDVBCLGJGyFWoqlRXNclJUWWQBcEVrByoh+q7hEj+OlTvWVy\nAdO+KsvYWgoZGSvmm91BmHx1A5JyGjk9jNK2l8TMZ5AdQSKlJokpMmbJ1byz/HHrwxjNwuftWg2E\ni8dGaXfW5scCPnkH7rNetm8Fd738W4KYN2Yu7516iX8/ySFQdFRJCEiiLdcXrpGIyRSMIkkl4WXO\na3LqqWfKQyKHhaO5e5kiTJVYYTSSbLFOe1ospzNVYoo4/8KxFyFJoI7ewdi2tDf/ueMcU2XqnMC7\n2Litfl/N2EEVHDrUiMj7KEN/IUPylCeQW/bXXCccJCOj1gtVg7xdYrVtO+QTr6V5u/5qqC7WAHjL\nJJTGAZANr22fvG1PA3LN5om4Qkdziv6hEoZp+ZHyjoYeLIPZEyBeSRZtuZp3/NhVyE7mLjcgxtMw\nAm0EyTtoqQhOhi4KevgeNcMK+fprJWUIErxsB9s8sLAFIg7AUHJYpTS25ZyrGCTiCrIsYztadHnT\nqZ42plt+H12t08o3CpO1c76LEHk7yT0kxcS0bAzTIqsNeoFCti152rmUKLK/wmffnGyixZkw3XSu\nSBaWpHt+WDdCtzebc8bL0byNGLph0eFoS7HxIpuZXJ8N9bfDnoptg9wUWAoGDNvid0uiBVsT10qk\nxHlFowSWjLZtltBuXXOrI2C4qTjr4knGNAvhQx29E6VjlzceAGXNyXtQSiPJlne+uz+pCGIp6xYt\nSeH3HigP+uTtnO8SnEsGleZYs38Mck6YY+Vkgfp6cZ6vecvYegJsWJ9ZizJqJ5IEiuW7GWwthW0H\nnoOsE1fiKLLiLAGVSMuNSIkCx09o8oPLzBjDeY3ZTWcgqQbxY1cj1w1jJ7NOH5IidqMk+rJjaCeP\n7XrSfxCGK4CIMY5PWRsao+A35Uac10JKTXJK58zQNkm2ScpJLzmPa2lQO/YwNGoJBb1ISvXT5SbV\nBEklSaacJaZIQrM2VRJxcb5hWpT2jQ9dwzZjnvvw2JaJIj4hVqalIeG9h+51Fdm3HoTa0OO159rD\njIi8jzI8vnOZSBRw3Es10xAGyeBHf1hbtT84eWu2+LusmTy8fKfIchUMOlGq28+X/PaD5kkXwQpS\nUsqv4OOlHJRsQbrgVeJJxAR52zbc9sd1dGcdE6ZD8oWSwc/+tI7t+4ZYvlVIvHaAmAbcDGSBicI1\nobkfYFk3uOuRDdz57OPszPjpX3/1WKAkZg0ff7DYA8AdD7/MMxt2+PsDWt7zG3q4/+ltXoY1gBUv\nB7JG1bBkDBULWJbN/zz0Mt/9zUrW7u0CyRZBOs49SorQvFVZwpZ1UnIdVqbTm1SCfmQ3iMc2Yz75\nB4Uwl0D2noDpaeZim6abFAMa6JT6qb5Glyiw28ls5b5DxbzM48sdM6yreTt+U9fE6fZxy0AXS9fu\n8zVcI0ZXT47lS2JYxXqkej/gUJIgZTfz2TmfADOOlWsRVhGnbbmhnyFpPzNaj6cp0YBWEtey4jl+\nt/lPDJYzKHoDZt94QAqR992PbWbzfiG8NSTqGN/a4l03Ptl5FxzXgusxsYvChy4lnVgKR4BKqkkS\nMZmybnoWiKyWDWhsYdLxfMoBUhjVmgYkFIe8pXiJtMNHSVX0W5Yc06/zPONOXeykEqg1YMvCwuCQ\ntyXrYc0dyA6oSDGd3WN+i5zyg62GCjpnj52HbVbTQVJJEFcV8jlQibNreE9ovyuY6Hum+hslv5JY\ncL16LeKb0Xo8lxx7MZIk0ZRoqNqfjqX9zHqm6iXsKSf3M1jOkI6F6y00JxrZm9/PQMOLYi4zVS+W\n4p7Ht1AeaGFy9p3e8dZwi5dpsi6pYhtxJFVHVWTf8uhp3lLoHqxcE+WNp4KexDBtNnVl+PkD6w95\nIaeREJH3UQYvwtGSR9C8fXJdvbWvan9wOYcZINpHV3RhmFY4KrZm1R0/GKyW5h3URCXVr9Lk19AN\nrNX1yFtm1mThk3txU6+fwcohnadW7eXZ9d3c8svneXHnDsDRLBHLSTzLgeFrIp3pDq8PAC9s7OXJ\nrmdZXlgUShm5bmcgM1egb66/2fXLu9jTm+dXj/vJN4JLjH58/1r+9MyOUCYwSxJ/nzdnfDga10Hf\ncJ6+TJElq/exbscgK7YJ4cQq1vtai2z4ZnPZICY596m7+Zb9PnqWF1MNkH+15l3YO9Zr31svr1ue\nyVnfNY2J6SkhzXt3r/PsHRLq2qvRtdsJ7nK1UvcenWdh9ApNRx23lb+8sMeL5G9K1lPWTdZvz2L2\nj64al3pzFJObJqEbFtZwC5IEckoIdW5ynrdPOFvcS05MY7uNjSLeApDxScMn7yKPPt/FcKmIbcP0\n8e00pfzJ2DYVLjhtgvfeubGK9YrQqpV0nqb6uDdeKVVohZpmehaIgVLGF5Zcn7kTeCincsSOXYXa\nIVZttNbVMXl0g9NH8b6NHS15edBd8vVQIfx1NjaGfttaCjlRIjZlFSYaKZf8HfK2HdO7jS0sHDbU\nxVNccf5xjG2vJ2ZWL29SZMVZlSJR6vOjqMubT0HvOs57zu3pZibEnWWkqk5RFXPP5I4OLxVqXQ3N\n+x+mLOSCSW8T/ayxfGx0U6OnOYPvv3aRVivJ2zGdpzYiyRa2odJQkRBmzqQp3t9mpsPLQJlMqCTk\nJHJcFwKPa4Gq4bcHMHomYGU7aUzHMEyLp1ftZdm6/fRnj0x+84i8jzJ4ZGHEvIQQQYQCoCo0ye99\n8kzGjvJfZMM2mDymkUmjGiiUDQzDqliPWi0cZEv+MqNaPu9g6khJMTzy9uoDy7XIW+GMmaOZM80h\nXOe6biajYPUeOT0szLmOyTc4oU0d3eH9PaqCvLsHi6GsX36DZu2/HXPgLv1lnt+/MnxOILCndi7j\ngHnc6d+0Cc186rJpVUdqZjkkhA3oTgnIYr0n8UuqCNBRFAkUHQXXzxj0AYoJzrUE2IbqTToN9YHP\nXNHF0idLEe3bEpYsyLikGb7mbsQo66YnxMjJgle8xBUWTC0WIka3L+75AP/90X9gUuMElPpBhu0e\n9sbEWH78nbO5/v2zAbDyTVXjIluOS8AwPdLxVg441291TNXZrF1V79pSfWuEbz1wBQwDhRjzZo5B\nkiTU3aeKZUKKiT12LRPGiOuVyqKm9LX/cBYA8+c1M3/2uEAwWIJETMEGGmMueQ/6AW+u5u1o7kpr\nN2rbPuQ6IYTc8E9v83IPWI5PefrxKe+7cjVvL6eMHo7GnzVugvf3tz8+jwmdghzV9n0YUtkjb1fz\ndp+Vi3Qsxa1Xn8Ox45qIqTIzx4r2bEOl05jBW8ecDvhLSs0+EX9QrzZgDY5G6fdzPnz742+lNS2+\nydiY7QzYuzm+ZSpffN+5/MvFM4BqzbshVs/ExrAZ28qFBZJxLS2hnPZSxZyUrqHNB2FrqVBFwpnH\ntPD2U8czPjlZXG+oDdW5P1mSOG5MBzYW31/zQz/4L0DeDTFfwDH7x5KMKzTVJzBMS9R4l6Ct6dBU\nX3wlROR9lMElC3dyrUTIh1rxosdUORSJbdg6MVUmnVTRdKeggHJgzXsoQN7lGpp3KFtWgLx9zdvv\nk0fejmTtfaTudR3ydgPzpGQOuW4IK9vmE1egv2ogAdKYOmfpiUMm/dlSyKzuJVEIEHZQcLED2ZTu\nWH936B6DwVmVZnWlfTdKu798zG1TUYTJOwjbktCscMrUYVMEuNmlOo+0pFiZZFxBloVA4+ZxxlKw\nLRkppnnpJstWwIXg3INhB56pE8ErGEEiIdWhSeKZarqFZrmFMWLifdATyCjIyYDP2yFRvaSCGcM2\nFc9c6xKrazaPqTJtyRaQID9KVGhS+o9lcvNEjxRcK0pojB0TsW5YHmkpDYNI8aInILgTaX8mXPEL\nwFQC5K255F1EqsuKwCzbJ8J0cRJG9zEAPLN/GT3SJm98VUVmVLodWZJZ2buGPnmrFxOQiiW9dzet\nOMVB9Kz/jjvjbzlL+jwyQBBZc6LJS0lslcWzzpQy3mqKSh+xvP2tnNJxovd7Zsdx3t8xVWFWy6zQ\n8UmPvMU4G/smc0LsLG99eqXw3ZRwnoNkc4w1jytnvFcc5xatGWrj7JaFvHvM+wHoqCCphrgjPIze\nSVxOcOnUd4b2B8n7golv4wunX0MlyhtOp7R2nve7M91BIjYyTbkCigvTDs95drE+RN5u8NtFoy6l\n+OLbwYyhBoJZ61RxDz3FXuRkwUmyI85RFZn6eB0fPuFyJmbeAbZMQzqGqsjohk1vtkRrQ7KqENTh\nQkTebzDolsEL3atGXPJVMv3JtbbPO1A4voJ8Y6rirSEGQLZEBKVTYSlb0MKm3Rqad1+gwENNzTto\nNlcM8k4ke9Eh76CJ17CdJTfOB5WIK0h1Wc8n7loO3OVZSpvwVZt943yTslJtogY/o5N7P2U9LJgk\nqHP6GPQHB9ZDl/2JqXISDd5DZWrP+JS1xKesCbTpkLcsoVNhTrNUdFsLZR3TLFdzjnvFFKR4mURM\n8QOeLH+JkK3HQfXJW7c1Z3mM4pnNTcIJQkLEJTWiSQWQRIpUL3LdUB3BSyItNSKlh4jPWYSUzHkC\nUbkozKlWoQEplUNp24OccM+Pe/fdFHdIIZHHzHQwpnwasiT7BXMMv7b4jBZhnVBNJ5LesDxBQB29\nk+QpTwrLhy15/s7eTMl7Z+aOOhWAMaXT/HE2Y9iGitLc65fRDFilEjEFK+dr/xa+5qwqMnElzsJj\nzmNYy/F88REUZy11OuYHU6WoR5UUitKQJxDalkIqoYAR9zK9uVAlX5sDMHSFhBJnsJxlq5NCdErT\nJIKQyg28a8qF3u+JjX5lrrgqc+boed56cPDJ0hUQsFSmpWbzDqeNyY1+8hcARXIEViksCfnr6yVa\n9anETfE8O5rDxOmSN8C5o89hQsPY0H41UBP+H45d6AsLQVgqdqGJy6dcwTWzP8bcUbOrqskFMXfU\n7NDv9x1/ifftg1jn71aQA0g6wlZSjXvvnRog2/p4WJMXFj4xfm5g2+mjT0XVxfuSTsaIKRKGKQJn\nK8fkcCIi7zcYFu94nP9Z93/cv/Xhmvu9IDFbqql5B3OaV5KvLNuUrTC5xlWZdDIG2BhNO/xoVaiK\nNpcb+1jc/cfqvgQQIrMKn7c6ertXHxfAQiz18j5OtRSuUSyHydsNGDKHW3yTskNoqiJj4pN3a7KV\nhJIIpYN1NSZzqIVOyzFhBwQc19xp5ZrQd87wtlea+4Jthgs01Fiap7h542VPsDGHm/nQ8R/ANhVM\nWw/lHNesssiFbckhzTsekz1BwF0/DIARR1I16p01/yaaFyTkBqwFyRtVJyb5ZFkvi0lIbhhk7eAa\nj7xtM0beqaLVoDoR5bKN2tnlkVCx4JiF841IEsSPXYM6QQRTeZYRSQpN0ma2jQ4nKU88oFG17lnI\nl97yWT4y/QOUN84hVRJJRXTDCsUyiPHQUOwEsiRjWlaoZOqxzZP477d/i3GcGDonKIyBsxzPQTyu\nYA2OQt80h45Um3+Q5QsYFx9zPh+Y8T5/V66JZEz1a0obNu2pdqz4MI0Nzn2ZKumEeBZuJjcXHzxB\ntOUSgmHatCRb6C8OsGlwK82JJi8ILvhadaTamdQwgYXHnIcs++MXU2WSCVUkxnFwaufJ4hoBzTKm\nysyfcDafnfMJPjDjn0J9mtYqgs7M3rApOzPsv++9mZIXhNpWURmsMemblMc3d3IgjFQO2MVJ7TM4\nrmUKkiQRj/vvu7Z9JlY5yWzlHXz5LZ9leutxofPG1o/m+jmf8n7bxbqQ5u0+r2CKYzVQddHVvF2Y\nw63e30GN2nUDphNqiPzbm4+MyRwi8j6ieOz5Lrp6wqkphwoaDyz1g5y2O8kLdgyJZSuPrOjinsc3\ne8uhvCQNstCUlq3bz12PbOS3T2xhqKCFfd4V5Js3CuGkIgHNW27qJT55HWpnIA96KEDGJhYo4wjV\nAWuWbYfK5EmKEYo2V8eJ7E5mthUz60ySkuV9nHvl1aH2RE1rvw61p7kYcZ/YHD92XVL1lr5JG9/G\njq4yacXPmAR45KdvO9ExHVdq3k7U9daTwIxTeukcZDPBkBZ+ZkENasPgZm596o/c+9ctxBM1stsF\nNG+3kIax5zjmjj0RLAXN0vnLCr82s47uCCaSuE9bglhZWCVc4UMPrO/V40iK5UUoIxu+VcJdR+ya\n6yUTSbZIyP6k26AKv3HsmHX8pe8BhmNOX4yY9+wanWPASTiiashWjGLJoqUhUdNnbet+bEWQvO1S\n2iPvYKnaJI2MruskEVOxsh1YTlSxHtC8XcjJAoolnv/9T2/HtGxithCwOlLtSJJUpa25goxVrMPM\ntnGccoa3TxwrYWY7mBYkA0vxtFZJkpg35jRa4+K9NbonElMV7zovbupF1uqRFJN0Y9k737VqeX57\nQNtyMjNahb/YNWnbNqT0jnwAACAASURBVExrOZaSWaZgFJnaPLlm8Q5FVvjc3E/xrikLKrZLwrxs\nJLC1BAoqJ7YLAVRRwiQPMLlpkhfU6eLE9hOIbTsXfdf00PZgVbvebJGCM1e11CdCxzUHyLsj3Uot\n/MeZX+Rrb72h5r4g4oHnlwz8bfZOoLzqbYxPTmZUXW0BIRiBbpdTNc3mSlCgCZJ3haBuF/0I+CDJ\nu0pJXSoW2t4RkfffH/b05vj1Y5u56X+eC23/5cMb+MNT2/ijk6fc9dmokkJfpshv/rKZxc+JZTa6\nqfumV4e871y0kcdf3MOi5bt4fkNPyOddGdzh1om2yk6QkWx6Pu9ahelD/uBE0VtD7aLSbL5++wAl\no+RPtorOcN5J0lLWQRITsbb5VH8Nsmx6H+cwvdiWRGnVOZhZ5+OXA+StaiKBhy37EbxOn+rTMXRL\nx9bjFLJJfvC71aTUdCi5hr+EJ4ahy1X36Js7nWIMRh2y1iisCZIFWMSnP4fS0uMtWQHYYDzDw8/u\n8pa+ubBtKeTzdjNC2UYMWZbEGnDZ4PHnffK2EMk3Jo6qByTQ48LnHTCbG5r/2bpaaSJtihSwiu6T\ntvMcOjqcNe+uoBPQLprjzc44OlYBxe+jm9K0PuaTr6Tqjt88TqFk0NaYrE3eAW3ZM5sjfPluLedY\nYFJWHRLz/LPOulndsJDNMEkAyGaSkmbw4DIh7F7UcQVXTv8nprUI7TFoKhX37rgjSmm0jXOZnvTN\n6u77Z9swJu2n6cSWQxM7wPunXIm2YwZm/1hiqkzKuc79T29n5y4np3Z6nTjdUkgnVc49ZayXZAXC\na59PmSpMvP9w5jGcMdrv07njzwx0vur2PbQ42cckSfJIpLT2TK4c93FPuw1mF4yrI5ugAS47ay7Y\nMmec4I/DhXP9wLjeTNEjrvGdgqynTxTvUFOAvNuStcm7OdHkrYmvhfEd4t0MCl+1zOYH8oMDTJFP\nQ983GZBFeteKtoKat+dWIOxDNzPtmAP+OAQ17wWnC5fD204ZGyJvNxvckUBUVewIoViuvfZv/4CY\nLF3Tn0veiqzSE8gnPVzQ6Q/UL0a2KGtmyHReLBuUpaDmHSbkYUeDtMtpSJRANompCnXJcO7lifHj\n2aVtCpO/G6S07xhGG7Pon/DnqoC1fFlDUkyxbjemISkG/UMlLMumZJSRZJvpbZP5+GfO56tPbKef\nHpAt74PSEZnF7HLaXx8qW77ZPKb564fLKSRkSAhLREdTim5LCwWaqXZSRKzLplgj6yWmUCmXbVER\nyCHsKWMbSU2qZ1sOT7CoS6pCY0oBqoYkWSL5DAifbkX0uhtjEJfjDL94BvETlgc0b9kb/1s+IqKX\nZTuGpYhc4u4MLSkGthHnuHHNfPby2Vz/2FKkZE5MHE77Wjkwm7sJJGI6LQ0xioqF5Wb0slQScoJU\nyuCmj8zllj8+AAjTopsfqjXRChUp6t1o9JyjeadUn4ileAlUDb2QwrJt0kmVn3ziHfz48TgbSi+i\nNGRoUBspBsgqpHlrqZqatxfxK0tIkh+kqBsWsRqEI5kJT7iYPKaBK+efSl+fbyFprwim0ndNI3Hc\nS+h7BbknAqbYoJY3OqTNSSGTKMC4pk7MHuGLjqkyHQHTsV1hGscU39aHFkyjdetuFu0Sgkaw1vak\n0Q386NpzhGUFeNv4M2lPtYX93QcoV/Ctj83zAh49Td2Ih4g0SE6V91OJd501hVMmt4a01ffNn8q7\nz5rMt3+9kr39ec+d0tqQ4NZrziYVF8cGY0Nqrek+GHz5I3PRdCtErkGzubftAH5wgOPUuazrEgpR\nkLxdn/dImncwT4W2KRA3QVggPPeUsZw+YxTppMpTq/wA1WT8yFFqpHkfIVg1UpUCVaYxw3KLhMih\nYhCFkkFf0c/JLSt+SktX8ivr1oE1b6fghuf/U0zH5+1XParPT+WczvnO/mCqVD/pQn3MCc6p8Hkv\nyQo/va0lhd9WFVWSBoZLXtnIpkQ9qiLTXu8kvohp3sdZtovexGa7NZklV/O2QdWwveAmmTq5Ednx\ng7c3J0WQn+l/1JYernYkMi4JE2nZ4V3h47dJxhUM1zfsCACphIrlraUuh0zwthFnsiYKIXjJLZzx\nPnPs6SiWWKftEroiS2TKQ0hIjKpvce6gVhIVE0yVeFymPhUThUEUC1s2sJVgoJjTDyeoTZcLtLW4\n5nKfHBpiDWS1Idqakshp8fyntvqaVGstDckQZnt3kp7deiqzW+eIcUgNI8m2l9WsLqkSjym0SZPR\nd8yEgQl8aMpVBNXFUGCSLdf0eYeIXJFFwiBElbRa0buSmfDM+lPGNFV9R5WBQ9bgaN7ffg22YyUI\nam5BIvdWKQT6EkRdYAKPq3LIxxn0N4MTsJZUkSSJ9nTAOmGE1x2nEiqyJCFJEv90/Lt5+4Szqu53\nJKiKHCLaWvcUJKr4K5C3JElV7cnOto7mJLphsddZMphOxqhLxjyiDa7jrmXyPxioilxlNamteR+Y\nvINCyiuZzYPHnth+AnVqmium/2NVm3WBQlCSJHn9DL67ifiRo9SIvI8QauUZr7Xd07wlhd6MT475\nUrXm7aIuJV4iTTdDRSrCmrfN5sw28Zdjcg6bzZ3lN6UpXuIKKWg293Ihq6QSKkk1GdK8dctgW2GD\nc7AFZswj/L5Myat85Urnk5pEQJJclyURU9AtQ0RKuyZ3JxmDu9YbVRdm4YD/s0FpEfV3FZ2OppQg\n74DmrZXcQLhAZivHZFly5CK1bT9Kx26x3MPSPHJXFYlEXMEoOwJATAv5usHGGhjDhORkp9604Res\nUBNiQgsUtFBkiaw2RH28zsvFbLqme1dIkiyRWMJUfFOuQ85lCuiKIF+9GCAMxyeXNftobnL8pwGz\nbGO8kbxeYG+xy8vHPGOUr9U1JeqIyeIeZdstpOFkbnPMo+lEgg/NfC9WOemZ122nIlk66QpbNnax\nAXXvKbSkwlHESSXBuPix6LunoioyTfV+JLoLNaAdKrLkFXoQmncN8jYSnvm2crKH2r7HukCyjrBZ\n1m+/MR7O8hXsl/gd9h8Hr2NraYrPBXzRAZ93Q9zXhG0zTN6viFfBg0HNVKkIbHu1cO91pxO3U1dJ\nskqck9pnVvnjXytqEXWlUHWg/alEtQl+pIC1hng93z7nZs4c+5aqNmu9ZxAm/1cSKg4lIvI+QhiB\nu6vglsNUZYW+rK9590gb+O2m+/0DA8TqlgYtaWaIUIOat9zcy7J9ItLbKjnLpGJlYorsmM2dNddy\niua0Y/JSqoO9MEVO6sZ4AwOlQTQnA1le9zOvGb0TsE0VJSau35sp8v/Yu/P4qMqzf/yfs81MJpls\nkAAJ+yabICgo4i5Qt69WWxUXcKlaRVu1daFUpbUPuFT9Wbva1trqQ12hllddeLpp1YLWlcUVtAjI\nkkD2zHaW3x9nmXMmM5mQZCYZ5vP+h8xkZnLmJMx1rvu+7uuOWevL7eA9uXqMeVyl+/CrDx7Gqzv+\nbZ6npJ7Y/knrzKBmNUZxFy/ZhVRCoB0DyvxQDc0zbG533rKDrjkkbZ6rcLsrWJTvRWNwM3a173Iy\nd0kU4VckaBFX8PZUrsdR3xSBz96yUo45vxO/5IMkCOb6Z8mcKxdFc7cjuwMUAGiqfYFiPq+4OLGk\nx+nnbL3fiNaGmGiNnESCieYeVrOaFr0egZB1fK7gXR4wA+nKj58xHx8NYGBxYs4x4JfNoXMAJdoQ\nCLGg01TEzmz9PslsEqO5A5V5UWF/gNt75IiC0GGeWBAEnFJ9DtQvx6KqPODMwbqzMzkp8/YOm4uY\nV3YhYlumIbLpaMS3j4PSNNK5uEgOIgBQXtJx7tGdOaWbUxUEAQtGXYzohzM7HFcyRRZR2mFnP8HZ\nIcuI+Z2LG89FQYoe+r3NfUHiHjbvjeAdjWnmErqkQCUIAr459RKcMvLkbv+MVFLOb2e4oHFfdLmH\nsv0phs2TL9DSCaYY4QAS9RoAg/dByT1s/t6n9dANsxfuftd2llu/bEJMtTJcw0BdY9jJAPeXJ/aA\n1ttLrAIq8zXNDy8De/WtqI/sT/xQV/C1h5dlQYbeWG0uMSpuhqJ4h819QgClwSLo4SDEUKPzGu7M\ne2d9G0q1oYjpcTyx/jWs/2C3s7etumeY9fqJIri6pjBiVqFdsbWOcnRlrbmOdsBufNGyHau2/MU8\nUCt428PmghL3NOZwF0KVyFaLVCWC0lJ7eU7iP09Ts13oZm1VKCUqsdvaBGCH+SErVdShrug980nW\n/2NZMiuW7QsdqWq7N/OW4tjXHIEMa3jWmuMHzEzTzLytD3YljrgRQVxXUe4aQraXfCnDzKYgpSF7\nIwvZmUqwq5TbjVZExCYYutnD2mn5GPfDiPuwH19ik/oP8xQ0JqqI7S077SmX2JbEOmDAXPs/sMgM\n3pE2H9o3HO08xh42d9bhC4lhUfu47OBk/32bc9YdPwztAJuuGtcdJCVR8BSs+WQRgwODoe0fAqO9\nFOquMdBVn7MbXjDFvvbuzMo5Blfm7Q48/qQ51YmV46C3mFXlyRciboospXyvV0y5GPH35gGaL2Xm\nndziMxv8nmJAd/DufnAZ6JqKSHXBlC2ZsuxMz3FPz9gXAuky784Up/g7AwBZTrwWg/dByJ15P7Rq\nA155dyfuXvmOp9HK8sfeRn2zOTcc0+PY1xxFZWkAJQEFQsRd9GP9J7IztiIFUtUObCsyd/s5acDp\nAOBtB2oF4UvGLwIMEXpbGUR/GIZoNfiQzbaZPsmHoF+GVl8LQdQhVe72PB+agoaWKN57y/xD/vN7\nr+PXaz7Axm3m4+zgamgydMEMmvubo4gLVvC2Mm9REJ0Mz3OeUvTrdg9ZuzPvErnEeZ8lQSvwuTJv\nLe7q2Caaeyy7s57wl0M9VePun2suvZGgt1ZgQukkSKFGSFWJZXSxz6bCMIC4nZl7Mm+/uYey9f7E\nYDPaNHsLy0TWO7bYWspTuQcQVZSUJC5A7A+BgUHz8a/sfx5hcb815SGgusIOgmaTFA0xRIw2xHeM\nxfTBiTXqdvAGzPXtRpv5evaccNAvozpoBqq2Fsks7LOCS0u7+Tu3i3wG+BJroO3gbVfXjq01f85h\nYwc6owLuoFhZGoAAYGhVx9854B16lCUhkXlrZuadnDFqmp5YrpNuODPpQzlV4RLQ8QPXPUfaWYGX\nnbFVliay/AGl5haVMsy/02CK4J343XXN6CHm//3Dxg3M8MhERul+T6LYu5k3kH4IORvsn+WTRQyz\nKtyryjpvhuK+6PLMSSuJkbVU3+/KcSSTPXPeuQverDbPESczKWmAPPQTfLjTu7ymTdgLsbzOmeON\najFEYqq5jlY30KaJEACE6meiWbaWFllV1MGA7GzacP74ryKyx9zooXaIjMtPnAXdMPBKXQPW7/3M\nyXy11lKIZXUIi/sQ9I81s1PV3NtWFAV8e958/PKjTzF1qoh3/55ocFLiC6IZifWPdr/o3U1WW0+7\nGMfOOiUVMVWHjihEeCtSJ9YOxsdNiZaR5vPNDz17pACAOd/tt+daXQ1GrOB95IwifN76mfVzXWug\nnUYuGgTr/BiajOKAbA25CmZBmL9jsxnJNSw4o2w2Pmr+wNmJKbLhWHO/agDtLQJQYgV915y3JEWc\nJVRicTNaVfPnuzPvb596PJa/vBX75E8hKDFUlgWxwzpGe8574UmH4uebXI1rrO5XQwYU47yTxiIU\n9OGdLwdgXcM/UOoL4ZRDvo5h1SFcEDYb5OwT/us89fCRI3HWCWbryTsunYn9zebWh9VNZlCwh8JP\nnjEUf39nBzTdQHFAdoYd506agj98bG7KcsnJhyEkDMCU0WbWfszUIRhUUYTRNWaf7GWXzkSFK6hV\nlRfh9kuPwODK1FXInjlvSUQsrsEwDKfaPDljVDU9MSef5kP1vmuPxusbduGZl825fk+Rmiu4JQc0\nd5CXU2Tw9187B63huJN1L7t0JprazIs+ewcrO4ja2Zq7u9hti7xVzJnMnjIYA8sCGF2ToiNZkvsW\nH43G1ljSnHfXC9Y6M6DU3BfdMHIbvAM+GcsunYnykB+KJODLfe2oTXMRaHNfdCkpRlnSFay53X/t\nHLz18V488Tdzu9p0GXqqi4NcYPDOEbswTSzdB6m0Ac2tewAkrh63la6Fuyg3psUQi5vLqEQB2Cuo\nKJaLIDQMhVi1y1xcJOowAAQUGYJsZn2TB0zA3z7ZZ3bv8oedtZhavVWQ5jOvnu250jZhPwRBgCDH\nYaiJhgMTBtdC/FhE3JpntTPvimAJmvfHzbXWmuQUpe1vbzH/muxqcSdwxs0GNIpVze5aQlJRFAK8\nsdvJrGPbJiIweb35GnIMorVlpN6ayFxDivke3t3/Nt7d/7Z5pyvzdobQ5Rj8483vG9EihII+TB0z\nEOs273aCfak+BOMGV2P9f8zzJEuCk50FjUrobaUQi5s9xwgATc0wg7ccd4oI/ZIPoiA4PbvF4ia0\nqOZzy1xz3n6fhOpQGfaFzfqD8jLRXLalJ4bNq0PeavD4DrOJSHFAdrLYE8dNw4mY5nmcX5FQWQpU\nxkc79w0vH4RqK3sqtiqFAWDW4MPx7KsfQ9s/BJNHVngyQ3e2NW5AotBt+qihngsxURBwyPBEtfWI\nwR23dxw5OH3w6ThsbjhD58mZt98nQdONRJerNMOZpUGf50PeHdB8nmHljnP0smQeQ6oP9oqQ31lf\nDQChoA+hoLeRjP1+3BcCK+bcDkkUUaIcWMFa8rntTFmJH2VJ8/2pmrR0hyyJqAz5sa85mnYIOVvc\nf0/2KE9n5DRLwVIWrKW4QAPM3/PIFH/Hydw1BRw2Pwg5w+ZWT+WInmKHKxd7yZdfkcwPJ1GDIvoQ\njWuQkpYYybIASUlsU1jXGIERC6BdS6x7tVtzhvzmB669WUMM7TAMwwrePkSsHbxkUUaFvxx14X0Q\ny/dCHmAPi7u2WlQVZ/mUvduYMyft7GGsojUchVhsZuYhV+FOyrWg9rB7WzmiH1vLk5QoxFAj9EgR\nEHd1B/OluPp27fhlX0BIFXsgKHFIrYOh7jSDn7OUyPp9+EQ/Lp9yEcT95m5DdsEaYFZdq9aOSoYu\neLL7BmsBgFi6z5mXtzd+QDwAI+aHEGzGjlZzyL0maSlSib1LkRxDaYld7Z0YNncXOk0UToTeYI6q\ndDXzce+6lG7tbUD2I/blKECXMLC8KG27R3exXbHcvXW86aQqWItZ65cVyRu8Az4JqmZkHDYHktY4\np8mQUs2P2wVv7u1dD4Q9kuD+PZX5Qx365OdCb2XeQOJiLpeZd3d4Mu8U1eBdybzNx2U+X+6Lg1R/\nS9nC4J0jTsGatYGCs/uTLWlLQ7tHud9ndmkSJA2KYG4Dam9q4ARvSXSGtQNyAPWNYQhqAO1qO1Td\nvD+shiEKIoKKGbTsIdKI0WbuYiQYgKZ4CuiqigagOdYC//h3nPvawq41yZriFGm1WNXmRofMW0Wj\n8hnE4hZUxMd4AkiqYOLeYcoZQg81QJDjHdbRFrn28lWsYUnPPLp1DGKRtca8bZIzn+tklFa2LMIq\nHrP+I8qS4BS6tEfi0PbVmIFb9cFd6mq0l0JvLYNUXg+p2mxp65f8zsWa3h6C6I9g475NKJKLMCxU\n63kPIcVV+e/XneO2P2R8kmvNtpgYdTiQzOeiCeeiSC7C5AET0j4mZm0vW1bs82Qi7vXSgiDg4gnn\n4uyxp3d7HW86SoqlYnbzEZ8ieoJOQJGg6ZmHzYH0WZV7Pa6U4jF2Zt3Y0vlFdjr2h36uM9RUpG4U\nZ6VjX8wV+/v+fXXGezHoyox9nS8VS9aF2J2x8U22MHhnQTiqoqXduyuY0yXMyvTiRlJ3LtVbgOEE\nb8Xa9UtUAV1GLK5Bttbl2vPjih28NRkCBNQ1heEXzMBoN2Zpj4dRJAcgSaK5VCfuh2EIaFWb8ceP\nVgEAtP2DPB9WA4OuTRosEVenOMOqKPcrIiL2Bh3OnLcVOBUVMdnMumvh3bIwOXifNOxYs2DKZjVZ\nEcvMPa6T23C650EXTVqAG2dcA3XXqMTx6e75bxHlYiLrtT+c7WAfMMzXtocYJUl0/qO3RVRA9SH+\n3ymIb0/sya3IImCIiG01h6ztna38kh/2SgB7HXZYi2B8+egOGzKU+q3aASXmbCBiaHLK5TElUuLi\npegAMp+ja2bivuN+2GlbSltxkeL5MEquDp9dMxNzhx/f5Z/dZUnLxlQtfebt90nQNHPY3C4sTCdd\n5uS+v7Pg3dDazeCdIvPuK+5h855edOVL5q2kec+pMu/OCtbc2/Wm09MLou5i8M6C6x96Ddc/9Jrn\nPrt6Fk7w9gZ3Q9CgR4LmGlZRcTbZ8CsSSopkCJKOPfti5iYMgt061GroIgnOhhRtERXhqIZia3/h\npqg51xpWwwhamar5QWj2zd7eth0fNXyKQfIIaPW1GDwgEVA9OyxZhg8yX3dAqd8pShs62Oc0QnEy\nb6dtpwpVtIbsFe/8kXsI8VuHXdlh/99xQ8xWlfb/PfcmAYD3P+DQkhqMLR8FGCnmvGF1RLOqdodV\nlzgZROyzqYhvH4ehhrk8ys4AZVFwisbs4VmtvhbaPnP4fNSQUqdHtxEtcvrFA4Bf9jkdLY32xEhA\nqsy3LJAI3s7fhC55AtL/G30Kjhx8OAJS9pbq2PPcQyqDnkrsQTnaaMHdrEiWBOiG4azEUBTJO2yu\nmHPebRHVHJXqJCBJXVjDW2o1jXFn92NqzIu5IQO6N8ztVyQU+eU++2B3S3Vx0l2DrL+T0mJfhkf2\nrXS/d/v3ka63eTK7F7zYyd9YV9eJ97b+ffmUp+xCG8MwnA+WRPA2/9XQMXhDC0DdNQbV46LYGfkC\nsLbLnDV5IF54C04wcipXreCtSCIgxWFEfE5L1XJ/KRoANESbMArmnLddLFUe8mPP/nZz/tgXRZEc\nwHdnX463yxow3bUcZXz5mMTxqTK+MuJknHTUZLz7aT3iqo6nt5hrzysrJWyPxc1kU1NwzNQhEMsF\n/CeyCZKswfBFYBhCh0zbfXt8xRgIgoAbzp0Gnyxi9/52zJo4CDe/9ifnfert3jluWRKwaOL52NL4\neYcLjbISH5paDRiGGfyrS8px+lEjURr0YfaUwSgOKLj27EPx8z9thLprDIQae7g8kXnbRU32nuTj\nhpbhzGNGYV9TBDPGV+FnqzbA3GFcgN40EKK1I5tn2Nx1wTFj0FQkq7CaqMiDvsCGfebPOe+YyZ6i\no1NGmu1qX3rjC+e+AaW9u2/wLRdMx0dfNGDK6AGIxTV8/YQxB1Qo1VPupZR2sLH/litCfs8oix3I\nW9oT+5ink/yhe+uF0xGNe7OpMTVluPTUCc4GGwBw8hFD4fdJmDHeu/NWV10wd5wzrN/XejN4H35I\nFRbOH4+jJg/utdfMhuRs+vuLDkdTa+Iz1/130dn5GTE4hEtPnYBDhqcfteqrCzQG7yxStcSmCppm\nz3mbHxya4A3eEPREYxIkdtzy+yRnq0l7DbOdeTt7RUsCdCEOQwtiZ521UUdxJT4PA/sjDeZuZLrq\nZN5V5QEzeFsXEjXFQ1CsFOG4ad4sa3jpUAwOVmN3+15EP5yFY4+ag1DQh+Om1WD9B4lK7WDQgICw\ntbm9gLOPHY09cRn/ec8cNheUKBD3IVDi/aAt9lQrm+996hgzCE8YYfX/1v3QxXarUKxjRe+RVYfj\nyCGHdzj35SV+8z+rIQCCgepQGfyKhLlHJPp6H35IFYJ+Ge1R1cmU7Yst93CsnXkfOnoAJo9MVH+7\nq5zj28cDhoihlRVQRNmpcTDCJdAjRThhzHTPHL1znEWJC5KdrealwNETRnV4HODNEFJ1EOuJytIA\njp4yBIBZiX3aUSMyPKN3uTNve5jX3rSnqjzgyYrt77dFVFRXdF44l5xVpbsYOW5ajee2KAgd7jsQ\n44ZmnqLIld4M3pIo4sQZQzM/sI8lz0PbIympZJpKyPR30JWitmzo+zGdg5j7Cl/Tra+tYXNnj2Xz\nljlfavfztpc7WTtuOXt0W8HSZ+/HbDdOkTSn4GxHnVn1XVtqZtANkUa0W/PRRdY+t1X2jkuy+bo1\nJemvom8+4jpEP5wJI1zq3bQ+oCSGyJUwRH8EmpUZK7LobK0nyCoEXxRGzO/Zlxfo2s5DJa3mHLPe\n2DED6uxDKRS06wLMoFDiSz386QzJG4bntuyZ844797l55v00H+LbJmGYPsN6Qet+Q0R0w3E4b/xZ\nKX9+kc/nFA8CZuFdukpud/FVLqtac8Gdedvnede+xI5x7mFz9+890/RBbwaufCX1g6H7XMvlUHa6\nTaeyrfB+qzlkL7sCEsPmguBu2WmxN9+wMm87wxZE1QreiblQAAiIAc9rGFYWb2gydlrBe0SlOV+8\nP9Jo7kcNIGgFVLvNYeyzQzEiNAynjpyb9j0E5IDTKtL9HyIYkJ3g/Z+ItZtYuGPwhq/dXI8eD3To\nhdyV4F0ePgSRjXMQ+++UDt/r7EMped/idEt07Kvu5P9/dntUIJF5JweCVEU79lW49+VStwwFzGVP\n0Q+OcgJ4mb/jDlm2rhTP5Bv7nRquM2af50TmXeQ59+7fe6bCqT76XO1XCvECJpdD2U5ilmMcNu9F\numF45lI8mbfmLVhzb7cJwargtoKz0SHztrqLWXPefsneDMMM3ppgXQioMnZY2/UNG1AJn6hgQ/1m\nJxjYa4ZDRebws948ELfMPK/L7y8580bS7kh24xdFFlFkWM1gfFZns5i/Q1WwLMo4YegcDAqmn1eU\nZbFDoZqts0KT5Cvv9MHb/Dd52FwUEsHbnitLfs1Uy4DsD8p0u8h1PE4RRqzIXG5WuRdiJzsu2Mv4\netJoo78RBAGGYSRl3lbw3tcOvyIhFFSSNjFxZ96dz3l39fdwMGPwzi7nsz3HDp5PgT62e387rrjn\nn/j7267+1/HEsqrkgjVB6ph5G9awuW7vNuWLwO9zZ97mtZZTdWy9hu7KvJvaYigt9iHgk+GTzCD9\nft0mAMCospEA9OeBHQAAIABJREFUgPJQ9ypFk5tcOHtuW/RwCLIkmPv/Wpm37rOat8T9Kfv+njv+\nLBw39Oi0P7OzZRzJnbHcyoq9c8IBOXWBlz13XGQdWyITTPS/brcadSRn+ikzbyl1Jp+JfcEW19MX\nOdmvOXxQ560h84m9JMtd4S675rQHlgc6jES4f++ZMu+DbXqhOwrxHOTygqWvLqaZefeStz7aCwBY\n+ddPnPvcm444QyuiO/M2AAiJPautYFgSrwX8m6AM+wQ++WRnztvOvINyEIh3zLxrKspR7h/gVMi2\nurbpBIDRZWYR0qSRlTh99ghMH9e1Stprzz4UX9a3ej4EKkJ+nHroNLypbcWYkrF4+8P9MNpKofgT\nFfHmkjd7NzJft1oHdjZ3lep7t1wwHe9vrcfXjh8DUQT+ZT9WTP2zrz17Cl5Y/wVOn20VaLlesqqi\nCKOGhPD5LnP0IPkDIdV8qzNsbkVaWRJxwdxxad8DAFx51hSsa9iBrZFdnuHjZGcdMwqqpuOsY1IX\ntOWj75w/Df/3n+04+fBEEdSx02rQ0h6Hbhg4ekqiHuPCueMQ8MnY+mWip26m4D24MohTjhyOKaMq\nO33cwcyvSDhzzshO29MebIr8svmeh6R/zxfPH9/lTUk6M2N8FU6cXotjpw3p8WsdCAbvA7S18b+o\nC9fjqCHezQXswCYEWiGW1UPbMwLRlJm3GagF0XA2FnGG0q3g7YsOQoV/KBqKdwBSvMOcd5EcgBAX\nnNakLaq5DehXpo/F7JpEj+sLJ3wNL3z+NzRGm5znAeaQ8NeOTywDy+TwQ6pw+CEdA/3Xjp6Mq6uO\nwsdb67B+rbkft/sqNCgH0BSzh/SVbgZvMem22XMaSD1sPmFEhVOpfv5J4/Avc5dMKGLq4dXqiiAu\nPTWx/lpAYthbFARcd85UfPfnr6c8lmCKLlPJAf74aTU4cXpth8e5nXncGEzYfiZ+uWEfFhxydtrH\nBQMyFn7lkLTfz0dDBhTjklO869/H1pbh21/vuKzOXimwbXeLc1+mYXNBEHDeiWN74Ujz21ePHZ35\nQQeZTO/5pF6qmpclsU/+XzJ4H6AH3vkFAGDW4Bmebln2XHdgqtmcJdJa4Q3emrdgDYCZfetyIhu3\nhs1jcQ2KHgREQBOirszbqjZXJBTFi9BqBe9P2z+CKIiYPND7ITin5kjMqTkSb+95DwNTNFzpLe4P\nUPeSnqASRFMssZtXd7bLSw6YPlmCqplDy501TrAdWzsbr+5ch9HWlEEmiepz89+yksQUQ3Kmn7pg\nzTts3tWGVhWBciyddWPXHlzg3Bdt7o0/iAoJ//K7SdVVZ04ZAJKnWARRSxo2TypYgznsbcQDEKwm\nJPYcciSmQdB9ZvAWI4iq1lIxe523LCIoF6FNaoXgb8OeyJeYWDnes4mF2+GDDuvRe83Ep4hmP2rd\n8GTe7mpyo7uZd9J8kt8nOXPQXWn1eN74s/DVMachIHdvXbS3mYP3WFIOm9tz3vbwd+FNN2ad5ClY\n40cYFSYWrHWTmlRY1CGQGELSsLm9zjuRedubeiSGzc3gFo1rEKyGJHEjirC1TttemuWTRQSVIkCO\nQRpgNvaYOWh6z99UNwmC4HyIuueQPOuVXZttHIjkbPdAd0USBfGAAnfyum+3mKp5bqfaijIx523/\nfEbv3uYtWOvfG2QQZQuDdzfFda3zBwg6/vi3T7Hps30AXJm36FoTaFecJw2bb9vdgi92mvPcUSOM\ntri53tVu0qLIEkqUIATRgFS1A7IgY2rV5J6/qR6wP0QVxTtsbjNUxbOTU1clD5tne79cZ847xfda\n2uOe26kL1rpXbU5dx8ybiMG725Izb7ufucMKyA88/T6AdMPmKgZVFGFghbWHtWvplb0dZtQIO3tx\n25m3IotOYBT9EQwtHppoitJHjpo0CANKAzh8fLVzX1BJtAOVoXSrjaA7eI8cHMKF88b37EAzSZEo\nf+/iGZgwvByzJ3v34lZkEbMmVnsKopKHzZl4976JIyowqDKISSMrUFHau21iifIFL1u7SdW9WVgs\nufuVNY9tf3inK1i7Y9FMPP3uK3izHYAuosgvIRzVnK0129VE8LaboiiyiGI90XQk5Ov7db9nHjMK\nZyYtYXIPm/vl7q0td+/zfPslR2R9swdnnbfr1zRuaDluuXBGx8cKAq4+y+z89vQ/twBwVZsbicdQ\n7xo3tBx3XXVUXx8GUZ9i5t1NquEdNjdbV7rms63MuzJkZsTJvc0Bs1GLTxFRVGT9GgzRqdy2s+zW\nWBva4+3wiT5nWN0niyj3JdYvhtIUqvU1d8FadyrNgUTBmiSaLUaz3Xwh0XGte+Pe9pJBnfVqRJRF\nWc28V6xYgffffx+CIGDp0qWYOjWxdnPlypVYs2YNRFHElClT8P3vfz+bh9LrkofN46ruZNsAnK8H\nWMN6qea8BVmFJIoIBqzgrZutIOubIs6weVu8De1qGAEpALs1hSKLKJMSwbs0zaYbfc09593duWq7\nOMkO2tnecEBI7pd6gOSkJi3MvIkoG7KWeb/55pvYtm0bnnrqKSxfvhzLly93vtfa2opHHnkEK1eu\nxBNPPIGtW7fivffey9ahZEVyG8u4qnn7lVvBu9Rqv6m72qMaqnnNJPniePHzv6MdjQDMOe9ia39i\nSfdBgIDWeBva42FnO0/ALFgr8yeCd1mgf3ZOch9z8qYkXWVn3nZGm+3t9xLD5t2M3slLBhm7iSgL\nsvZJuG7dOsyda+5WNWbMGDQ1NaG11exzrSgKFEVBe3s7VFVFOBxGWVn6/Vb7Ql1jGI+t/djZDjJZ\nqszbvVOY0/LUCgLujUnsrFoYsAN/+XwtXtn5uvVY0Wk6IUkiipUgGqNNiGgRTxaryKI3ePeDOe9U\nZDExsNPtzFtK7K8N5K5Pc7eLxa0n9tU2gURUGLI2bF5fX4/JkxPLlyorK1FXV4eSkhL4/X5ce+21\nmDt3Lvx+P04//XSMGtV5v+aKiiBkuXeXCVVVpZ8rXrHyHWzZ3oiyUABXnNVxO8rikOJ5viCJiXXb\ngJN5y4qEqqoQJFkCYEAQAD3uAwLtHY+nrBhl1hy5IokYXl6DD+o+BQBUliSC9eDqEAJFiYA9fNAg\nVA3su3nvdOfRCNYC7wB6OIjSEn+n5zudygpzSkCWRef5AZ+E8cMruvV6mVx46kT86JE3cN68Q7r1\n+iWhAKqqQrj6nKn45aoNOHXO6C69TjbeS6HhOewdPI89l4tzmLNqc/cwZGtrKx5++GG89NJLKCkp\nwSWXXIKPPvoIEyZMSPv8hoaOwa4nqqpCqKtrSfv9fY1h69/2lI/b19CCOiVxf2tbzDNsPn54CB/s\nBMLhOOrqWtAeiSWK1TQZhi4msnPLVacfin+tM3+uKAoY5B+ED2AGb9lINKNoaQ5DiyZ+dVq72Ol7\nyabOzqMAH04oPh8vvl0HjDO6dYzhtqj1WnCe/7MbjoMgICvveVRVMX57y4kQRaFbr9/cHEZdXQtm\njhuIw7v4Opn+FikznsPewfPYc719DtNdCGRt2Ly6uhr19fXO7b1796KqytzcYuvWrRg2bBgqKyvh\n8/lwxBFHYNOmTdk6lG6xLzbSjdJ2GDbX9KRtPs3MW7NeJ2q0QvBHrBcXALXjdZMsys7wuiyJqA3V\nON9zL7tSJNFTCFWi9M9hcwCo8g0GNB/8Svf+1Ox13u4qc9GqPM+W3hqaL8StGIkoN7IWvOfMmYO1\na9cCADZv3ozq6mqUlJhBpra2Flu3bkUkYgazTZs2YeTIkdk6lG4xUqzTdY8edChYi2uA7B42N7Nq\nOxjvqFqDwNRXrRcSYcQ7NpdQRBmqtaRMEgUMK0kEb/dabrt/+IiQuctSd/t254IdfANK9wZ5ZDm3\nc91ERPkga8PmM2bMwOTJk7FgwQIIgoBly5Zh9erVCIVCmDdvHr7xjW9g0aJFkCQJ06dPxxFHHJH5\nRXPIvdRH1VX88aNVmO3aBlQ1OmbeYlFiqMQQzO87Veae1xZgtJVBLDYff+GEr+HThs9RVTQQmlYH\nwCxYqykZjONqZ0MWZcypmYUnsN45JgD47uGLu70eOVfsgjNfN1qjAole6WKWq8x7C+vUiCgXsjrn\nfdNNN3luu+e0FyxYgAULFmTzx/eIu8nGxvoP8cbut/HG7red76tJvc1jqg6xsinxfGgQBQGaYeCL\nvc3eFzdE6K3lQPUOAImtO4FEm1VZFCAKIs7vZH9nScxun+/eYGfe3a02l6zny3mSeff3iykiOjjk\nRzrTBxKZd+pGG8lz3jEtBiHYAr3NrArXoEIUBei6gR/8YZ33yboIvS310rhZE83+2ccfVpPy+/mm\nImQO6Q8o7V7v9UTm3b+D9xGHmPUcIwf3zzX3RHRwYW/zDARBgF/s2Jc7ntzbXG6EIBjQWiogBJuh\nG6qzx7Wn8xoAASLu/8ZXsOaLCMaVj/Z878hJgzC2tgyVKTZc+NkNxyY6teWJMbVluPvq2RhY1r3g\nbQ+79/fg/c2zJuO8ligGlhVlfjARUQ8xeKfhDJsLqbt6JQ+bq4K5lE2PBiHpkpN5R2Kad/03zD2m\ny0sCWDTp/JQ/e0CaQJevexdXl3c/oLl7m/dnkigycBNRznDYPI3EUjEBmqF3+H6HLUFFa9vOmB/Q\nRWiGBkkUsLehvUPmDZ2nvavsXuH9PfMmIsolRpE03FXDWlKWDXirzQ3DgC5Za7jjPhi6BNWIQxIF\nGAY6ZN727mCUmZ1550vBGhFRLjCKpJEp845riYC8e387oJidwIy4HzBEqIaayBalpODP4N1lSp7M\neRMR5RKjSBruOW/N6Dzz/vmfNkFQYgCAUn8I0BKZNwAIHQrWGIi6yqdIkCUBRT6WZxAR2Ri803A3\nadFTDZu75rwjMRWCEoUiKvjhJbNRO6AUcT0OwT67ycPmev9fn91fyJKI755/GM4/aWxfHwoRUb/B\ndCYDM/NOVbCWCOiabkDyx1DmC6G02I/yYDF2RXRIkpW+JxesaflZNd5XDhle0deHQETUrzDzTkN3\nNWlJOeftWuet6ToMKWoOmQPwS+YabdGa6xaS57w1XjMREVH3MXin47RHFVLPebuGzXUhBggGQtbu\nXgEreAv2RiVi0rC51rHpCxERUVcxeKdhrxRLV7AW01QnOzdEs9K8WDG37fRbu3wJaTNvDpsTEVH3\nMXhnIKYpWPt8dyN+9Zy5B7kmmpXmQTt4S1ZmbQftDpk3h82JiKj7GLy7INWcN0QNb31sbt9pWMG7\nWDaDtzNszsybiIiygME7A90wUg6bu7umGZJZvNZh2FxUARgQ/OGkF2XmTURE3cfgnYFupMm8reBt\nGAYMKXnY3NoRTFQhDdoGsbjZ05iFTVqIiKgnGLwzMAwj5Zy3ORRuQDcMCLKdeZu7StnD5pBUSKX7\nAQCXTlqQk+MlIqKDH4N3BuaweYrMGwAkFf/e+SaU2q0AgGDSnLchqBCKWmDEFVQFB+bkeImI6ODH\n4J2BYaRYKmavAZdUPPnpaufu5DlvTYpADIRhREKQRc5zExFR72DwzkDXOxasybDntL33FyctFYvI\n9eY3wqEO+38TERF1F4N3BobRcT9vUU/MaeuRIud+RTSXgNnD5mFxHwBAiIYwpHgQJMOH+M4xEFiv\nRkREPcDgnYFhGNCT5rwF3cysBUmFEU0Eb8GKyvawuV1ULmh++CQfZukLoe4cl/2DJiKigxqDdwYp\nC9bsJiuSCgjmBPjo8Hzn285SMYtgPV4wmHITEVHPMXin8NH+TyH4zMYqqQrWjLgVjK3gbRgCSvUa\n5/uKKENxFagJujeYExER9QSDd5KWWCt++t5v4J/2CgAz8/7vnibPY1S79kxSIQgGYAiQRG9WHfKF\nnK9FnbuIERFR72HwTtIWbwcAp6hM1XTsaWjzPMauX7MzbxgCxKQzGfKVOF+LOnuZExFR72HwThLT\nY57bcVV35rVtumadNsnsXW4Gb++pLHUFbwHmELoB7+sQERF1B4N3koga8dyOxXVAMAvWoh8dgaJw\nLbR6c37bnXlLSeu/Qkpi2Dz5e0RERD3B4J2kPSl4x1XNybz15gEI7T0aajQAABCUqBO8haQzWepP\nBG8hKXgn3yYiIjoQ7NmZJBz3bt8Zs4bNDQMABLRH44Dqg6EqEAJt1lruFAVrimvOW2SwJiKi3sPM\nO0lY6zhsLgg6YJinqj1ilprr4WIIgTAEUYNhCB0CtGepGDNtIiLqRQzeSbyZt4G4pjtD4wAQjpql\n5kakGIJgQPBFAUPskHlLouR8bX+L5WpERNQbGLyTeDJvwUAsrgGCDlmUMLqmFLo5fg4jXJx4nCFA\nTMquJw+YABgCYtsmdPgeERFRTzB4JwnHXcFbVJ05b1EQ4VcS2bQeDSYel2LYPOQrwYTGi6DtGclh\ncyIi6lUM3knCqmvYXNSdanMR3uAN3fV1ig5rAKwit8SwORERUW9g8E4Sdi0VEyTVWectQoLf5w7Y\n7lPXMfMG4AyxC4zeRETUixi8k3gzbw2abkBwhs1dp8u9Q1iKOW8gEbyd2M2KNSIi6gUM3kncTVoE\n0W5ibgZvn2vY3NATpy7VUjEAmDKyEgBw2NiBnvs5BU5ERD3BJi1JYpqrt7lkB28doiBBkdJn3qnm\nvOfOHIZDhldgWHVJh+8RERF1F4N3kqh7Y5KkzFv2BG9vIE+VeYuCgBGDQx3uJyIi6gkGbxfDMBDX\n4s7txLC5DkmQkoK3O1innvMmIiLKBs55u8R11bttp5TIvCVBhCy5h8q9WXiqYfNkrFcjIqLewODt\nYs93S4JZmCaIGiCqEATAJ/o9mbe7YC3dsHk6zNGJiKgnGLxdolbwDohW9zRRg+Azq89LlNABF6wl\nG1xpvu7omrLeOWAiIipInPN2iVvFakVSEG1aCyCp5sYjAEqVEGQxdcGakWadd7KTZtSiOCBj+riB\nGR9LRESUDoO3i5N5C2aGLEgaBMXMvEt9pZCTsm33110ZNpclEXMOHdJ7B0xERAWJw+YuMavS3O8M\nm6vOsHmZrzT9UrE07VGJiIiyIWPw3rp1ay6Oo1+IWcPmPqMIgJV5W8Pm5f4yyHLP5ryJiIh6Q8bg\n/e1vfxsXXHABVq1ahXA4nOnhec0eNldgBm9zztvMvCuKyrwFa8jc25yIiCgbMs55P//88/jkk0/w\n4osvYuHChZg4cSLOPfdcTJ06NRfHl1N2gxZBV2BoIgRJBXwRGLqIkFKMqBRJ/URD5LA5ERHlTJfm\nvMePH4/rr78eS5YswdatW7F48WJcdNFF+O9//5vlw8stO/OGIQG6DLG4GWKgHdq+wVCUpA5rbhw2\nJyKiHMqYee/cuRN/+tOf8Je//AVjx47F1VdfjWOPPRYbN27EzTffjGeeeSYXx5kT9py3oUkwNAmC\nYt6v7hwHRfL2Nvd0WwOYeRMRUc5kDN4LFy7E17/+dfzhD3/AoEGDnPunTp2aceh8xYoVeP/99yEI\nApYuXep5/K5du/Cd73wH8XgckyZNwp133tmDt9E77A5rhi4CmnlqDEOAEQtAlrztURXZvc5b5Jw3\nERHlTMZh8zVr1mDkyJFO4H7iiSfQ1tYGALj99tvTPu/NN9/Etm3b8NRTT2H58uVYvny55/t33303\nLr/8cjz77LOQJAlffvllT95Hr7CXihmqBEO39u6OKwAESJLgqTZP7rbGYXMiIsqVjMH7e9/7Hurr\n653bkUgEt9xyS8YXXrduHebOnQsAGDNmDJqamtDa2goA0HUdb7/9Nk466SQAwLJly1BTU9OtN9Cb\n7DlvXXNl3roMSTSryd0BO3nZGIfNiYgoVzIG78bGRixatMi5fdlll6G5uTnjC9fX16OiosK5XVlZ\nibq6OgDA/v37UVxcjLvuugsXXHAB7r///u4ce6+z57x1VYSzFEyTnEDtnvNOzrwZvImIKFcyznnH\n43Fs3boVY8aMAQBs2rQJ8Xg8w7M6MgzD8/WePXuwaNEi1NbW4qqrrsLLL7+ME044Ie3zKyqCkGXp\ngH9uZ6qqQp7bwqfmMUqiD7D28jZ0CQFFQlVVCEUlifddFFDgXMIYAgYOKO7weoWiUN93b+I57Dme\nw97B89hzuTiHGYP39773PSxevBgtLS3QNA2VlZW49957M75wdXW1Z7h97969qKqqAgBUVFSgpqYG\nw4cPBwDMnj0bn376aafBu6GhPePPPBBVVSHU1bV47mtpN39GuN2A4Lf28tYlSKKAuroWxOJa4sGu\nixEYApoa2+EvwOQ71XmkA8Nz2HM8h72D57HnevscprsQyDhsPm3aNKxduxbPP/881q5dixdffLFL\nmfecOXOwdu1aAMDmzZtRXV2NkpISAIAsyxg2bJizTnzz5s0YNWpUV99L1tjV5mocTuYNXXIqy93z\n3JJnqRg7rBERUe5kzLxbW1vx5z//GQ0NDQDMYfRVq1bhtdde6/R5M2bMwOTJk7FgwQIIgoBly5Zh\n9erVCIVCmDdvHpYuXYolS5bAMAyMHz/eKV7rS1E9BlmUoaqAPedtqDJ8VtB2B2jJ9bVhCDBARESU\nGxmD9w033ICamhq89tpr+MpXvoLXX38dP/jBD7r04jfddJPn9oQJE5yvR4wYgSeeeOLAjjbL4loc\nPlFBXNUhfjEdyvCPEd5+CJSqjgMU7gI1QTAYvImIKGcyDptHo1HceeedqK2txa233orHHnsML774\nYi6OLeeiWgw+yYeYqkNRy1C291hA9UNJUSgnJbdKNRi+iYgoNzIG73g8jvb2dui6joaGBpSXl2P7\n9u25OLaci2kx+CQz81ZkCbpuBmR3NzWbuymLJAuoCAVydpxERFTYMg6bn3XWWXj66adx7rnn4rTT\nTkNlZSVGjBiRi2PLuZgeQ7lYigZVQzCgQNV0AHDmvN3cwfvsY0alDPBERETZkDF42wVngLmka9++\nfZg4cWLWDyzXDMNATIvDJ/kQ13Qosoj2iAogc+bNGW8iIsqljOmiu7vaoEGDMGnSJCeYH0xUXYUB\nwwzeqg6fLELVzcxbSbEVqOgJ3kRERLmTMfOeOHEifvKTn2D69OlQFMW5f/bs2Vk9sFyLWq1RFVGB\nqhlQZBGaZs15KykK1kT3rmIM30RElDsZg/eHH34IAHjrrbec+wRBOOiCt92gRbY28VZkyZnzTpV5\ne4fN9RwcIRERkSlj8H788cdzcRx9zt4ONBG8RSd4y3IiUFeVB1DXGPEOmzPzJiKiHMoYvC+88MKU\nc9wrV67MygH1leTM2yeLUK1hc9k1RL78yqMQi2t4+p9bnftYsEZERLnUpQ5rtng8jvXr1yMYDGb1\noPqCvZe3CHN+293HXHb1MZcl0dkaVI8UQQyEUSQX5fBIiYio0GUM3rNmzfLcnjNnDq688sqsHVBf\nienmsLmExLC5rUM3Nfs5Hx+B4NCdOPb4g2v+n4iI+reMwTu5m9quXbvw+eefZ+2A+krMybzNU+Ju\nzCKLqZbGGTCixZD3TIFPUlJ8n4iIKDsyBu9LLrnE+VoQBJSUlOC6667L6kH1BSd4GzIA3ZN5y510\nTzv4VrwTEVF/lzF4/+Mf/4Cu6xCtoq14PO5Z732wiOnu4B3zbEYipxk2JyIi6gsZo9LatWuxePFi\n5/ZFF12El156KasH1RfsgjUY5ilxr+2WUgybc3UYERH1lYzB+9FHH8WPf/xj5/bvfvc7PProo1k9\nqL4Qt9Z5Q7fmvBV3wVr6wfGDsVUsERH1bxmDt2EYCIVCzu2SkpKDMmBFtCgAQLCCtzvzdq/ztjHx\nJiKivpJxznvKlCm44YYbMGvWLBiGgVdffRVTpkzJxbHllB287cxbUdzrvDnnTURE/UfG4H3bbbdh\nzZo12LBhAwRBwJlnnolTTjklF8eWU1HVCt6anXm7C9YOvpEGIiLKXxmDdzgchqIouP322wEATzzx\nBMLhMIqLi7N+cLlkZ96GZgZt91KxUNDX4fFVZQEAQG3VwXUeiIio/8s4Hnzrrbeivr7euR2JRHDL\nLbdk9aD6gp1566oZvH2yiOVXHolLTjkEIwaHOjz+lCOH44KTx+HKMybl9DiJiIgyBu/GxkYsWrTI\nuX3ZZZehubk5qwfVFyJaBIqoQNPM24osYsiAYhx/WG3KxyuyhHkzh6XMyomIiLIpY/COx+PYujWx\ng9bGjRsRj8ezelB9IaJFEZD8iKvWHt6ddFUjIiLqSxnnvL/3ve9h8eLFaGlpga7rqKiowL333puL\nY8upqBpFQPYjxuBNRET9XMYINW3aNKxduxarVq3CkiVLUF1djWuuuSYXx5ZTyZm3z9UelYiIqD/J\nmHm/9957WL16NV544QXouo4f/ehHmD9/fi6OLWd0Q0dUi8Ev+xHXmHkTEVH/ljZC/eY3v8Fpp52G\nG2+8EZWVlVi1ahWGDx+O008//aDbmMTuax6Q/IjHzYo1Bm8iIuqv0mbeDz74IMaOHYs77rgDRx11\nFICDt4931FrjHZADaGPmTURE/Vza4P3yyy/jT3/6E5YtWwZd13H22WcflFXmABCx1nj7JbNgTRBS\n7yRGRETUH6RNL6uqqnDVVVdh7dq1WLFiBb744gvs3LkTV199NV555ZVcHmPWOZm3VbDmk6WDdpSB\niIjyX5fGhmfOnIm7774br776Kk444QT8/Oc/z/Zx5VRYjQCAWbCm6hwyJyKifu2AolRJSQkWLFiA\np59+OlvH0ye8mbfG4E1ERP0aoxSA9ngYAFAkB9DcHkdx4OCqpiciooMLgzeAdtUM3qLuRzSmoao8\n0MdHRERElB6DN4D2eDsAIBoxT0dVeVFfHg4REVGnGLwBtFmZd6SNwZuIiPo/Bm8kMu+WVvM2gzcR\nEfVnDN5IzHk3NZnd1TjnTURE/RmDN4C2eDsUUUFLmxm8K0L+Pj4iIiKi9Bi8YQ6bFytBRGPmpiQ+\nhduBEhFR/8XgDXPYPCgXIRLX4FNEiGyNSkRE/VjBB2/d0BFWIwgqRYjFNfiZdRMRUT9X8ME7rEZg\nwECxHEQ6NoGoAAAYmElEQVSUwZuIiPIAg7ddad6so6E5Cr+PwZuIiPq3gg/eMc3co3zL9jYYADNv\nIiLq9wo+eMd1M3gbunkqGLyJiKi/Y/DWVfMLBm8iIsoTDN5W5g3dDNo+peBPCRER9XMFH6ni1pw3\nDPNUBFiwRkRE/RyDtzPnbWfeDN5ERNS/MXhzzpuIiPIMg7cz583gTURE+YHBW/MOmzN4ExFRf5fV\n4L1ixQqcf/75WLBgATZs2JDyMffffz8WLlyYzcPolDNsbhWsiSI3JSEiov4ta8H7zTffxLZt2/DU\nU09h+fLlWL58eYfHbNmyBf/5z3+ydQhdkrxUTNP0PjwaIiKizLIWvNetW4e5c+cCAMaMGYOmpia0\ntrZ6HnP33XfjxhtvzNYhdEksqcOapht9eThEREQZZS1419fXo6KiwrldWVmJuro65/bq1asxa9Ys\n1NbWZusQukR1qs3NzLu4SOnDoyEiIspMztUPMoxERtvY2IjVq1fj0UcfxZ49e7r0/IqKIGS5d4vJ\nqqpCkD63bugi5h85Al89aTwkznsfkKqqUF8fQt7jOew5nsPewfPYc7k4h1kL3tXV1aivr3du7927\nF1VVVQCA9evXY//+/bjooosQi8XwxRdfYMWKFVi6dGna12toaO/V46uqCqGurgXN7ebrGrqEuTNq\nsH9fa4Znkpt9Hqn7eA57juewd/A89lxvn8N0FwJZGzafM2cO1q5dCwDYvHkzqqurUVJSAgA45ZRT\n8MILL+Dpp5/Gz372M0yePLnTwJ1NqqvaXBILfuUcERHlgaxl3jNmzMDkyZOxYMECCIKAZcuWYfXq\n1QiFQpg3b162fuwBi7matMgSh8uJiKj/y+qc90033eS5PWHChA6PGTp0KB5//PFsHkannI1JdAmy\nxMybiIj6v4KPVqquWg1aBBaqERFRXij44B3T4xAMs4qdmTcREeWDgo9WcSt4CwJboxIRUX5g8NZU\nCKw0JyKiPFLwESuuxwFDYqU5ERHlDQZvPW4tEyv4U0FERHmioCOWYRiIaXFAl1lpTkREeaOgg7dq\naDBgwGCDFiIiyiMFHbxjWsz8QpcgcdiciIjyREFHLDt4G5rEYXMiIsobhR28rb7mhsaCNSIiyh8F\nHbHszFtn5k1ERHmkwIM3M28iIso/BR2xYnoi82a1ORER5YvCDt4sWCMiojxU4MHb3stb5FIxIiLK\nGwUdsRLrvGXOeRMRUd4o6IjlLBXTRQ6bExFR3ijs4O3qsMaCNSIiyhcM3gCgSdzPm4iI8kZBR6zE\nsLkERSnoU0FERHmkoCOWe9hcYcEaERHliYKOWFFnqZgEHzNvIiLKEwUdseJWhzWDmTcREeWRgo5Y\nMVfmrchS3x4MERFRFxV08I46c94iFLmgTwUREeWRgo5Yqq5CggRAgI/Bm4iI8kRBRyzVUCEKMgAw\n8yYiorxR0BErrschwpzrZvAmIqJ8UdARK66pruDNgjUiIsoPBR28VUOFYJingJk3ERHli4KOWKqu\nQrAybxasERFRvijoiBXXVQgG57yJiCi/FGzEMgzDzLw5bE5ERHmmYCOWqqvmF8y8iYgozxRsxIpr\nVvDW7cyb1eZERJQfCjd423t5W8PmLFgjIqJ8UbARy868DZ1z3kRElF8KNmLFrMwbmghBACRR6NsD\nIiIi6qKCDd6qlXnrugBFFiEIDN5ERJQfCjZ423t5G5oIRSrY00BERHmoYKOWXbCmaQJ8CivNiYgo\nfxRu8LaHzTWBmTcREeWVgo1acd0O3iIUpWBPAxER5aGCjVpxa85bUwXIYsGeBiIiykMFG7Xcw+ay\nzEpzIiLKH4UbvK2CNV0TITHzJiKiPFKwUcvpbW6IkCVm3kRElD8KN3jbvc11ETKrzYmIKI8UbNSy\nm7TAENkalYiI8krBBm9nP29dhMTMm4iI8kjBRq2Ys6uYBJmZNxER5ZGCDd5x97A5C9aIiCiPyNl8\n8RUrVuD999+HIAhYunQppk6d6nxv/fr1eOCBByCKIkaNGoXly5dDzOGSrYgaNb/QJBasERFRXsla\n1HrzzTexbds2PPXUU1i+fDmWL1/u+f4dd9yBhx56CE8++STa2trw6quvZutQUgqrEQCAocssWCMi\norySteC9bt06zJ07FwAwZswYNDU1obW11fn+6tWrMXjwYABAZWUlGhoasnUoKUXiZvCGJjPzJiKi\nvJK1qFVfX4+KigrndmVlJerq6pzbJSUlAIC9e/fi9ddfx/HHH5+tQ0kprEYhQLCqzZl5ExFR/sjq\nnLebYRgd7tu3bx+uvvpqLFu2zBPoU6moCEKWe2/f7XA8Ap/kRzsElJYEUFUV6rXXLjQ8dz3Hc9hz\nPIe9g+ex53JxDrMWvKurq1FfX+/c3rt3L6qqqpzbra2tuPLKK3HDDTfgmGOOyfh6DQ3tvXp8YTUC\nBQoAIBqNo66upVdfv1BUVYV47nqI57DneA57B89jz/X2OUx3IZC1YfM5c+Zg7dq1AIDNmzejurra\nGSoHgLvvvhuXXHIJjjvuuGwdQqci8QgU0QcALFgjIqK8krXMe8aMGZg8eTIWLFgAQRCwbNkyrF69\nGqFQCMcccwyee+45bNu2Dc8++ywA4IwzzsD555+frcPpIKxGUSGXAgAL1oiIKK9kdc77pptu8tye\nMGGC8/WmTZuy+aM7FddVqLoKQTffPoM3EVHfevnlv+OEE07u0mN/8pP7ce65C1BTU5vlo+q/CjJq\nRa0GLbv2xgBw2JyIqC/t2vUl/va3tV1+/PXXf7egAzeQw2rz/iSimcHb0MzqdS4VIyLqOw88cA8+\n/HAzHn30N9B1HV9+uRO7dn2JBx/8Be66607U1e1FOBzG5ZdfhTlzjsV1112F73znFvzzn39HW1sr\nvvhiG3bu3IFvf/u7mD17jvO6qqpi+fIfdHj+J598hPvvvweiKGDKlGm49trrU95n/5zRo8di1aqn\n0NjYiOnTD8eTT/4v2tvbcd11N+Ldd9/Gyy//HbquY/bsObj11u+ipaUFd955G9ra2lBSUoI77vgf\nXH75Rfj9759AMBjEhg3v4cknV2LFih93+5wVZPCOWsEb9rB5DtuyEhH1Z0//Ywv+89HeXn3NmROq\ncd5JY9N+/4ILFmL16qdx2WVX4pFHHoaqxvGLX/wWDQ37MWvWUTj11DOwc+cO3H77EsyZc6znuXv3\n7sF99z2E9ev/jT//eZUneLe0NKd8/oMP3oebb16KsWPH4Uc/ugO7d+9KeV86W7duwRNPrIbP58O7\n776NX/zitxBFEeeddxauvfabeOKJxzFr1myce+4CPPXUSrzzzls47rgT8dpr/8L8+afgtddewbx5\nX+nROS3I4G33NTc08+0z8yYi6j8mTpwMAAiFSvHhh5uxZs1qCIKI5uamDo+dOvUwAObyZHcXz86e\n/8UX2zB27DgAwO2335n2vnTGjh0Hn89crRQIBHDddVdBkiQ0NjaisbERn3zyEa644hoAwPnnXwQA\nqKmpxW9/+0vMn38K3n33bXzjG1cf+IlxKczgrSU2JQFYsEZEZDvvpLGdZsm5oChmD46//vUlNDc3\n4+c//y2am5txxRULOzxWkhLNu5KbgaV7fqpNsFLdJwiJxE5V1Q7Ht3v3Ljz11Er87ncrEQwGsXDh\nedZrSTAM3fNaY8eOw759+/Dhh5sxatQY+P3+zk9CBgUZtSL2piR25s2CNSKiPiOKIjRN63B/Y2Mj\nhgypgSiKeOWVfyAejx/Q66Z7/siRo7B5s7ni6a677sR///t5yvuKi4uxb5/ZbGzjxvdTvn5FRQWC\nwSA+/vgj7N69G/F4HBMnTsLbb/8HAPDcc6vw4ot/AQCcdNI8PPDAPZg375QDeh+pFGTwthlx88qH\nmTcRUd8ZMWIUPv74Izz00P2e+0844ST8+9+v4vrrr0FRURGqq6vx6KO/6fLrpnv+9dffhJ/97P/D\nNdd8A6FQKUaOHJXyvjPPPAf3338vbr75egwcWNXh9ceNG4+ioiCuueZy/P3v/4ezzjoHP/zhD3Hu\nuRdg06YNuO66q/Dvf7+G448/EQBw8snzsHfvXhx++MyenTAAgpGq6Xg/1Jvt5uJaHNf89hnojdWA\nIeKWC6ZjwojOe6tTamyn2HM8hz3Hc9g7eB57rrNz+Pzza7B79y584xvfPKDXS6Ug57wVSYHeMNi5\nzcybiIiy6Z57/gdffrkTd911X6+8XkEG72SsNiciomy69dbbevX1CjLl1HXvTAEL1oiIKJ8UZPCO\nxr1VjRw2JyKifFKQUSvWIXgz8yYiovxRkME7OfOW2B6ViIjySEFGrWjc2/mGmTcRUd96+eW/H/Bz\n3nvvHTQ07M/C0fR/hRm8Y0mZN+e8iYj6zIFuCWp7/vk1BRu8C3KpWMdhc2beRER9xb0l6PnnX4gV\nK36IlpYWaJqGG264GWPHjsP//u/v8cor/4Qoipgz51hMnDgJr776Mj7//DP8z//ci8GDzd4dfbEN\n6OWXX+VsAxqLReD3F2VlG1A3Bm+w2pyIyLZ6y1/w7t6Nvfqa06sPxTljz0j7ffeWoL///W9x5JFH\n4//9v6/i888/w09+ch8efPAXePLJ/8Vzz70ESZLw3HOrMHPmURg7djy+851bnMAN9M02oOeff6Gz\nDejixVfiZz/7VVa2AXVj8AabtBAR9RcbN25AY2MD1q59AQAQjZobSZ1wwsm44YbFmDfvFMyfn35j\nj77YBrS5uTkn24C6FWTwrgz54ZNF6IYBVTMgCgzeREQAcM7YMzrNkrNNUWTceOPNmDJlquf+m276\nHrZt+y/+8Y+/4lvf+iZ+/es/pHz+wbwNqOfYe+2V8sghwyvw1IrT8fBNJ+DXN5/Q14dDRFTQ3FuC\nTpo0Bf/618sAgM8//wxPPvm/aG1txaOP/gYjRozEZZddiVCoDO3tbSm3Ej2YtwH1nLNefbU8Iksi\nBEHgfDcRUR9zbwn69a+fj507t2Px4itwzz3/g8MOm4GSkhI0NjbgyisX4dvfvhqTJ09BaWkZDjts\nBm677VZ89tlW57X6YhvQ+++/x9kGdOHChVnbBtStILcEBbj1XW/heew5nsOe4znsHTyPPZd8Druz\nDWjy66VSkHPeRERE2dbb24C6MXgTERFlQW9vA+rGCV8iIqI8w+BNRESUZxi8iYiI8gyDNxERUZ5h\n8CYiIsozDN5ERER5hsGbiIgozzB4ExER5Zm8aY9KREREJmbeREREeYbBm4iIKM8weBMREeUZBm8i\nIqI8w+BNRESUZxi8iYiI8kxB7ue9YsUKvP/++xAEAUuXLsXUqVP7+pD6tU8++QSLFy/GpZdeiosv\nvhi7du3CLbfcAk3TUFVVhR//+Mfw+XxYs2YN/vCHP0AURZx33nk499xz+/rQ+417770Xb7/9NlRV\nxTe/+U0ceuihPIcHIBwOY8mSJdi3bx+i0SgWL16MCRMm8Bx2UyQSwRlnnIHFixdj9uzZPI8H4I03\n3sD111+PcePGAQDGjx+PK664Ivfn0Cgwb7zxhnHVVVcZhmEYW7ZsMc4777w+PqL+ra2tzbj44ouN\n2267zXj88ccNwzCMJUuWGC+88IJhGIZx//33GytXrjTa2tqM+fPnG83NzUY4HDZOP/10o6GhoS8P\nvd9Yt26dccUVVxiGYRj79+83jj/+eJ7DA/T8888bv/71rw3DMIwdO3YY8+fP5znsgQceeMA455xz\njFWrVvE8HqD169cb3/rWtzz39cU5LLhh83Xr1mHu3LkAgDFjxqCpqQmtra19fFT9l8/nw29+8xtU\nV1c7973xxhs4+eSTAQAnnngi1q1bh/fffx+HHnooQqEQAoEAZsyYgXfeeaevDrtfmTlzJn7yk58A\nAEpLSxEOh3kOD9Bpp52GK6+8EgCwa9cuDBo0iOewm7Zu3YotW7bghBNOAMD/z72hL85hwQXv+vp6\nVFRUOLcrKytRV1fXh0fUv8myjEAg4LkvHA7D5/MBAAYMGIC6ujrU19ejsrLSeQzPa4IkSQgGgwCA\nZ599FscddxzPYTctWLAAN910E5YuXcpz2E333HMPlixZ4tzmeTxwW7ZswdVXX40LLrgAr7/+ep+c\nw4Kc83Yz2B22R9KdP57Xjv72t7/h2Wefxe9+9zvMnz/fuZ/nsOuefPJJfPjhh7j55ps954fnsGue\ne+45HHbYYRg2bFjK7/M8ZjZy5Ehcd911OPXUU7F9+3YsWrQImqY538/VOSy44F1dXY36+nrn9t69\ne1FVVdWHR5R/gsEgIpEIAoEA9uzZg+rq6pTn9bDDDuvDo+xfXn31VfzqV7/Cb3/7W4RCIZ7DA7Rp\n0yYMGDAAQ4YMwcSJE6FpGoqLi3kOD9DLL7+M7du34+WXX8bu3bvh8/n4t3iABg0ahNNOOw0AMHz4\ncAwcOBAbN27M+TksuGHzOXPmYO3atQCAzZs3o7q6GiUlJX18VPnl6KOPds7h//3f/+HYY4/FtGnT\nsHHjRjQ3N6OtrQ3vvPMOjjjiiD4+0v6hpaUF9957Lx5++GGUl5cD4Dk8UG+99RZ+97vfATCnvtrb\n23kOu+HBBx/EqlWr8PTTT+Pcc8/F4sWLeR4P0Jo1a/DII48AAOrq6rBv3z6cc845OT+HBbmr2H33\n3Ye33noLgiBg2bJlmDBhQl8fUr+1adMm3HPPPdi5cydkWcagQYNw3333YcmSJYhGo6ipqcFdd90F\nRVHw0ksv4ZFHHoEgCLj44otx5pln9vXh9wtPPfUUfvrTn2LUqFHOfXfffTduu+02nsMuikQi+P73\nv49du3YhEonguuuuw5QpU3DrrbfyHHbTT3/6U9TW1uKYY47heTwAra2tuOmmm9Dc3Ix4PI7rrrsO\nEydOzPk5LMjgTURElM8KbticiIgo3zF4ExER5RkGbyIiojzD4E1ERJRnGLyJiIjyTME1aSHKN/fe\ney82btyIaDSKDz74ANOnTwcAfO1rX8NXv/rVLr3Gr3/9a4wfP97pZ53KwoUL8fvf/x6SJPXGYXvs\n2bMHn332GWbPnt3rr01UiLhUjChP7NixAxdeeCH+9a9/9fWhHLA1a9Zg69atuPHGG/v6UIgOCsy8\nifLYT3/6U+zYsQNffvklbr31VkQiEdx3333w+XyIRCJYtmwZJk+ejCVLluDwww/H7Nmzcc011+CY\nY47Bhg0b0NbWhocffhiDBg3CIYccgs2bN+OXv/wlGhsbsXv3bmzbtg1HHnkkbr/9dkSjUdx6663Y\nuXMnBg8eDEmSMGfOHM8exW1tbfjud7+L5uZmqKqKE088EWeccQYefPBBGIaB8vJyXHTRRbjzzjux\nbds2tLW14YwzzsDll1+O1atX469//SsEQcCePXswevRorFixAoqi9OEZJuqfOOdNlOd27NiBxx57\nDFOmTEFjYyN+8IMf4LHHHsOiRYvw8MMPd3j81q1bcc4552DlypWYOHEiXnzxxQ6P+eCDD/DQQw/h\n2WefxerVq9HU1IQ1a9ZAVVU888wzuOOOO/D66693eN6///1vqKqKP/7xj3jyyScRDAZRW1uLs88+\nG2eeeSYuu+wyPPbYY6iursbjjz+OZ555Bs8//zw++ugjAMDGjRv///bu2CW1MIzj+NcONQQRQi3W\nYnBsjDoSBFKNOVaEo0M4REO4HGyrKQin5ob+gDBaoiVyECEipakhWkKkQKFoiERPd5DOzYxLlysX\njvw+4+F5X97tx/PyHh7S6TSHh4eUy2VP3jKI/A/qvEU8bmJiAp/PB8DQ0BC7u7u8vb3x8vLC4OBg\nW73f78c0TQACgQBPT09tNZZlYRgGhmHg9/t5fn7m5uaG6elpAIaHh7Esq23d1NQUe3t7bGxsMDc3\nx8rKCj09rT3CxcUFDw8PXF5eAlCr1bi/v3fXf4xPnZyc5O7uzp2TLCK/KbxFPO7ztbJt22xvbzMz\nM8P5+bk7zOOzrw/Svnv28l2N4zgtQfw1lKE5y/j4+JhiscjZ2RnLy8scHR211PT19bG+vs7CwkLL\n90wmg+M4fzyXiDTp2lyki1QqFUzTpNFocHp6Sq1W69jeY2NjFItFAKrVKldXV201uVyObDaLZVnY\ntk1/fz/VahWfz0e9XgeaXf3HVb3jOOzs7Ljd//X1Na+vr7y/v1MoFBgfH+/Y+UW6iTpvkS6SSCSI\nx+MEAgFWV1exbZuDg4OO7L20tEQ2myUWizE6Oko4HG7r0IPBIKlUiv39fQzDIBKJMDIyQjgcJplM\n0tvby9raGre3t8RiMRqNBvPz8+6o1FAoxObmJqVSCdM0iUQiHTm7SLfRr2Ii8iOPj48UCgWi0SiO\n47C4uMjW1pb73/m/ymQy5PN50ul0R/YT6WbqvEXkRwYGBjg5OXHnE8/OznYsuEXk76jzFhER8Rg9\nWBMREfEYhbeIiIjHKLxFREQ8RuEtIiLiMQpvERERj1F4i4iIeMwvRph4T/csGFUAAAAASUVORK5C\nYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "metadata": { + "id": "HNqUFL4deCsL", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# 4. Case study: building an RNN\n" + ] + }, + { + "metadata": { + "id": "YkC1k4HEQ7rw", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "In this exercise we build and train a model similar to the RNNColorbot model that was used in the main Eager notebook. The model is adapted for converting and training in graph mode." + ] + }, + { + "metadata": { + "id": "7nkPDl5CTCNb", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "To get started, we load the colorbot dataset. The code is identical to that used in the other exercise and its details are unimportant." + ] + }, + { + "metadata": { + "id": "A0uREmVXCQEw", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "def parse(line):\n", + " \"\"\"Parses a line from the colors dataset.\n", + " \n", + " Args:\n", + " line: A comma-separated string containing four items:\n", + " color_name, red, green, and blue, representing the name and\n", + " respectively the RGB value of the color, as an integer\n", + " between 0 and 255.\n", + "\n", + " Returns:\n", + " A tuple of three tensors (rgb, chars, length), of shapes: (batch_size, 3),\n", + " (batch_size, max_sequence_length, 256) and respectively (batch_size).\n", + " \"\"\"\n", + " items = tf.string_split([line], \",\").values\n", + " rgb = tf.string_to_number(items[1:], out_type=tf.float32) / 255.0\n", + " color_name = items[0]\n", + " chars = tf.one_hot(tf.decode_raw(color_name, tf.uint8), depth=256)\n", + " length = tf.cast(tf.shape(chars)[0], dtype=tf.int64)\n", + " return rgb, chars, length\n", + "\n", + "\n", + "def maybe_download(filename, work_directory, source_url):\n", + " \"\"\"Downloads the data from source url.\"\"\"\n", + " if not tf.gfile.Exists(work_directory):\n", + " tf.gfile.MakeDirs(work_directory)\n", + " filepath = os.path.join(work_directory, filename)\n", + " if not tf.gfile.Exists(filepath):\n", + " temp_file_name, _ = six.moves.urllib.request.urlretrieve(source_url)\n", + " tf.gfile.Copy(temp_file_name, filepath)\n", + " with tf.gfile.GFile(filepath) as f:\n", + " size = f.size()\n", + " print('Successfully downloaded', filename, size, 'bytes.')\n", + " return filepath\n", + "\n", + "\n", + "def load_dataset(data_dir, url, batch_size, training=True):\n", + " \"\"\"Loads the colors data at path into a tf.PaddedDataset.\"\"\"\n", + " path = maybe_download(os.path.basename(url), data_dir, url)\n", + " dataset = tf.data.TextLineDataset(path)\n", + " dataset = dataset.skip(1)\n", + " dataset = dataset.map(parse)\n", + " dataset = dataset.cache()\n", + " dataset = dataset.repeat()\n", + " if training:\n", + " dataset = dataset.shuffle(buffer_size=3000)\n", + " dataset = dataset.padded_batch(batch_size, padded_shapes=([None], [None, None], []))\n", + " return dataset\n", + "\n", + "\n", + "train_url = \"https://raw.githubusercontent.com/random-forests/tensorflow-workshop/master/extras/colorbot/data/train.csv\"\n", + "test_url = \"https://raw.githubusercontent.com/random-forests/tensorflow-workshop/master/extras/colorbot/data/test.csv\"\n", + "data_dir = \"tmp/rnn/data\"" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "waZ89t3DTUla", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Next, we set up the RNNColobot model, which is very similar to the one we used in the main exercise.\n", + "\n", + "Autograph doesn't fully support classes yet (but it will soon!), so we'll write the model using simple functions." + ] + }, + { + "metadata": { + "id": "9v8AJouiC44V", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "def model_components():\n", + " lower_cell = tf.contrib.rnn.LSTMBlockCell(256)\n", + " lower_cell.build(tf.TensorShape((None, 256)))\n", + " upper_cell = tf.contrib.rnn.LSTMBlockCell(128)\n", + " upper_cell.build(tf.TensorShape((None, 256)))\n", + " relu_layer = tf.layers.Dense(3, activation=tf.nn.relu)\n", + " relu_layer.build(tf.TensorShape((None, 128)))\n", + " return lower_cell, upper_cell, relu_layer\n", + "\n", + "\n", + "def rnn_layer(chars, cell, batch_size, training):\n", + " \"\"\"A simple RNN layer.\n", + " \n", + " Args:\n", + " chars: A Tensor of shape (max_sequence_length, batch_size, input_size)\n", + " cell: An object of type tf.contrib.rnn.LSTMBlockCell\n", + " batch_size: Int, the batch size to use\n", + " training: Boolean, whether the layer is used for training\n", + "\n", + " Returns:\n", + " A Tensor of shape (max_sequence_length, batch_size, output_size).\n", + " \"\"\"\n", + " hidden_outputs = []\n", + " autograph.utils.set_element_type(hidden_outputs, tf.float32)\n", + " state, output = cell.zero_state(batch_size, tf.float32)\n", + " n = tf.shape(chars)[0]\n", + " i = 0\n", + " while i < n:\n", + " ch = chars[i]\n", + " cell_output, (state, output) = cell.call(ch, (state, output))\n", + " hidden_outputs.append(cell_output)\n", + " i += 1\n", + " hidden_outputs = hidden_outputs.stack()\n", + " if training:\n", + " hidden_outputs = tf.nn.dropout(hidden_outputs, 0.5)\n", + " return hidden_outputs\n", + "\n", + "\n", + "def model(inputs, lower_cell, upper_cell, relu_layer, batch_size, training):\n", + " \"\"\"RNNColorbot model.\n", + " \n", + " The model consists of two RNN layers (made by lower_cell and upper_cell),\n", + " followed by a fully connected layer with ReLU activation.\n", + " \n", + " Args:\n", + " inputs: A tuple (chars, length)\n", + " lower_cell: An object of type tf.contrib.rnn.LSTMBlockCell\n", + " upper_cell: An object of type tf.contrib.rnn.LSTMBlockCell\n", + " relu_layer: An object of type tf.layers.Dense\n", + " batch_size: Int, the batch size to use\n", + " training: Boolean, whether the layer is used for training\n", + " \n", + " Returns:\n", + " A Tensor of shape (batch_size, 3) - the model predictions.\n", + " \"\"\"\n", + " (chars, length) = inputs\n", + " chars_time_major = tf.transpose(chars, [1, 0, 2])\n", + " chars_time_major.set_shape((None, batch_size, 256))\n", + "\n", + " hidden_outputs = rnn_layer(chars_time_major, lower_cell, batch_size, training)\n", + " final_outputs = rnn_layer(hidden_outputs, upper_cell, batch_size, training)\n", + "\n", + " # Grab just the end-of-sequence from each output.\n", + " indices = tf.stack([length - 1, range(batch_size)], axis=1)\n", + " sequence_ends = tf.gather_nd(final_outputs, indices)\n", + " return relu_layer(sequence_ends)\n", + "\n", + "def loss_fn(labels, predictions):\n", + " return tf.reduce_mean((predictions - labels) ** 2)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "JjK4gXFvFsf4", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "The train and test functions are also similar to the ones used in the Eager notebook. Since the network requires a fixed batch size, we'll train in a single shot, rather than by epoch." + ] + }, + { + "metadata": { + "id": "ZWQMExk0S6X6", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "def train(optimizer, train_data, lower_cell, upper_cell, relu_layer, batch_size, num_steps):\n", + " iterator = train_data.make_one_shot_iterator()\n", + " step = 0\n", + " while step < num_steps:\n", + " labels, chars, sequence_length = iterator.get_next()\n", + " predictions = model((chars, sequence_length), lower_cell, upper_cell, relu_layer, batch_size, training=True)\n", + " loss = loss_fn(labels, predictions)\n", + " optimizer.minimize(loss)\n", + " if step % (num_steps // 10) == 0:\n", + " print('Step', step, 'train loss', loss)\n", + " step += 1\n", + " return step\n", + "\n", + "\n", + "def test(eval_data, lower_cell, upper_cell, relu_layer, batch_size, num_steps):\n", + " total_loss = 0.0\n", + " iterator = eval_data.make_one_shot_iterator()\n", + " step = 0\n", + " while step < num_steps:\n", + " labels, chars, sequence_length = iterator.get_next()\n", + " predictions = model((chars, sequence_length), lower_cell, upper_cell, relu_layer, batch_size, training=False)\n", + " total_loss += loss_fn(labels, predictions)\n", + " step += 1\n", + " print('Test loss', total_loss)\n", + " return total_loss\n", + "\n", + "\n", + "def train_model(train_data, eval_data, batch_size, lower_cell, upper_cell, relu_layer, train_steps):\n", + " optimizer = tf.train.AdamOptimizer(learning_rate=0.01)\n", + "\n", + " train(optimizer, train_data, lower_cell, upper_cell, relu_layer, batch_size, num_steps=tf.constant(train_steps))\n", + " test(eval_data, lower_cell, upper_cell, relu_layer, 50, num_steps=tf.constant(2))\n", + "\n", + " print('Colorbot is ready to generate colors!\\n\\n')\n", + " \n", + " # In graph mode, every op needs to be a dependent of another op.\n", + " # Here, we create a no_op that will drive the execution of all other code in\n", + " # this function. Autograph will add the necessary control dependencies.\n", + " return tf.no_op()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "iopcs5hXG2od", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Finally, we add code to run inference on a single input, which we'll read from the input.\n", + "\n", + "Note the `do_not_convert` annotation that lets us disable conversion for certain functions and run them as a `py_func` instead, so you can still call them from compiled code." + ] + }, + { + "metadata": { + "id": "DyU0wnnAFEYj", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "@autograph.do_not_convert(run_as=autograph.RunMode.PY_FUNC)\n", + "def draw_prediction(color_name, pred):\n", + " pred = pred * 255\n", + " pred = pred.astype(np.uint8)\n", + " plt.axis('off')\n", + " plt.imshow(pred)\n", + " plt.title(color_name)\n", + " plt.show()\n", + "\n", + "\n", + "def inference(color_name, lower_cell, upper_cell, relu_layer):\n", + " _, chars, sequence_length = parse(color_name)\n", + " chars = tf.expand_dims(chars, 0)\n", + " sequence_length = tf.expand_dims(sequence_length, 0)\n", + " pred = model((chars, sequence_length), lower_cell, upper_cell, relu_layer, 1, training=False)\n", + " pred = tf.minimum(pred, 1.0)\n", + " pred = tf.expand_dims(pred, 0)\n", + " draw_prediction(color_name, pred)\n", + " # Create an op that will drive the entire function.\n", + " return tf.no_op()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "Nt0Kv5OCHip0", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Finally, we put everything together.\n", + "\n", + "Note that the entire training and testing code is all compiled into a single op (`tf_train_model`) that you only execute once! We also still use a `sess.run` loop for the inference part, because that requires keyboard input." + ] + }, + { + "metadata": { + "id": "-GmWa0GtYWdh", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "output_extras": [ + {}, + {}, + {}, + {}, + {}, + {}, + {}, + {}, + {}, + {}, + {}, + {}, + {}, + {}, + {}, + {}, + {}, + {}, + {}, + {}, + {}, + {}, + {} + ], + "base_uri": "https://localhost:8080/", + "height": 668 + }, + "outputId": "61f4af1d-c81e-44db-9079-1a7b8ed8ce58", + "executionInfo": { + "status": "ok", + "timestamp": 1522345877153, + "user_tz": 240, + "elapsed": 75500, + "user": { + "displayName": "Dan Moldovan", + "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg", + "userId": "112023154726779574577" + } + } + }, + "cell_type": "code", + "source": [ + "def run_input_loop(sess, inference_ops, color_name_placeholder):\n", + " \"\"\"Helper function that reads from input and calls the inference ops in a loop.\"\"\"\n", + "\n", + " tb = widgets.TabBar([\"RNN Colorbot\"])\n", + " while True:\n", + " with tb.output_to(0):\n", + " try:\n", + " color_name = six.moves.input(\"Give me a color name (or press 'enter' to exit): \")\n", + " except (EOFError, KeyboardInterrupt):\n", + " break\n", + " if not color_name:\n", + " break\n", + " with tb.output_to(0):\n", + " tb.clear_tab()\n", + " sess.run(inference_ops, {color_name_placeholder: color_name})\n", + " plt.show()\n", + "\n", + "with tf.Graph().as_default():\n", + " # Read the data.\n", + " batch_size = 64\n", + " train_data = load_dataset(data_dir, train_url, batch_size)\n", + " eval_data = load_dataset(data_dir, test_url, 50, training=False)\n", + " \n", + " # Create the model components.\n", + " lower_cell, upper_cell, relu_layer = model_components()\n", + " # Create the helper placeholder for inference.\n", + " color_name_placeholder = tf.placeholder(tf.string, shape=())\n", + " \n", + " # Compile the train / test code.\n", + " tf_train_model = autograph.to_graph(train_model)\n", + " train_model_ops = tf_train_model(\n", + " train_data, eval_data, batch_size, lower_cell, upper_cell, relu_layer, train_steps=100)\n", + " \n", + " # Compile the inference code.\n", + " tf_inference = autograph.to_graph(inference)\n", + " inference_ops = tf_inference(color_name_placeholder, lower_cell, upper_cell, relu_layer)\n", + " \n", + " with tf.Session() as sess:\n", + " sess.run(tf.global_variables_initializer())\n", + " \n", + " # Run training and testing.\n", + " sess.run(train_model_ops)\n", + " \n", + " # Run the inference loop.\n", + " run_input_loop(sess, inference_ops, color_name_placeholder)" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "('Successfully downloaded', 'train.csv', 28010L, 'bytes.')\n", + "('Successfully downloaded', 'test.csv', 2414L, 'bytes.')\n", + "Step 0 train loss 0.37890616\n", + "Step 10 train loss 0.18515904\n", + "Step 20 train loss 0.0892782\n", + "Step 30 train loss 0.07883155\n", + "Step 40 train loss 0.08585831\n", + "Step 50 train loss 0.09302989\n", + "Step 60 train loss 0.089012615\n", + "Step 70 train loss 0.07275697\n", + "Step 80 train loss 0.06644974\n", + "Step 90 train loss 0.0854013\n", + "Test loss 0.13216865Colorbot is ready to generate colors!\n", + "\n", + "\n", + "\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "" + ] + }, + "metadata": { + "tags": [ + "outputarea_id1" + ] + } + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "" + ] + }, + "metadata": { + "tags": [ + "outputarea_id1" + ] + } + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "
" + ] + }, + "metadata": { + "tags": [ + "outputarea_id1" + ] + } + }, + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "window[\"b102d936-3379-11e8-ac70-0242ac110002\"] = colab_lib.createTabBar({\"contentBorder\": [\"0px\"], \"borderColor\": [\"#a7a7a7\"], \"tabNames\": [\"RNN Colorbot\"], \"initialSelection\": 0, \"location\": \"top\", \"contentHeight\": [\"initial\"], \"elementId\": \"id1\"});\n", + "//# sourceURL=js_e223a56194" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [ + "outputarea_id1" + ] + } + }, + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "window[\"b103532a-3379-11e8-ac70-0242ac110002\"] = window[\"id1\"].setSelectedTabIndex(0);\n", + "//# sourceURL=js_b8c6a821fb" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [ + "outputarea_id1" + ] + } + }, + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "window[\"b105b28c-3379-11e8-ac70-0242ac110002\"] = google.colab.output.getActiveOutputArea();\n", + "//# sourceURL=js_44805e254b" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + } + }, + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "window[\"b106197a-3379-11e8-ac70-0242ac110002\"] = document.querySelector(\"#id1_content_0\");\n", + "//# sourceURL=js_a63d3c6c47" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + } + }, + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "window[\"b1069f44-3379-11e8-ac70-0242ac110002\"] = google.colab.output.setActiveOutputArea(window[\"b106197a-3379-11e8-ac70-0242ac110002\"]);\n", + "//# sourceURL=js_7e203b8bce" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + } + }, + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "window[\"b1070f38-3379-11e8-ac70-0242ac110002\"] = window[\"id1\"].setSelectedTabIndex(0);\n", + "//# sourceURL=js_d53293d4a7" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + } + }, + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "window[\"c6d90d5c-3379-11e8-ac70-0242ac110002\"] = google.colab.output.setActiveOutputArea(window[\"b105b28c-3379-11e8-ac70-0242ac110002\"]);\n", + "//# sourceURL=js_3000dc2c05" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + } + }, + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "window[\"c6da872c-3379-11e8-ac70-0242ac110002\"] = google.colab.output.getActiveOutputArea();\n", + "//# sourceURL=js_4136f669a3" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + } + }, + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "window[\"c6dac868-3379-11e8-ac70-0242ac110002\"] = document.querySelector(\"#id1_content_0\");\n", + "//# sourceURL=js_2f70dd9aee" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + } + }, + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "window[\"c6db07d8-3379-11e8-ac70-0242ac110002\"] = google.colab.output.setActiveOutputArea(window[\"c6dac868-3379-11e8-ac70-0242ac110002\"]);\n", + "//# sourceURL=js_7226726048" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + } + }, + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "window[\"c6dcc6fe-3379-11e8-ac70-0242ac110002\"] = window[\"id1\"].setSelectedTabIndex(0);\n", + "//# sourceURL=js_72e7709865" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + } + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVQAAAFZCAYAAADHDNdrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAB9JJREFUeJzt3E1Lle0ax+HTF4jeEAyMBhE0DawI\nwsCH0AIlaGBWNJBo0CDoA0TQhmDXuKAGDioiCA2KlEAlnl05FD9Co8BeaGCQoBDa2jPZsXt4Bvu/\n0+o4Rmvd1zW4rsmP84bFamo0Go0C4H/WvNYHAPhVCCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKDy\nUxgeHq5Dhw7V4OBgPXz4sHp7e+vWrVt15cqVOnnyZN2/f78ajUbdvn27+vr6qqenp65du1YrKytV\nVfXhw4e6cOFC9fX1VV9fX01PT1dV1dzcXHV3d9eDBw/q+PHj9ccff9TExMRaXpWfWOtaHwD+zuvX\nr+vOnTs1MTFRbW1tdf78+dW16enpGh8fr/b29hobG6upqal6/Phxbdy4sS5evFgjIyM1NDRUly5d\nqv3799fw8HC9efOmTp8+XVNTU1VV9enTp2pubq5nz57V5ORk3bhxo44dO7ZW1+UnZkJl3Zudna2D\nBw9WR0dHbdiwoQYHB1fX9u7dW+3t7VVV9fLlyxocHKytW7dWa2trnTp1qp4/f16Li4s1MzNT586d\nq6qqXbt21YEDB1an1OXl5Tpx4kRVVe3Zs6fevXv3Yy/IL8OEyrr3+fPnamtrW/2+ffv21c//+Xxh\nYaHu3r1bjx49qqqqlZWVam9vr4WFhWo0GnXmzJnVvYuLi9XV1VVVVS0tLbVp06aqqmpubq6vX7/+\nX+/Dr0tQWfe2bNlSi4uLq98/fvz43X0dHR3V29tbQ0ND3zxfXl6ulpaWevLkSW3evPmbtbm5ufyB\n+W155Wfd6+zsrJmZmZqfn68vX77U2NjYd/cdOXKkxsfHa2lpqaqqRkdH6+nTp9Xa2lqHDx+u0dHR\nqqpaWlqqy5cv1/v373/YHfg9CCrrXmdnZw0MDNTAwECdPXu2enp6vrvv6NGj1dPTUwMDA9Xf318v\nXryo7u7uqqq6evVqzc7OVn9/fw0MDNTOnTtrx44dP/Ia/Aaa/B8qP4NGo1FNTU1VVfXq1au6efPm\nX06qsFZMqKx78/Pz1dXVVW/fvq1Go1GTk5O1b9++tT4W/BcTKj+FkZGRunfvXjU1NdXu3bvr+vXr\ntW3btrU+FnxDUAFCvPIDhAgqQMi6+WH/kX8eXesjAPytf/3jz79cM6EChAgqQIigAoQIKkCIoAKE\nCCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQI\nKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgq\nQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpA\niKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCI\noAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIig\nAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAC\nhAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKE\nCCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQI\nKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgq\nQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpA\niKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCI\noAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIig\nAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAC\nhAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKE\nCCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQI\nKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgq\nQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpA\niKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkBI\nU6PRaKz1IQB+BSZUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBB\nBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAkH8D1Aj8lNhhe7QAAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1", + "user_output" + ] + } + }, + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "window[\"c70592aa-3379-11e8-ac70-0242ac110002\"] = google.colab.output.setActiveOutputArea(window[\"c6da872c-3379-11e8-ac70-0242ac110002\"]);\n", + "//# sourceURL=js_25c3aaf79a" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + } + }, + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "window[\"c70842c0-3379-11e8-ac70-0242ac110002\"] = google.colab.output.getActiveOutputArea();\n", + "//# sourceURL=js_984c56b816" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + } + }, + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "window[\"c708dec4-3379-11e8-ac70-0242ac110002\"] = document.querySelector(\"#id1_content_0\");\n", + "//# sourceURL=js_e0451a1217" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + } + }, + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "window[\"c7092726-3379-11e8-ac70-0242ac110002\"] = google.colab.output.setActiveOutputArea(window[\"c708dec4-3379-11e8-ac70-0242ac110002\"]);\n", + "//# sourceURL=js_7aa23d7385" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + } + }, + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "window[\"c7099044-3379-11e8-ac70-0242ac110002\"] = window[\"id1\"].setSelectedTabIndex(0);\n", + "//# sourceURL=js_5722756ddb" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + } + }, + { + "output_type": "stream", + "text": [ + "Give me a color name (or press 'enter' to exit): \n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "window[\"c7baac12-3379-11e8-ac70-0242ac110002\"] = google.colab.output.setActiveOutputArea(window[\"c70842c0-3379-11e8-ac70-0242ac110002\"]);\n", + "//# sourceURL=js_cdd622e58f" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + } + } + ] + }, + { + "metadata": { + "id": "AHJ2c47U-A5W", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Where do we go next?\n", + "\n", + "Autograph is available in tensorflow.contrib, but it's still in its early stages. We're excited about the possibilities it brings — write your machine learning code in the flexible Eager style, but still enjoy all the benefits that come with running in graph mode. A beta version will be available soon -- stay tuned!" + ] + } + ] +} diff --git a/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py b/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py index 985177e897..d193a8459d 100644 --- a/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py +++ b/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py @@ -44,14 +44,14 @@ def expectation_importance_sampler(f, n=None, seed=None, name='expectation_importance_sampler'): - r"""Monte Carlo estimate of `E_p[f(Z)] = E_q[f(Z) p(Z) / q(Z)]`. + r"""Monte Carlo estimate of `\\(E_p[f(Z)] = E_q[f(Z) p(Z) / q(Z)]\\)`. - With `p(z) := exp{log_p(z)}`, this `Op` returns + With `\\(p(z) := exp^{log_p(z)}\\)`, this `Op` returns ``` - n^{-1} sum_{i=1}^n [ f(z_i) p(z_i) / q(z_i) ], z_i ~ q, - \approx E_q[ f(Z) p(Z) / q(Z) ] - = E_p[f(Z)] + \\(n^{-1} sum_{i=1}^n [ f(z_i) p(z_i) / q(z_i) ], z_i ~ q,\\) + \\(\approx E_q[ f(Z) p(Z) / q(Z) ]\\) + \\(= E_p[f(Z)]\\) ``` This integral is done in log-space with max-subtraction to better handle the @@ -95,9 +95,9 @@ def expectation_importance_sampler(f, log_values = log_f_z + log_p_z - q_log_prob_z return _logspace_mean(log_values) - # With f_plus(z) = max(0, f(z)), f_minus(z) = max(0, -f(z)), - # E_p[f(Z)] = E_p[f_plus(Z)] - E_p[f_minus(Z)] - # = E_p[f_plus(Z) + 1] - E_p[f_minus(Z) + 1] + # With \\(f_{plus}(z) = max(0, f(z)), f_{minus}(z) = max(0, -f(z))\\), + # \\(E_p[f(Z)] = E_p[f_{plus}(Z)] - E_p[f_{minus}(Z)]\\) + # \\( = E_p[f_{plus}(Z) + 1] - E_p[f_{minus}(Z) + 1]\\) # Without incurring bias, 1 is added to each to prevent zeros in logspace. # The logarithm is approximately linear around 1 + epsilon, so this is good # for small values of 'z' as well. @@ -121,13 +121,13 @@ def expectation_importance_sampler_logspace( name='expectation_importance_sampler_logspace'): r"""Importance sampling with a positive function, in log-space. - With `p(z) := exp{log_p(z)}`, and `f(z) = exp{log_f(z)}`, this `Op` - returns + With `\\(p(z) := exp^{log_p(z)}\\)`, and `\\(f(z) = exp{log_f(z)}\\)`, + this `Op` returns ``` - Log[ n^{-1} sum_{i=1}^n [ f(z_i) p(z_i) / q(z_i) ] ], z_i ~ q, - \approx Log[ E_q[ f(Z) p(Z) / q(Z) ] ] - = Log[E_p[f(Z)]] + \\(Log[ n^{-1} sum_{i=1}^n [ f(z_i) p(z_i) / q(z_i) ] ], z_i ~ q,\\) + \\(\approx Log[ E_q[ f(Z) p(Z) / q(Z) ] ]\\) + \\(= Log[E_p[f(Z)]]\\) ``` This integral is done in log-space with max-subtraction to better handle the @@ -196,12 +196,12 @@ def _logspace_mean(log_values): def expectation(f, samples, log_prob=None, use_reparametrization=True, axis=0, keep_dims=False, name=None): - """Computes the Monte-Carlo approximation of `E_p[f(X)]`. + """Computes the Monte-Carlo approximation of `\\(E_p[f(X)]\\)`. This function computes the Monte-Carlo approximation of an expectation, i.e., ```none - E_p[f(X)] approx= m**-1 sum_i^m f(x_j), x_j ~iid p(X) + \\(E_p[f(X)] \approx= m^{-1} sum_i^m f(x_j), x_j\ ~iid\ p(X)\\) ``` where: @@ -216,8 +216,8 @@ def expectation(f, samples, log_prob=None, use_reparametrization=True, parameterless distribution (e.g., `Normal(Y; m, s) <=> Y = sX + m, X ~ Normal(0,1)`), we can swap gradient and expectation, i.e., - `grad[ Avg{ s_i : i=1...n } ] = Avg{ grad[s_i] : i=1...n }` where - `S_n = Avg{s_i}` and `s_i = f(x_i), x_i ~ p`. + `grad[ Avg{ \\(s_i : i=1...n\\) } ] = Avg{ grad[\\(s_i\\)] : i=1...n }` where + `S_n = Avg{\\(s_i\\)}` and `\\(s_i = f(x_i), x_i ~ p\\)`. However, if p is not reparameterized, TensorFlow's gradient will be incorrect since the chain-rule stops at samples of non-reparameterized distributions. @@ -296,7 +296,8 @@ def expectation(f, samples, log_prob=None, use_reparametrization=True, Args: f: Python callable which can return `f(samples)`. samples: `Tensor` of samples used to form the Monte-Carlo approximation of - `E_p[f(X)]`. A batch of samples should be indexed by `axis` dimensions. + `\\(E_p[f(X)]\\)`. A batch of samples should be indexed by `axis` + dimensions. log_prob: Python callable which can return `log_prob(samples)`. Must correspond to the natural-logarithm of the pdf/pmf of each sample. Only required/used if `use_reparametrization=False`. @@ -316,7 +317,7 @@ def expectation(f, samples, log_prob=None, use_reparametrization=True, Returns: approx_expectation: `Tensor` corresponding to the Monte-Carlo approximation - of `E_p[f(X)]`. + of `\\(E_p[f(X)]\\)`. Raises: ValueError: if `f` is not a Python `callable`. diff --git a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py index a520a06bd7..5a2771229d 100644 --- a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py +++ b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py @@ -75,7 +75,7 @@ class TPUClusterResolver(ClusterResolver): zone=None, project=None, job_name='worker', - coordinator_name='coordinator', + coordinator_name=None, coordinator_address=None, credentials='default', service=None): diff --git a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py index cfddca1063..dff7a03b68 100644 --- a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py +++ b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py @@ -117,7 +117,8 @@ class TPUClusterResolverTest(test.TestCase): zone=None, tpu=['test-tpu-1'], credentials=None, - service=self.mock_service_client(tpu_map=tpu_map)) + service=self.mock_service_client(tpu_map=tpu_map), + coordinator_name='coordinator') actual_cluster_spec = tpu_cluster_resolver.cluster_spec() expected_proto = """ @@ -170,6 +171,7 @@ class TPUClusterResolverTest(test.TestCase): project='test-project', zone='us-central1-c', tpu=['test-tpu-1'], + coordinator_name='coordinator', coordinator_address='10.128.1.5:10203', credentials=None, service=self.mock_service_client(tpu_map=tpu_map)) @@ -196,6 +198,7 @@ class TPUClusterResolverTest(test.TestCase): project='test-project', zone='us-central1-c', tpu='test-tpu-1', + coordinator_name='coordinator', coordinator_address='10.128.1.5:10203', credentials=None, service=self.mock_service_client(tpu_map=tpu_map)) @@ -239,7 +242,8 @@ class TPUClusterResolverTest(test.TestCase): tpu_cluster_resolver = TPUClusterResolver( tpu='test-tpu-1', credentials=None, - service=self.mock_service_client(tpu_map=tpu_map)) + service=self.mock_service_client(tpu_map=tpu_map), + coordinator_name='coordinator') actual_cluster_spec = tpu_cluster_resolver.cluster_spec() expected_proto = """ diff --git a/tensorflow/contrib/cmake/python_modules.txt b/tensorflow/contrib/cmake/python_modules.txt index 340be61971..de84af866b 100644 --- a/tensorflow/contrib/cmake/python_modules.txt +++ b/tensorflow/contrib/cmake/python_modules.txt @@ -337,6 +337,7 @@ tensorflow/contrib/nccl/kernels tensorflow/contrib/nccl/ops tensorflow/contrib/nccl/python tensorflow/contrib/nccl/python/ops +tensorflow/contrib/nearest_neighbor tensorflow/contrib/nearest_neighbor/kernels tensorflow/contrib/nearest_neighbor/ops tensorflow/contrib/nearest_neighbor/python diff --git a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py index 1dd490b386..c28c3a18e4 100644 --- a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py +++ b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py @@ -88,19 +88,23 @@ class CudnnCompatibleGRUCell(rnn_cell_impl.GRUCell): Cudnn compatible GRU (from Cudnn library user guide): ```python - r_t = sigma(x_t * W_r + h_t-1 * R_h + b_Wr + b_Rr) # reset gate - u_t = sigma(x_t * W_u + h_t-1 * R_u + b_Wu + b_Ru) # update gate - h'_t = tanh(x_t * W_h + r_t .* (h_t-1 * R_h + b_Rh) + b_Wh) # new memory gate - h_t = (1 - u_t) .* h'_t + u_t .* h_t-1 + # reset gate + $$r_t = \sigma(x_t * W_r + h_t-1 * R_h + b_{Wr} + b_{Rr})$$ + # update gate + $$u_t = \sigma(x_t * W_u + h_t-1 * R_u + b_{Wu} + b_{Ru})$$ + # new memory gate + $$h'_t = tanh(x_t * W_h + r_t .* (h_t-1 * R_h + b_{Rh}) + b_{Wh})$$ + $$h_t = (1 - u_t) .* h'_t + u_t .* h_t-1$$ ``` Other GRU (see @{tf.nn.rnn_cell.GRUCell} and @{tf.contrib.rnn.GRUBlockCell}): ```python - h'_t = tanh(x_t * W_h + (r_t .* h_t-1) * R_h + b_Wh) # new memory gate + # new memory gate + \\(h'_t = tanh(x_t * W_h + (r_t .* h_t-1) * R_h + b_{Wh})\\) ``` which is not equivalent to Cudnn GRU: in addition to the extra bias term b_Rh, ```python - r .* (h * R) != (r .* h) * R + \\(r .* (h * R) != (r .* h) * R\\) ``` """ diff --git a/tensorflow/contrib/data/python/kernel_tests/sequence_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/sequence_dataset_op_test.py index 36ddf30042..b044ff1775 100644 --- a/tensorflow/contrib/data/python/kernel_tests/sequence_dataset_op_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/sequence_dataset_op_test.py @@ -100,6 +100,12 @@ class SequenceDatasetSerializationTest( # Test repeat empty dataset self.run_core_tests(lambda: self._build_repeat_dataset(-1, 0), None, 0) + def testInvalidRepeat(self): + with self.assertRaisesRegexp( + ValueError, 'Shape must be rank 0 but is rank 1'): + self.run_core_tests(lambda: self._build_repeat_dataset([1, 2], 0), + None, 0) + if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/data/python/ops/resampling.py b/tensorflow/contrib/data/python/ops/resampling.py index a182dddd38..b465397437 100644 --- a/tensorflow/contrib/data/python/ops/resampling.py +++ b/tensorflow/contrib/data/python/ops/resampling.py @@ -110,6 +110,7 @@ def rejection_resample(class_func, target_dist, initial_dist=None, seed=None): .filter(lambda _1, p, _2: random_ops.random_uniform([], seed=seed) < p)) return filtered_ds.map(lambda class_value, _, data: (class_value, data)) + return _apply_fn diff --git a/tensorflow/contrib/distribute/python/cross_tower_ops.py b/tensorflow/contrib/distribute/python/cross_tower_ops.py index 68f202ea62..bbe5e877d5 100644 --- a/tensorflow/contrib/distribute/python/cross_tower_ops.py +++ b/tensorflow/contrib/distribute/python/cross_tower_ops.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Classes for different algortihms of reduction and broadcasting.""" +"""Classes for different algorithms of reduction and broadcasting.""" from __future__ import absolute_import from __future__ import division @@ -155,7 +155,7 @@ class CrossTowerOps(object): Args: method_string: either 'sum' or 'mean' specifying the reduction method. value_destination_pairs: a list or a tuple of tuples of PerDevice objects - and destinations. If a destionation is None, then the destinations + and destinations. If a destination is None, then the destinations are set to match the devices of the input PerDevice object. Returns: diff --git a/tensorflow/contrib/distribute/python/cross_tower_utils.py b/tensorflow/contrib/distribute/python/cross_tower_utils.py index 0dc6b8db6b..fc04e2195f 100644 --- a/tensorflow/contrib/distribute/python/cross_tower_utils.py +++ b/tensorflow/contrib/distribute/python/cross_tower_utils.py @@ -316,7 +316,7 @@ def unpack_small_tensors(tower_grads, packing): it made to tower_grads. Returns: - new_tower_grads: identical to tower_grads except that concatentations + new_tower_grads: identical to tower_grads except that concatenations of small tensors have been split apart and returned to their original positions, paired with their original variables. """ diff --git a/tensorflow/contrib/distribute/python/shared_variable_creator.py b/tensorflow/contrib/distribute/python/shared_variable_creator.py index aca9c7af05..a7083e279f 100644 --- a/tensorflow/contrib/distribute/python/shared_variable_creator.py +++ b/tensorflow/contrib/distribute/python/shared_variable_creator.py @@ -46,7 +46,7 @@ def make_fn(shared_variable_store, device_id): error. Additionally, we de-uniquify variable names before checking for matches. This helps re-use variables which are intended to be the same but have different - names due to variable uniquificaton happening upstream. Since this might + names due to variable uniquification happening upstream. Since this might mean we may have multiple variables with the same canonical name, we store them in a list per canonical name and return them in the same order as well. diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/kumaraswamy_bijector_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/kumaraswamy_bijector_test.py index ad11d9f248..074b5f275d 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/kumaraswamy_bijector_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/kumaraswamy_bijector_test.py @@ -69,7 +69,7 @@ class KumaraswamyBijectorTest(test.TestCase): bijector = Kumaraswamy( concentration1=concentration1, concentration0=concentration0, validate_args=True) - # Omitting the endpoints 0 and 1, since idlj will be inifinity at these + # Omitting the endpoints 0 and 1, since idlj will be infinity at these # endpoints. y = np.linspace(.01, 0.99, num=10).astype(np.float32) x = 1 - (1 - y ** concentration1) ** concentration0 diff --git a/tensorflow/contrib/distributions/python/ops/estimator.py b/tensorflow/contrib/distributions/python/ops/estimator.py index 6b53338c45..98edd337fe 100644 --- a/tensorflow/contrib/distributions/python/ops/estimator.py +++ b/tensorflow/contrib/distributions/python/ops/estimator.py @@ -75,7 +75,7 @@ def estimator_head_distribution_regression(make_distribution_fn, class _DistributionRegressionHead(_RegressionHead): - """Creates a _RegressionHead instance from an arbitray `Distribution`.""" + """Creates a _RegressionHead instance from an arbitrary `Distribution`.""" def __init__(self, make_distribution_fn, diff --git a/tensorflow/contrib/distributions/python/ops/independent.py b/tensorflow/contrib/distributions/python/ops/independent.py index 7dcb3e3ac4..b1bacb91b0 100644 --- a/tensorflow/contrib/distributions/python/ops/independent.py +++ b/tensorflow/contrib/distributions/python/ops/independent.py @@ -36,7 +36,7 @@ class Independent(distribution_lib.Distribution): This distribution is useful for regarding a collection of independent, non-identical distributions as a single random variable. For example, the - `Indpendent` distribution composed of a collection of `Bernoulli` + `Independent` distribution composed of a collection of `Bernoulli` distributions might define a distribution over an image (where each `Bernoulli` is a distribution over each pixel). diff --git a/tensorflow/contrib/distributions/python/ops/onehot_categorical.py b/tensorflow/contrib/distributions/python/ops/onehot_categorical.py index 46c2cc8b7a..e3e40b2e9c 100644 --- a/tensorflow/contrib/distributions/python/ops/onehot_categorical.py +++ b/tensorflow/contrib/distributions/python/ops/onehot_categorical.py @@ -52,7 +52,7 @@ class OneHotCategorical(distribution.Distribution): #### Examples - Creates a 3-class distiribution, with the 2nd class, the most likely to be + Creates a 3-class distribution, with the 2nd class, the most likely to be drawn from. ```python @@ -60,7 +60,7 @@ class OneHotCategorical(distribution.Distribution): dist = OneHotCategorical(probs=p) ``` - Creates a 3-class distiribution, with the 2nd class the most likely to be + Creates a 3-class distribution, with the 2nd class the most likely to be drawn from, using logits. ```python diff --git a/tensorflow/contrib/distributions/python/ops/relaxed_bernoulli.py b/tensorflow/contrib/distributions/python/ops/relaxed_bernoulli.py index b525809015..e454a53c62 100644 --- a/tensorflow/contrib/distributions/python/ops/relaxed_bernoulli.py +++ b/tensorflow/contrib/distributions/python/ops/relaxed_bernoulli.py @@ -35,10 +35,10 @@ class RelaxedBernoulli(transformed_distribution.TransformedDistribution): The RelaxedBernoulli is a distribution over the unit interval (0,1), which continuously approximates a Bernoulli. The degree of approximation is - controlled by a temperature: as the temperaturegoes to 0 the RelaxedBernoulli - becomes discrete with a distribution described by the `logits` or `probs` - parameters, as the temperature goes to infinity the RelaxedBernoulli - becomes the constant distribution that is identically 0.5. + controlled by a temperature: as the temperature goes to 0 the + RelaxedBernoulli becomes discrete with a distribution described by the + `logits` or `probs` parameters, as the temperature goes to infinity the + RelaxedBernoulli becomes the constant distribution that is identically 0.5. The RelaxedBernoulli distribution is a reparameterized continuous distribution that is the binary special case of the RelaxedOneHotCategorical diff --git a/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py b/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py index ff33f327c7..f56ba07816 100644 --- a/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py +++ b/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py @@ -303,7 +303,7 @@ class RelaxedOneHotCategorical( The RelaxedOneHotCategorical is a distribution over random probability vectors, vectors of positive real values that sum to one, which continuously approximates a OneHotCategorical. The degree of approximation is controlled by - a temperature: as the temperaturegoes to 0 the RelaxedOneHotCategorical + a temperature: as the temperature goes to 0 the RelaxedOneHotCategorical becomes discrete with a distribution described by the `logits` or `probs` parameters, as the temperature goes to infinity the RelaxedOneHotCategorical becomes the constant distribution that is identically the constant vector of diff --git a/tensorflow/contrib/distributions/python/ops/vector_student_t.py b/tensorflow/contrib/distributions/python/ops/vector_student_t.py index 8c67647a61..887981d64e 100644 --- a/tensorflow/contrib/distributions/python/ops/vector_student_t.py +++ b/tensorflow/contrib/distributions/python/ops/vector_student_t.py @@ -66,7 +66,7 @@ class _VectorStudentT(transformed_distribution.TransformedDistribution): This distribution is an Affine transformation of iid [Student's t-distributions]( https://en.wikipedia.org/wiki/Student%27s_t-distribution) - and should not be confused with the [Multivate Student's t-distribution]( + and should not be confused with the [Multivariate Student's t-distribution]( https://en.wikipedia.org/wiki/Multivariate_t-distribution). The traditional Multivariate Student's t-distribution is type of [elliptical distribution]( diff --git a/tensorflow/contrib/factorization/python/ops/clustering_ops.py b/tensorflow/contrib/factorization/python/ops/clustering_ops.py index 23137e0a97..84e80791f4 100644 --- a/tensorflow/contrib/factorization/python/ops/clustering_ops.py +++ b/tensorflow/contrib/factorization/python/ops/clustering_ops.py @@ -41,11 +41,12 @@ from tensorflow.python.platform import resource_loader _clustering_ops = loader.load_op_library( resource_loader.get_path_to_datafile('_clustering_ops.so')) -# Euclidean distance between vectors U and V is defined as ||U - V||_F which is -# the square root of the sum of the absolute squares of the elements difference. +# Euclidean distance between vectors U and V is defined as \\(||U - V||_F\\) +# which is the square root of the sum of the absolute squares of the elements +# difference. SQUARED_EUCLIDEAN_DISTANCE = 'squared_euclidean' # Cosine distance between vectors U and V is defined as -# 1 - (U \dot V) / (||U||_F ||V||_F) +# \\(1 - (U \dot V) / (||U||_F ||V||_F)\\) COSINE_DISTANCE = 'cosine' RANDOM_INIT = 'random' @@ -472,8 +473,8 @@ class KMeans(object): # Locally compute the sum of inputs mapped to each id. # For a cluster with old cluster value x, old count n, and with data # d_1,...d_k newly assigned to it, we recompute the new value as - # x += (sum_i(d_i) - k * x) / (n + k). - # Compute sum_i(d_i), see comment above. + # \\(x += (sum_i(d_i) - k * x) / (n + k)\\). + # Compute \\(sum_i(d_i)\\), see comment above. cluster_center_updates = math_ops.unsorted_segment_sum( inp, unique_idx, num_unique_cluster_idx) # Shape to enable broadcasting count_updates and learning_rate to inp. diff --git a/tensorflow/contrib/factorization/python/ops/factorization_ops.py b/tensorflow/contrib/factorization/python/ops/factorization_ops.py index 8e0ed1d80e..811fa89bc3 100644 --- a/tensorflow/contrib/factorization/python/ops/factorization_ops.py +++ b/tensorflow/contrib/factorization/python/ops/factorization_ops.py @@ -51,9 +51,9 @@ class WALSModel(object): r"""A model for Weighted Alternating Least Squares matrix factorization. It minimizes the following loss function over U, V: - \\( - \|\sqrt W \odot (A - U V^T) \|_F^2 + \lambda (\|U\|_F^2 + \|V\|_F^2) - )\\ + $$ + \|\sqrt W \odot (A - U V^T)\|_F^2 + \lambda (\|U\|_F^2 + \|V\|_F^2) + $$ where, A: input matrix, W: weight matrix. Note that the (element-wise) square root of the weights @@ -61,12 +61,12 @@ class WALSModel(object): U, V: row_factors and column_factors matrices, \\(\lambda)\\: regularization. Also we assume that W is of the following special form: - \\( W_{ij} = W_0 + R_i * C_j )\\ if \\(A_{ij} \ne 0)\\, - \\(W_{ij} = W_0)\\ otherwise. + \\( W_{ij} = W_0 + R_i * C_j \\) if \\(A_{ij} \ne 0\\), + \\(W_{ij} = W_0\\) otherwise. where, - \\(W_0)\\: unobserved_weight, - \\(R_i)\\: row_weights, - \\(C_j)\\: col_weights. + \\(W_0\\): unobserved_weight, + \\(R_i\\): row_weights, + \\(C_j\\): col_weights. Note that the current implementation supports two operation modes: The default mode is for the condition where row_factors and col_factors can individually @@ -82,14 +82,15 @@ class WALSModel(object): normalized as follows: _, _, unregularized_loss, regularization, sum_weights = update_row_factors(sp_input) - if sp_input contains the rows {A_i, i \in I}, and the input matrix A has n - total rows, then the minibatch loss = unregularized_loss + regularization is - \\( + if sp_input contains the rows \\({A_i, i \in I}\\), and the input matrix A + has n total rows, then the minibatch loss = unregularized_loss + + regularization is + $$ (\|\sqrt W_I \odot (A_I - U_I V^T)\|_F^2 + \lambda \|U_I\|_F^2) * n / |I| + \lambda \|V\|_F^2 - )\\ + $$ The sum_weights tensor contains the normalized sum of weights - sum(W_I) * n / |I|. + \\(sum(W_I) * n / |I|\\). A typical usage example (pseudocode): @@ -223,7 +224,7 @@ class WALSModel(object): factor shard. In this case, w_ij = unobserved_weight + row_weights[i] * col_weights[j]. - If this is a single non-negative real number, this value is used for - all row weights and w_ij = unobserved_weight + row_weights * + all row weights and \\(w_ij\\) = unobserved_weight + row_weights * col_weights[j]. Note that it is allowed to have row_weights as a list while col_weights a single number or vice versa. @@ -665,18 +666,18 @@ class WALSModel(object): factors. unregularized_loss: A tensor (scalar) that contains the normalized minibatch loss corresponding to sp_input, without the regularization - term. If sp_input contains the rows {A_{i, :}, i \in I}, and the input - matrix A has n total rows, then the unregularized loss is: - (\|\sqrt W_I \odot (A_I - U_I V^T)\|_F^2 * n / |I| + term. If sp_input contains the rows \\({A_{i, :}, i \in I}\\), and the + input matrix A has n total rows, then the unregularized loss is: + \\(\|\sqrt W_I \odot (A_I - U_I V^T)\|_F^2 * n / |I|\\) The total loss is unregularized_loss + regularization. regularization: A tensor (scalar) that contains the normalized regularization term for the minibatch loss corresponding to sp_input. - If sp_input contains the rows {A_{i, :}, i \in I}, and the input matrix - A has n total rows, then the regularization term is: - \lambda \|U_I\|_F^2) * n / |I| + \lambda \|V\|_F^2. + If sp_input contains the rows \\({A_{i, :}, i \in I}\\), and the input + matrix A has n total rows, then the regularization term is: + \\(\lambda \|U_I\|_F^2) * n / |I| + \lambda \|V\|_F^2\\). sum_weights: The sum of the weights W_I corresponding to sp_input, - normalized by a factor of n / |I|. The root weighted squared error is: - \sqrt(unregularized_loss / sum_weights). + normalized by a factor of \\(n / |I|\\). The root weighted squared + error is: \sqrt(unregularized_loss / sum_weights). """ return self._process_input_helper( True, sp_input=sp_input, transpose_input=transpose_input) @@ -698,18 +699,18 @@ class WALSModel(object): factors. unregularized_loss: A tensor (scalar) that contains the normalized minibatch loss corresponding to sp_input, without the regularization - term. If sp_input contains the columns {A_{:, j}, j \in J}, and the - input matrix A has m total columns, then the unregularized loss is: - (\|\sqrt W_J \odot (A_J - U V_J^T)\|_F^2 * m / |I| + term. If sp_input contains the columns \\({A_{:, j}, j \in J}\\), and + the input matrix A has m total columns, then the unregularized loss is: + \\(\|\sqrt W_J \odot (A_J - U V_J^T)\|_F^2 * m / |I|\\) The total loss is unregularized_loss + regularization. regularization: A tensor (scalar) that contains the normalized regularization term for the minibatch loss corresponding to sp_input. - If sp_input contains the columns {A_{:, j}, j \in J}, and the input - matrix A has m total columns, then the regularization term is: - \lambda \|V_J\|_F^2) * m / |J| + \lambda \|U\|_F^2. + If sp_input contains the columns \\({A_{:, j}, j \in J}\\), and the + input matrix A has m total columns, then the regularization term is: + \\(\lambda \|V_J\|_F^2) * m / |J| + \lambda \|U\|_F^2\\). sum_weights: The sum of the weights W_J corresponding to sp_input, - normalized by a factor of m / |J|. The root weighted squared error is: - \sqrt(unregularized_loss / sum_weights). + normalized by a factor of \\(m / |J|\\). The root weighted squared + error is: \sqrt(unregularized_loss / sum_weights). """ return self._process_input_helper( False, sp_input=sp_input, transpose_input=transpose_input) @@ -720,8 +721,8 @@ class WALSModel(object): projection_weights=None): """Projects the row factors. - This computes the row embedding u_i for an observed row a_i by solving - one iteration of the update equations. + This computes the row embedding \\(u_i\\) for an observed row \\(a_i\\) by + solving one iteration of the update equations. Args: sp_input: A SparseTensor representing a set of rows. Please note that the @@ -753,8 +754,8 @@ class WALSModel(object): projection_weights=None): """Projects the column factors. - This computes the column embedding v_j for an observed column a_j by solving - one iteration of the update equations. + This computes the column embedding \\(v_j\\) for an observed column + \\(a_j\\) by solving one iteration of the update equations. Args: sp_input: A SparseTensor representing a set of columns. Please note that @@ -938,7 +939,7 @@ class WALSModel(object): loss_sp_input = (sparse_ops.sparse_transpose(new_sp_input) if transpose_input else new_sp_input) # sp_approx is the low rank estimate of the input matrix, formed by - # computing the product for (i, j) in loss_sp_input.indices. + # computing the product <\\(u_i, v_j\\)> for (i, j) in loss_sp_input.indices. sp_approx_vals = gen_factorization_ops.masked_matmul( new_left_values, right, diff --git a/tensorflow/contrib/factorization/python/ops/gmm_ops.py b/tensorflow/contrib/factorization/python/ops/gmm_ops.py index 14d4c733e3..5d77bc77e1 100644 --- a/tensorflow/contrib/factorization/python/ops/gmm_ops.py +++ b/tensorflow/contrib/factorization/python/ops/gmm_ops.py @@ -357,8 +357,8 @@ class GmmAlgorithm(object): # Shape broadcasting. probs = array_ops.expand_dims(self._probs[shard_id], 0) # Membership weights are computed as: - # w_{ik} = \frac{\alpha_k f(\mathbf{y_i}|\mathbf{\theta}_k)} - # {\sum_{m=1}^{K}\alpha_mf(\mathbf{y_i}|\mathbf{\theta}_m)} + # $$w_{ik} = \frac{\alpha_k f(\mathbf{y_i}|\mathbf{\theta}_k)}$$ + # $$ {\sum_{m=1}^{K}\alpha_mf(\mathbf{y_i}|\mathbf{\theta}_m)}$$ # where "i" is the i-th example, "k" is the k-th mixture, theta are # the model parameters and y_i the observations. # These are defined for each shard. diff --git a/tensorflow/contrib/factorization/python/ops/kmeans.py b/tensorflow/contrib/factorization/python/ops/kmeans.py index 38faca119d..bfe338c9f9 100644 --- a/tensorflow/contrib/factorization/python/ops/kmeans.py +++ b/tensorflow/contrib/factorization/python/ops/kmeans.py @@ -374,11 +374,11 @@ class KMeansClustering(estimator.Estimator): than `num_clusters`, a TensorFlow runtime error occurs. distance_metric: The distance metric used for clustering. One of: * `KMeansClustering.SQUARED_EUCLIDEAN_DISTANCE`: Euclidean distance - between vectors `u` and `v` is defined as `||u - v||_2` which is - the square root of the sum of the absolute squares of the elements' - difference. + between vectors `u` and `v` is defined as `\\(||u - v||_2\\)` + which is the square root of the sum of the absolute squares of + the elements' difference. * `KMeansClustering.COSINE_DISTANCE`: Cosine distance between vectors - `u` and `v` is defined as `1 - (u . v) / (||u||_2 ||v||_2)`. + `u` and `v` is defined as `\\(1 - (u . v) / (||u||_2 ||v||_2)\\)`. random_seed: Python integer. Seed for PRNG used to initialize centers. use_mini_batch: A boolean specifying whether to use the mini-batch k-means algorithm. See explanation above. diff --git a/tensorflow/contrib/factorization/python/ops/wals.py b/tensorflow/contrib/factorization/python/ops/wals.py index 62db3bb4c4..ca46c39baa 100644 --- a/tensorflow/contrib/factorization/python/ops/wals.py +++ b/tensorflow/contrib/factorization/python/ops/wals.py @@ -216,7 +216,7 @@ def _wals_factorization_model_function(features, labels, mode, params): name=WALSMatrixFactorization.LOSS, collections=[ops.GraphKeys.GLOBAL_VARIABLES]) # The root weighted squared error = - # \sqrt( \sum_{i,j} w_ij * (a_ij - r_ij)^2 / \sum_{i,j} w_ij ) + # \\(\sqrt( \sum_{i,j} w_ij * (a_ij - r_ij)^2 / \sum_{i,j} w_ij )\\) rwse_var = variable_scope.variable( 0., trainable=False, @@ -490,11 +490,11 @@ class WALSMatrixFactorization(estimator.Estimator): and the problem simplifies to ALS. Note that, in this case, col_weights must also be set to "None". - List of lists of non-negative scalars, of the form - [[w_0, w_1, ...], [w_k, ... ], [...]], + \\([[w_0, w_1, ...], [w_k, ... ], [...]]\\), where the number of inner lists equal to the number of row factor shards and the elements in each inner list are the weights for the rows of that shard. In this case, - w_ij = unonbserved_weight + row_weights[i] * col_weights[j]. + \\(w_ij = unonbserved_weight + row_weights[i] * col_weights[j]\\). - A non-negative scalar: This value is used for all row weights. Note that it is allowed to have row_weights as a list and col_weights as a scalar, or vice-versa. diff --git a/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py b/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py index 082c42eba1..e3fc6bf0f0 100644 --- a/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py +++ b/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py @@ -88,8 +88,8 @@ class GANEstimator(estimator.Estimator): discriminator_fn=discriminator_fn, generator_loss_fn=tfgan.losses.wasserstein_generator_loss, discriminator_loss_fn=tfgan.losses.wasserstein_discriminator_loss, - generator_optimizer=tf.train.AdamOptimizier(0.1, 0.5), - discriminator_optimizer=tf.train.AdamOptimizier(0.1, 0.5)) + generator_optimizer=tf.train.AdamOptimizer(0.1, 0.5), + discriminator_optimizer=tf.train.AdamOptimizer(0.1, 0.5)) # Train estimator. gan_estimator.train(train_input_fn, steps) diff --git a/tensorflow/contrib/gan/python/losses/python/losses_impl.py b/tensorflow/contrib/gan/python/losses/python/losses_impl.py index 39588b7219..1ba3a64167 100644 --- a/tensorflow/contrib/gan/python/losses/python/losses_impl.py +++ b/tensorflow/contrib/gan/python/losses/python/losses_impl.py @@ -306,6 +306,7 @@ def wasserstein_gradient_penalty( discriminator_scope, epsilon=1e-10, target=1.0, + one_sided=False, weights=1.0, scope=None, loss_collection=ops.GraphKeys.LOSSES, @@ -327,6 +328,8 @@ def wasserstein_gradient_penalty( computing the gradient norm. target: Optional Python number or `Tensor` indicating the target value of gradient norm. Defaults to 1.0. + one_sided: If `True`, penalty proposed in https://arxiv.org/abs/1709.08894 + is used. Defaults to `False`. weights: Optional `Tensor` whose rank is either 0, or the same rank as `real_data` and `generated_data`, and must be broadcastable to them (i.e., all dimensions must be either `1`, or the same as the @@ -377,10 +380,13 @@ def wasserstein_gradient_penalty( # For numerical stability, add epsilon to the sum before taking the square # root. Note tf.norm does not add epsilon. slopes = math_ops.sqrt(gradient_squares + epsilon) - penalties = math_ops.square(slopes / target - 1.0) + penalties = slopes / target - 1.0 + if one_sided: + penalties = math_ops.maximum(0., penalties) + penalties_squared = math_ops.square(penalties) penalty = losses.compute_weighted_loss( - penalties, weights, scope=scope, loss_collection=loss_collection, - reduction=reduction) + penalties_squared, weights, scope=scope, + loss_collection=loss_collection, reduction=reduction) if add_summaries: summary.scalar('gradient_penalty_loss', penalty) @@ -665,7 +671,7 @@ def least_squares_discriminator_loss( loss_collection=ops.GraphKeys.LOSSES, reduction=losses.Reduction.SUM_BY_NONZERO_WEIGHTS, add_summaries=False): - """Least squares generator loss. + """Least squares discriminator loss. This loss comes from `Least Squares Generative Adversarial Networks` (https://arxiv.org/abs/1611.04076). diff --git a/tensorflow/contrib/gan/python/losses/python/losses_impl_test.py b/tensorflow/contrib/gan/python/losses/python/losses_impl_test.py index dbaa624ae9..2889e93743 100644 --- a/tensorflow/contrib/gan/python/losses/python/losses_impl_test.py +++ b/tensorflow/contrib/gan/python/losses/python/losses_impl_test.py @@ -481,6 +481,28 @@ class GradientPenaltyTest(test.TestCase, _PenaltyTest): }) self.assertAlmostEqual(self._expected_loss, loss, 5) + def test_loss_using_one_sided_mode(self): + generated_data = array_ops.placeholder(dtypes.float32, shape=(None, None)) + real_data = array_ops.placeholder(dtypes.float32, shape=(None, None)) + + loss = tfgan_losses.wasserstein_gradient_penalty( + generated_data, + real_data, + self._kwargs['generator_inputs'], + self._kwargs['discriminator_fn'], + self._kwargs['discriminator_scope'], + one_sided=True) + self.assertEqual(generated_data.dtype, loss.dtype) + + with self.test_session() as sess: + variables.global_variables_initializer().run() + loss = sess.run(loss, + feed_dict={ + generated_data: self._generated_data_np, + real_data: self._real_data_np, + }) + self.assertAlmostEqual(self._expected_loss, loss, 5) + def test_loss_with_gradient_norm_target(self): """Test loss value with non default gradient norm target.""" generated_data = array_ops.placeholder(dtypes.float32, shape=(None, None)) diff --git a/tensorflow/contrib/gan/python/train.py b/tensorflow/contrib/gan/python/train.py index 776eb11ecb..73acd05b60 100644 --- a/tensorflow/contrib/gan/python/train.py +++ b/tensorflow/contrib/gan/python/train.py @@ -461,6 +461,7 @@ def gan_loss( gradient_penalty_weight=None, gradient_penalty_epsilon=1e-10, gradient_penalty_target=1.0, + gradient_penalty_one_sided=False, mutual_information_penalty_weight=None, aux_cond_generator_weight=None, aux_cond_discriminator_weight=None, @@ -485,6 +486,8 @@ def gan_loss( gradient_penalty_target: If `gradient_penalty_weight` is not None, a Python number or `Tensor` indicating the target value of gradient norm. See the CIFAR10 section of https://arxiv.org/abs/1710.10196. Defaults to 1.0. + gradient_penalty_one_sided: If `True`, penalty proposed in + https://arxiv.org/abs/1709.08894 is used. Defaults to `False`. mutual_information_penalty_weight: If not `None`, must be a non-negative Python number or Tensor indicating how much to weight the mutual information penalty. See https://arxiv.org/abs/1606.03657 for more @@ -546,6 +549,7 @@ def gan_loss( model, epsilon=gradient_penalty_epsilon, target=gradient_penalty_target, + one_sided=gradient_penalty_one_sided, add_summaries=add_summaries) dis_loss += gradient_penalty_weight * gp_loss if _use_aux_loss(mutual_information_penalty_weight): diff --git a/tensorflow/contrib/gan/python/train_test.py b/tensorflow/contrib/gan/python/train_test.py index f9bdaa74c9..3ebbe55d05 100644 --- a/tensorflow/contrib/gan/python/train_test.py +++ b/tensorflow/contrib/gan/python/train_test.py @@ -359,10 +359,12 @@ class GANLossTest(test.TestCase): self.assertGreater(len(ops.get_collection(ops.GraphKeys.SUMMARIES)), 0) # Test gradient penalty option. - def _test_grad_penalty_helper(self, create_gan_model_fn): + def _test_grad_penalty_helper(self, create_gan_model_fn, one_sided=False): model = create_gan_model_fn() loss = train.gan_loss(model) - loss_gp = train.gan_loss(model, gradient_penalty_weight=1.0) + loss_gp = train.gan_loss(model, + gradient_penalty_weight=1.0, + gradient_penalty_one_sided=one_sided) self.assertTrue(isinstance(loss_gp, namedtuples.GANLoss)) # Check values. @@ -394,6 +396,25 @@ class GANLossTest(test.TestCase): def test_grad_penalty_callable_acgan(self): self._test_grad_penalty_helper(create_callable_acgan_model) + def test_grad_penalty_one_sided_gan(self): + self._test_grad_penalty_helper(create_gan_model, one_sided=True) + + def test_grad_penalty_one_sided_callable_gan(self): + self._test_grad_penalty_helper(create_callable_gan_model, one_sided=True) + + def test_grad_penalty_one_sided_infogan(self): + self._test_grad_penalty_helper(create_infogan_model, one_sided=True) + + def test_grad_penalty_one_sided_callable_infogan(self): + self._test_grad_penalty_helper( + create_callable_infogan_model, one_sided=True) + + def test_grad_penalty_one_sided_acgan(self): + self._test_grad_penalty_helper(create_acgan_model, one_sided=True) + + def test_grad_penalty_one_sided_callable_acgan(self): + self._test_grad_penalty_helper(create_callable_acgan_model, one_sided=True) + # Test mutual information penalty option. def _test_mutual_info_penalty_helper(self, create_gan_model_fn): train.gan_loss(create_gan_model_fn(), diff --git a/tensorflow/contrib/layers/python/layers/layers.py b/tensorflow/contrib/layers/python/layers/layers.py index 350bcb3bca..10d7f6d076 100644 --- a/tensorflow/contrib/layers/python/layers/layers.py +++ b/tensorflow/contrib/layers/python/layers/layers.py @@ -3045,16 +3045,16 @@ def legacy_fully_connected(x, `activation_fn` is `None`, the result of `y = w * x + b` is returned. - If `x` has shape [\\\(\\text{dim}_0, \\text{dim}_1, ..., \\text{dim}_n\\\)] - with more than 2 dimensions (\\\(n > 1\\\)), then we repeat the matrix + If `x` has shape [\\(\text{dim}_0, \text{dim}_1, ..., \text{dim}_n\\)] + with more than 2 dimensions (\\(n > 1\\)), then we repeat the matrix multiply along the first dimensions. The result r is a tensor of shape - [\\\(\\text{dim}_0, ..., \\text{dim}_{n-1},\\\) `num_output_units`], - where \\\( r_{i_0, ..., i_{n-1}, k} = - \\sum_{0 \\leq j < \\text{dim}_n} x_{i_0, ... i_{n-1}, j} \cdot w_{j, k}\\\). + [\\(\text{dim}_0, ..., \text{dim}_{n-1},\\) `num_output_units`], + where \\( r_{i_0, ..., i_{n-1}, k} = + \sum_{0 \leq j < \text{dim}_n} x_{i_0, ... i_{n-1}, j} \cdot w_{j, k}\\). This is accomplished by reshaping `x` to 2-D - [\\\(\\text{dim}_0 \\cdot ... \\cdot \\text{dim}_{n-1}, \\text{dim}_n\\\)] + [\\(\text{dim}_0 \cdot ... \cdot \text{dim}_{n-1}, \text{dim}_n\\)] before the matrix multiply and afterwards reshaping it to - [\\\(\\text{dim}_0, ..., \\text{dim}_{n-1},\\\) `num_output_units`]. + [\\(\text{dim}_0, ..., \text{dim}_{n-1},\\) `num_output_units`]. This op creates `w` and optionally `b`. Bias (`b`) can be disabled by setting `bias_init` to `None`. diff --git a/tensorflow/contrib/linalg/python/ops/linear_operator_block_diag.py b/tensorflow/contrib/linalg/python/ops/linear_operator_block_diag.py index 80649bd52d..9d3af66c92 100644 --- a/tensorflow/contrib/linalg/python/ops/linear_operator_block_diag.py +++ b/tensorflow/contrib/linalg/python/ops/linear_operator_block_diag.py @@ -138,8 +138,7 @@ class LinearOperatorBlockDiag(linear_operator.LinearOperator): meaning the quadratic form `x^H A x` has positive real part for all nonzero `x`. Note that we do not require the operator to be self-adjoint to be positive-definite. See: - https://en.wikipedia.org/wiki/Positive-definite_matrix\ - #Extension_for_non_symmetric_matrices + https://en.wikipedia.org/wiki/Positive-definite_matrix#Extension_for_non-symmetric_matrices is_square: Expect that this operator acts like square [batch] matrices. This is true by default, and will raise a `ValueError` otherwise. name: A name for this `LinearOperator`. Default is the individual diff --git a/tensorflow/contrib/lite/toco/graph_transformations/unpartition_embedding_lookup.cc b/tensorflow/contrib/lite/toco/graph_transformations/unpartition_embedding_lookup.cc index 48c326651f..cbea39bcc0 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/unpartition_embedding_lookup.cc +++ b/tensorflow/contrib/lite/toco/graph_transformations/unpartition_embedding_lookup.cc @@ -165,7 +165,7 @@ bool UnpartitionEmbeddingLookup::Run(Model* model, std::size_t op_index) { CHECK(mod_op && mod_op->type == OperatorType::kFloorMod) << "Unsupported partition strategy"; CHECK_EQ(mod_op, GetOpWithOutput(*model, indices_partition_op->inputs[1])) - << "Indices and data parition ops require the same partition strategy " + << "Indices and data partition ops require the same partition strategy " "and inputs"; // Glob together all of the gather data. This is not yet in the correct order. diff --git a/tensorflow/contrib/lite/toco/python/BUILD b/tensorflow/contrib/lite/toco/python/BUILD index 5a40451b3a..6c4f8e12cd 100644 --- a/tensorflow/contrib/lite/toco/python/BUILD +++ b/tensorflow/contrib/lite/toco/python/BUILD @@ -45,9 +45,6 @@ py_binary( name = "toco_wrapper", srcs = ["toco_wrapper.py"], srcs_version = "PY2AND3", - deps = [ - "//tensorflow:tensorflow_py", - ], ) tf_py_test( diff --git a/tensorflow/contrib/lite/toco/python/toco_wrapper.py b/tensorflow/contrib/lite/toco/python/toco_wrapper.py index e39b5f22c7..6d6b500d7e 100644 --- a/tensorflow/contrib/lite/toco/python/toco_wrapper.py +++ b/tensorflow/contrib/lite/toco/python/toco_wrapper.py @@ -22,14 +22,19 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import os import sys -import tensorflow as tf def main(): # Pip installs the binary in aux-bin off of main site-package install. # Just find it and exec, passing all arguments in the process. # TODO(aselle): it is unfortunate to use all of tensorflow to lookup binary. - binary = os.path.join(tf.__path__[0], 'aux-bin/toco') - os.execvp(binary, sys.argv) + print("""TOCO from pip install is currently not working on command line. +Please use the python TOCO API or use +bazel run tensorflow/contrib/lite:toco -- from a TensorFlow source dir. +""") + sys.exit(1) + # TODO(aselle): Replace this when we find a way to run toco without + # blowing up executable size. + # binary = os.path.join(tf.__path__[0], 'aux-bin/toco') + # os.execvp(binary, sys.argv) diff --git a/tensorflow/contrib/lookup/lookup_ops.py b/tensorflow/contrib/lookup/lookup_ops.py index a03e731be3..4942d94176 100644 --- a/tensorflow/contrib/lookup/lookup_ops.py +++ b/tensorflow/contrib/lookup/lookup_ops.py @@ -298,7 +298,7 @@ class MutableHashTable(LookupInterface): table = tf.contrib.lookup.MutableHashTable(key_dtype=tf.string, value_dtype=tf.int64, default_value=-1) - table.insert(keys, values) + sess.run(table.insert(keys, values)) out = table.lookup(query_keys) print(out.eval()) ``` diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py index 07b3ad71d4..d508cf3f9d 100644 --- a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py +++ b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py @@ -353,6 +353,42 @@ class AttentionWrapperTest(test.TestCase): attention_mechanism_depth=9, name='testLuongNotNormalized') + def testLuongScaledDType(self): + # Test case for GitHub issue 18099 + for dtype in [np.float16, np.float32, np.float64]: + num_units = 128 + encoder_outputs = array_ops.placeholder(dtype, shape=[64, None, 256]) + encoder_sequence_length = array_ops.placeholder(dtypes.int32, shape=[64]) + decoder_inputs = array_ops.placeholder(dtype, shape=[64, None, 128]) + decoder_sequence_length = array_ops.placeholder(dtypes.int32, shape=[64]) + batch_size = 64 + attention_mechanism = wrapper.LuongAttention( + num_units=num_units, + memory=encoder_outputs, + memory_sequence_length=encoder_sequence_length, + scale=True, + dtype=dtype, + ) + cell = rnn_cell.LSTMCell(num_units) + cell = wrapper.AttentionWrapper(cell, attention_mechanism) + + helper = helper_py.TrainingHelper(decoder_inputs, + decoder_sequence_length) + my_decoder = basic_decoder.BasicDecoder( + cell=cell, + helper=helper, + initial_state=cell.zero_state( + dtype=dtype, batch_size=batch_size)) + + final_outputs, final_state, _ = decoder.dynamic_decode(my_decoder) + self.assertTrue( + isinstance(final_outputs, basic_decoder.BasicDecoderOutput)) + self.assertEqual(final_outputs.rnn_output.dtype, dtype) + self.assertTrue( + isinstance(final_state, wrapper.AttentionWrapperState)) + self.assertTrue( + isinstance(final_state.cell_state, rnn_cell.LSTMStateTuple)) + def testLuongScaled(self): create_attention_mechanism = functools.partial( wrapper.LuongAttention, scale=True) diff --git a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py index be53779826..9e0d69593f 100644 --- a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py +++ b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py @@ -339,7 +339,8 @@ def _luong_score(query, keys, scale): if scale: # Scalar used in weight scaling g = variable_scope.get_variable( - "attention_g", dtype=dtype, initializer=1.) + "attention_g", dtype=dtype, + initializer=init_ops.ones_initializer, shape=()) score = g * score return score diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc index 370911e4d9..e920a797fe 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc +++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc @@ -346,11 +346,10 @@ void ReorderCKtoKC(const TRT_ShapedWeights& iweights, break; } case tensorflow::DataType::DT_HALF: { - Reorder2( - {k, c}, static_cast(iweights.GetValues()), - istrides, - static_cast(const_cast(oweights->GetValues())), - ostrides); + Reorder2({k, c}, static_cast(iweights.GetValues()), + istrides, static_cast( + const_cast(oweights->GetValues())), + ostrides); break; } default: diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD index af572d8124..d2746032a0 100644 --- a/tensorflow/contrib/timeseries/python/timeseries/BUILD +++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD @@ -246,6 +246,7 @@ py_test( ], srcs_version = "PY2AND3", tags = [ + "no_oss", "no_pip", # b/64527635 "no_pip_gpu", # b/63391119 ], diff --git a/tensorflow/contrib/tpu/tpu_estimator.md b/tensorflow/contrib/tpu/tpu_estimator.md index 4ef8f9eebd..639e708169 100644 --- a/tensorflow/contrib/tpu/tpu_estimator.md +++ b/tensorflow/contrib/tpu/tpu_estimator.md @@ -172,7 +172,7 @@ It is always recommended to port a small, simple model first to make sure that you are familiar with the basic concepts of `TPUEstimator` and test end-to-end behavior. Once your simple model runs, gradually add more functionality. In addition, there are several sample models, available at -[github.com/tensorflow/tpu-demos](https://github.com/tensorflow/tpu-demos). +[github.com/tensorflow/tpu](https://github.com/tensorflow/tpu). To convert your code from the vanilla `Estimator` class to use TPUs, change the following (note some of the details may change over time): diff --git a/tensorflow/contrib/training/python/training/evaluation.py b/tensorflow/contrib/training/python/training/evaluation.py index 1a5fb45be0..4bb53e8678 100644 --- a/tensorflow/contrib/training/python/training/evaluation.py +++ b/tensorflow/contrib/training/python/training/evaluation.py @@ -36,9 +36,8 @@ out the metrics values to stdout: # Choose the metrics to compute: names_to_values, names_to_updates = tf.contrib.metrics.aggregate_metric_map({ - "accuracy": tf.contrib.metrics.streaming_accuracy(predictions, labels), - "mse": tf.contrib.metrics.streaming_mean_squared_error( - predictions, labels), + "accuracy": tf.metrics.accuracy(labels, predictions), + "mse": tf.metrics.mean_squared_error(labels, predictions), }) # Define the summaries to write: @@ -81,9 +80,8 @@ more summaries and call the evaluate_repeatedly method: # Choose the metrics to compute: names_to_values, names_to_updates = tf.contrib.metrics.aggregate_metric_map({ - "accuracy": tf.contrib.metrics.streaming_accuracy(predictions, labels), - "mse": tf.contrib.metrics.streaming_mean_squared_error( - predictions, labels), + "accuracy": tf.metrics.accuracy(labels, predictions), + "mse": tf.metrics.mean_squared_error(labels, predictions), }) # Define the summaries to write: diff --git a/tensorflow/contrib/training/python/training/evaluation_test.py b/tensorflow/contrib/training/python/training/evaluation_test.py index b07039916c..c36d00e842 100644 --- a/tensorflow/contrib/training/python/training/evaluation_test.py +++ b/tensorflow/contrib/training/python/training/evaluation_test.py @@ -27,7 +27,6 @@ import numpy as np from tensorflow.contrib.framework.python.ops import variables from tensorflow.contrib.layers.python.layers import layers from tensorflow.contrib.losses.python.losses import loss_ops -from tensorflow.contrib.metrics.python.ops import metric_ops from tensorflow.contrib.training.python.training import evaluation from tensorflow.contrib.training.python.training import training from tensorflow.core.protobuf import config_pb2 @@ -38,6 +37,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import random_seed from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops import metrics from tensorflow.python.ops import state_ops from tensorflow.python.ops import variables as variables_lib from tensorflow.python.platform import gfile @@ -196,7 +196,8 @@ class EvaluateOnceTest(test.TestCase): logits = logistic_classifier(inputs) predictions = math_ops.round(logits) - accuracy, update_op = metric_ops.streaming_accuracy(predictions, labels) + accuracy, update_op = metrics.accuracy( + predictions=predictions, labels=labels) checkpoint_path = evaluation.wait_for_new_checkpoint(checkpoint_dir) @@ -311,7 +312,8 @@ class EvaluateRepeatedlyTest(test.TestCase): logits = logistic_classifier(inputs) predictions = math_ops.round(logits) - accuracy, update_op = metric_ops.streaming_accuracy(predictions, labels) + accuracy, update_op = metrics.accuracy( + predictions=predictions, labels=labels) final_values = evaluation.evaluate_repeatedly( checkpoint_dir=checkpoint_dir, @@ -365,7 +367,8 @@ class EvaluateRepeatedlyTest(test.TestCase): logits = logistic_classifier(inputs) predictions = math_ops.round(logits) - accuracy, update_op = metric_ops.streaming_accuracy(predictions, labels) + accuracy, update_op = metrics.accuracy( + predictions=predictions, labels=labels) timeout_fn_calls = [0] def timeout_fn(): @@ -417,9 +420,8 @@ class EvaluateRepeatedlyTest(test.TestCase): self.assertEqual(final_values['my_var'], expected_value) def _create_names_to_metrics(self, predictions, labels): - accuracy0, update_op0 = metric_ops.streaming_accuracy(predictions, labels) - accuracy1, update_op1 = metric_ops.streaming_accuracy( - predictions + 1, labels) + accuracy0, update_op0 = metrics.accuracy(labels, predictions) + accuracy1, update_op1 = metrics.accuracy(labels, predictions + 1) names_to_values = {'Accuracy': accuracy0, 'Another_accuracy': accuracy1} names_to_updates = {'Accuracy': update_op0, 'Another_accuracy': update_op1} diff --git a/tensorflow/contrib/verbs/rdma.h b/tensorflow/contrib/verbs/rdma.h index 94203ee2b3..c9df6beb6b 100644 --- a/tensorflow/contrib/verbs/rdma.h +++ b/tensorflow/contrib/verbs/rdma.h @@ -262,7 +262,7 @@ class RdmaTensorRequest { // Receive tensor content (RDMA write was completed). // // Decode proto if required and/or move to GPU if the content was not - // written to it directly (GPU direct is not avaliable). Afterwards, + // written to it directly (GPU direct is not available). Afterwards, // invoke Done(). void RecvTensorContent(); diff --git a/tensorflow/core/common_runtime/scoped_allocator_mgr.cc b/tensorflow/core/common_runtime/scoped_allocator_mgr.cc index e1f70404e3..be79cc4507 100644 --- a/tensorflow/core/common_runtime/scoped_allocator_mgr.cc +++ b/tensorflow/core/common_runtime/scoped_allocator_mgr.cc @@ -103,7 +103,7 @@ ScopedAllocatorContainer::~ScopedAllocatorContainer() { // In normal execution the table should be empty and all of its // contents deleted via Drop. When when a step ends early // (e.g. through abnormal termination) we need to clean up - // explicitly. So long as graph exection of the associated step has + // explicitly. So long as graph execution of the associated step has // completey terminated this should be safe. for (auto& it : allocators_) { if (it.second.field_index == ScopedAllocator::kBackingIndex) { diff --git a/tensorflow/core/kernels/mkl_input_conversion_op.cc b/tensorflow/core/kernels/mkl_input_conversion_op.cc index d91f7107c5..68d3e1c9ab 100644 --- a/tensorflow/core/kernels/mkl_input_conversion_op.cc +++ b/tensorflow/core/kernels/mkl_input_conversion_op.cc @@ -263,21 +263,18 @@ class MklInputConversionOp : public OpKernel { private: void Compute(OpKernelContext* context) override { - const Tensor& input_tensor_0 = MklGetInput(context, 0); + const int kInputIndex_0 = 0, kInputIndex_1 = 1; + const Tensor& input_tensor_0 = MklGetInput(context, kInputIndex_0); MklDnnShape input_shape_0; - GetMklShape(context, 0, &input_shape_0); + GetMklShape(context, kInputIndex_0, &input_shape_0); - const Tensor& input_tensor_1 = MklGetInput(context, 1); + const Tensor& input_tensor_1 = MklGetInput(context, kInputIndex_1); MklDnnShape input_shape_1; - GetMklShape(context, 1, &input_shape_1); - - bool tf_shapes_are_same = - context->input(0).shape() == context->input(1).shape(); + GetMklShape(context, kInputIndex_1, &input_shape_1); - VLOG(1) << "MklInputConversionOp: Input shapes are " - << (tf_shapes_are_same ? "*same*" : "*different*") << ": " - << context->input(0).shape().DebugString() << " and " - << context->input(1).shape().DebugString(); + VLOG(1) << "MklInputConversionOp: Input shapes are: " + << context->input(kInputIndex_0).shape().DebugString() << " and " + << context->input(kInputIndex_1).shape().DebugString(); // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // if both inputs are in TF format, just copy input tensors to output. @@ -285,15 +282,19 @@ class MklInputConversionOp : public OpKernel { VLOG(1) << "MklInputConversionOp: No conversion needed, " << "copying TF inputs to output"; - ForwardTfTensorInToOut(context, 0, 0); - ForwardTfTensorInToOut(context, 1, 1); + ForwardTfTensorInToOut(context, kInputIndex_0, kInputIndex_0); + ForwardTfTensorInToOut(context, kInputIndex_1, kInputIndex_1); return; } // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // If both inputs are in MKL format if (input_shape_0.IsMklTensor() && input_shape_1.IsMklTensor()) { - if (tf_shapes_are_same) { + // It is safer to compare the original TensorFlow shapes than to compare + // Mkl shapes since element wise ops are forwarded to Eigen implementation. + TensorShape tf_shape0 = input_shape_0.GetTfShape(); + TensorShape tf_shape1 = input_shape_1.GetTfShape(); + if (tf_shape0 == tf_shape1) { auto input0_md = input_shape_0.GetMklLayout(); auto input1_md = input_shape_1.GetMklLayout(); @@ -302,8 +303,8 @@ class MklInputConversionOp : public OpKernel { VLOG(1) << "MklInputConversionOp: No conversion needed, " << "copying MKL inputs with identical shapes to output"; - ForwardMklTensorInToOut(context, 0, 0); - ForwardMklTensorInToOut(context, 1, 1); + ForwardMklTensorInToOut(context, kInputIndex_0, kInputIndex_0); + ForwardMklTensorInToOut(context, kInputIndex_1, kInputIndex_1); return; } else { VLOG(1) << "MklInputConversionOp: Shape is same, but format is " @@ -324,7 +325,7 @@ class MklInputConversionOp : public OpKernel { mkl_output_mkl_shape.SetMklLayout(&input1_md); // Create output Mkl tensor for index 0 - AllocateOutputSetMklShape(context, 0, &tensor_out, + AllocateOutputSetMklShape(context, kInputIndex_0, &tensor_out, input_tensor_0.shape(), mkl_output_mkl_shape); @@ -342,7 +343,7 @@ class MklInputConversionOp : public OpKernel { stream(stream::kind::eager).submit(net).wait(); // Input1 will be passed through - ForwardMklTensorInToOut(context, 1, 1); + ForwardMklTensorInToOut(context, kInputIndex_1, kInputIndex_1); return; } } @@ -361,11 +362,11 @@ class MklInputConversionOp : public OpKernel { << "converted MKL inputs to TF format"; MklToTfOp::ConvertMklToTf(this, context, data_format_str, - op_data_type, has_avx512f_, 0); + op_data_type, has_avx512f_, kInputIndex_0); MklToTfOp::ConvertMklToTf(this, context, data_format_str, - op_data_type, has_avx512f_, 1); - SetDummyMklShapeOutput(context, 0); - SetDummyMklShapeOutput(context, 1); + op_data_type, has_avx512f_, kInputIndex_1); + SetDummyMklShapeOutput(context, kInputIndex_0); + SetDummyMklShapeOutput(context, kInputIndex_1); return; } @@ -377,7 +378,6 @@ class MklInputConversionOp : public OpKernel { const Tensor* mkl_tensor; const MklDnnShape* mkl_shape; const Tensor* tf_tensor; - MklDnnShape* tf_mkl_shape; uint mkl_tensor_index; uint tf_tensor_index; if (input_shape_0.IsMklTensor() && !input_shape_1.IsMklTensor()) { @@ -385,14 +385,12 @@ class MklInputConversionOp : public OpKernel { mkl_shape = &input_shape_0; mkl_tensor_index = 0; tf_tensor = &input_tensor_1; - tf_mkl_shape = &input_shape_1; tf_tensor_index = 1; } else if (!input_shape_0.IsMklTensor() && input_shape_1.IsMklTensor()) { mkl_tensor = &input_tensor_1; mkl_shape = &input_shape_1; mkl_tensor_index = 1; tf_tensor = &input_tensor_0; - tf_mkl_shape = &input_shape_0; tf_tensor_index = 0; } else { CHECK(false) << "MklInputConversionOp: Unexpected combination of input " @@ -466,8 +464,8 @@ class MklInputConversionOp : public OpKernel { } VLOG(1) << "MklInputConversionOp: Shapes (output): " - << context->mutable_output(0)->shape().DebugString() << " and " - << context->mutable_output(1)->shape().DebugString(); + << context->mutable_output(kInputIndex_0)->shape().DebugString() << " and " + << context->mutable_output(kInputIndex_1)->shape().DebugString(); VLOG(1) << "MklInputConversion completed successfully."; } diff --git a/tensorflow/core/kernels/mkl_softmax_op.cc b/tensorflow/core/kernels/mkl_softmax_op.cc index 170523b5b4..f79e18cff2 100644 --- a/tensorflow/core/kernels/mkl_softmax_op.cc +++ b/tensorflow/core/kernels/mkl_softmax_op.cc @@ -102,7 +102,7 @@ class MklSoftmaxOp : public OpKernel { // Softmax MklDnn output layout is same as input layout. auto dst_pd = src.GetUsrMemPrimDesc(); - // if input is MKL shape, ouput is also MKL shape. + // if input is MKL shape, output is also MKL shape. // if input is TF shape, output is also TF shape if (src_mkl_shape.IsMklTensor()) { output_mkl_shape.SetMklTensor(true); diff --git a/tensorflow/core/kernels/reduction_gpu_kernels.cu.h b/tensorflow/core/kernels/reduction_gpu_kernels.cu.h index 9237fa51d8..0de2ebb590 100644 --- a/tensorflow/core/kernels/reduction_gpu_kernels.cu.h +++ b/tensorflow/core/kernels/reduction_gpu_kernels.cu.h @@ -244,6 +244,33 @@ __global__ void RowReduceKernel( if (row < num_rows && lane == 0) out[row] = sum; } +template +struct storage_type { + T1 val; + __host__ __device__ storage_type() {} + __host__ __device__ operator T1() { return val; } + __host__ __device__ storage_type& operator=(const T1& in) { + val = in; + return *this; + } +}; + +template +struct storage_type> { + T2 real; + T2 imag; + __host__ __device__ storage_type() {} + __host__ __device__ operator std::complex() { + return std::complex(real, imag); + } + __host__ __device__ storage_type>& operator=( + const std::complex& in) { + real = in.real(); + imag = in.imag(); + return *this; + } +}; + // Works only if there are <= 16 columns // each warps sums over multiple rows at once template @@ -268,7 +295,7 @@ __global__ void ColumnReduceMax16ColumnsKernel( // 1D array necessary due to bug in CUDA 9 compiler. // TODO(nluehr) revert to 2D array when compiler is ready. - __shared__ value_type partial_sums[32 * 33]; + __shared__ storage_type partial_sums[32 * 33]; row += rows_per_warp * gridDim.y * blockDim.y; for (; row < num_rows; row += rows_per_warp * gridDim.y * blockDim.y) { @@ -294,7 +321,8 @@ __global__ void ColumnReduceMax16ColumnsKernel( if (blockDim.y > 1) { for (int row = 1; row < blockDim.y; ++row) { - s = op(s, partial_sums[threadIdx.x * 33 + row]); + value_type t = partial_sums[threadIdx.x * 33 + row]; + s = op(s, t); } } @@ -316,7 +344,7 @@ __global__ void ColumnReduceKernel( // 1D array necessary due to bug in CUDA 9 compiler. // TODO(nluehr) revert to 2D array when compiler is ready. - __shared__ value_type partial_sums[32 * 33]; + __shared__ storage_type partial_sums[32 * 33]; row += gridDim.y * blockDim.y; @@ -347,7 +375,8 @@ __global__ void ColumnReduceKernel( min(blockDim.y, num_rows - blockIdx.y * blockDim.y); for (int row = 1; row < numRowsThisBlock; ++row) { - s = op(s, partial_sums[threadIdx.x * 33 + row]); + value_type t = partial_sums[threadIdx.x * 33 + row]; + s = op(s, t); } out[col * gridDim.y + blockIdx.y] = s; diff --git a/tensorflow/core/kernels/segment_reduction_ops.h b/tensorflow/core/kernels/segment_reduction_ops.h index 7badc00572..a5186bdacb 100644 --- a/tensorflow/core/kernels/segment_reduction_ops.h +++ b/tensorflow/core/kernels/segment_reduction_ops.h @@ -16,6 +16,14 @@ limitations under the License. #ifndef TENSORFLOW_CORE_KERNELS_SEGMENT_REDUCTION_OPS_H_ #define TENSORFLOW_CORE_KERNELS_SEGMENT_REDUCTION_OPS_H_ + +// This file requires the following include because it uses CudaAtomicMax: +// #include "tensorflow/core/util/cuda_kernel_helper.h" + +// Unfortunately we can't add the #include, since it breaks compilation for +// non-GPU targets. This only breaks in clang, because it's more strict for +// template code and CudaAtomicMax is used in template context. + // This file requires the following include because it uses CudaAtomicMax: // #include "tensorflow/core/util/cuda_kernel_helper.h" diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc index e2453b9712..2852c49e19 100644 --- a/tensorflow/core/ops/dataset_ops.cc +++ b/tensorflow/core/ops/dataset_ops.cc @@ -105,8 +105,11 @@ REGISTER_OP("RepeatDataset") .Output("handle: variant") .Attr("output_types: list(type) >= 1") .Attr("output_shapes: list(shape) >= 1") - .SetShapeFn(shape_inference::ScalarShape); // TODO(mrry): Validate the - // shape of `count`. + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle count_shape; + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &count_shape)); + return shape_inference::ScalarShape(c); + }); REGISTER_OP("TakeDataset") .Input("input_dataset: variant") diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc index 6c2fc60bab..12d6dc5eaf 100644 --- a/tensorflow/core/ops/nn_ops.cc +++ b/tensorflow/core/ops/nn_ops.cc @@ -472,7 +472,7 @@ REGISTER_OP("DepthwiseConv2dNativeBackpropInput") .Input("filter: T") .Input("out_backprop: T") .Output("output: T") - .Attr("T: {bfloat16, float, double}") + .Attr("T: {half, bfloat16, float, double}") .Attr("strides: list(int)") .Attr(GetPaddingAttrString()) .Attr(GetConvnetDataFormatAttrString()) @@ -490,7 +490,7 @@ REGISTER_OP("DepthwiseConv2dNativeBackpropFilter") .Input("filter_sizes: int32") .Input("out_backprop: T") .Output("output: T") - .Attr("T: {bfloat16, float, double}") + .Attr("T: {half, bfloat16, float, double}") .Attr("strides: list(int)") .Attr(GetPaddingAttrString()) .Attr(GetConvnetDataFormatAttrString()) @@ -589,7 +589,7 @@ REGISTER_OP("AvgPool3D") .Attr("strides: list(int) >= 5") .Attr(GetPaddingAttrString()) .Attr(GetConvnet3dDataFormatAttrString()) - .Attr("T: {bfloat16, float, double}") + .Attr("T: {half, bfloat16, float, double}") .SetShapeFn(shape_inference::Pool3DShape); REGISTER_OP("AvgPool3DGrad") @@ -600,7 +600,7 @@ REGISTER_OP("AvgPool3DGrad") .Attr("strides: list(int) >= 5") .Attr(GetPaddingAttrString()) .Attr(GetConvnet3dDataFormatAttrString()) - .Attr("T: {bfloat16, float, double}") + .Attr("T: {half, bfloat16, float, double}") .SetShapeFn([](InferenceContext* c) { ShapeHandle s; TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(0, &s)); @@ -618,7 +618,7 @@ REGISTER_OP("MaxPool3D") .Attr("strides: list(int) >= 5") .Attr(GetPaddingAttrString()) .Attr(GetConvnet3dDataFormatAttrString()) - .Attr("T: {bfloat16, float}") + .Attr("T: {half, bfloat16, float}") .SetShapeFn(shape_inference::Pool3DShape); REGISTER_OP("MaxPool3DGrad") @@ -630,8 +630,8 @@ REGISTER_OP("MaxPool3DGrad") .Attr("strides: list(int) >= 5") .Attr(GetPaddingAttrString()) .Attr(GetConvnet3dDataFormatAttrString()) - .Attr("T: {bfloat16, float} = DT_FLOAT") - .Attr("TInput: {bfloat16, float} = DT_FLOAT") + .Attr("T: {half, bfloat16, float} = DT_FLOAT") + .Attr("TInput: {half, bfloat16, float} = DT_FLOAT") .SetShapeFn([](InferenceContext* c) { return UnchangedShapeWithRank(c, 5); }); @@ -1170,9 +1170,9 @@ Status TopKShapeFn(InferenceContext* c) { DimensionHandle last_dim = c->Dim(input, -1); if (c->ValueKnown(last_dim) && c->ValueKnown(k_dim) && c->Value(last_dim) < c->Value(k_dim)) { - return errors::InvalidArgument( - "input must have last dimension >= k = ", c->Value(k_dim), " but is ", - c->Value(last_dim)); + return errors::InvalidArgument("input must have last dimension >= k = ", + c->Value(k_dim), " but is ", + c->Value(last_dim)); } // Replace last_dim with k_dim. @@ -1226,9 +1226,9 @@ REGISTER_OP("NthElement") DimensionHandle last_dim = c->Dim(input, -1); if (c->ValueKnown(last_dim) && c->ValueKnown(n_dim) && c->Value(last_dim) <= c->Value(n_dim)) { - return errors::InvalidArgument( - "Input must have last dimension > n = ", c->Value(n_dim), - " but is ", c->Value(last_dim)); + return errors::InvalidArgument("Input must have last dimension > n = ", + c->Value(n_dim), " but is ", + c->Value(last_dim)); } // Reduce last_dim for output tensor diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index 40eebd1db0..706968d347 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -24,7 +24,7 @@ limitations under the License. // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1", // "-beta", "-rc", "-rc.1") -#define TF_VERSION_SUFFIX "-rc1" +#define TF_VERSION_SUFFIX "" #define TF_STR_HELPER(x) #x #define TF_STR(x) TF_STR_HELPER(x) diff --git a/tensorflow/docs_src/api_guides/python/contrib.graph_editor.md b/tensorflow/docs_src/api_guides/python/contrib.graph_editor.md index de4f126507..20fe88a799 100644 --- a/tensorflow/docs_src/api_guides/python/contrib.graph_editor.md +++ b/tensorflow/docs_src/api_guides/python/contrib.graph_editor.md @@ -61,21 +61,21 @@ A subgraph can be created in several ways: * using a list of ops: -```python -my_sgv = ge.sgv(ops) -``` + ```python + my_sgv = ge.sgv(ops) + ``` * from a name scope: -```python -my_sgv = ge.sgv_scope("foo/bar", graph=tf.get_default_graph()) -``` + ```python + my_sgv = ge.sgv_scope("foo/bar", graph=tf.get_default_graph()) + ``` * using regular expression: -```python -my_sgv = ge.sgv("foo/.*/.*read$", graph=tf.get_default_graph()) -``` + ```python + my_sgv = ge.sgv("foo/.*/.*read$", graph=tf.get_default_graph()) + ``` Note that the Graph Editor is meant to manipulate several graphs at the same time, typically during transform or copy operation. For that reason, diff --git a/tensorflow/docs_src/api_guides/python/io_ops.md b/tensorflow/docs_src/api_guides/python/io_ops.md index 94cf0de32a..86b4b39409 100644 --- a/tensorflow/docs_src/api_guides/python/io_ops.md +++ b/tensorflow/docs_src/api_guides/python/io_ops.md @@ -8,7 +8,7 @@ Note: Functions taking `Tensor` arguments can also take anything accepted by ## Placeholders TensorFlow provides a placeholder operation that must be fed with data -on execution. For more info, see the section on @{$reading_data#feeding$Feeding data}. +on execution. For more info, see the section on @{$reading_data#Feeding$Feeding data}. * @{tf.placeholder} * @{tf.placeholder_with_default} @@ -42,7 +42,7 @@ formats into tensors. ### Example protocol buffer -TensorFlow's @{$reading_data#standard-tensorflow-format$recommended format for training examples} +TensorFlow's @{$reading_data#standard_tensorflow_format$recommended format for training examples} is serialized `Example` protocol buffers, [described here](https://www.tensorflow.org/code/tensorflow/core/example/example.proto). They contain `Features`, [described diff --git a/tensorflow/docs_src/api_guides/python/nn.md b/tensorflow/docs_src/api_guides/python/nn.md index 8e6fd1cff9..8d8daaae19 100644 --- a/tensorflow/docs_src/api_guides/python/nn.md +++ b/tensorflow/docs_src/api_guides/python/nn.md @@ -89,7 +89,7 @@ bottom. Note that this is different from existing libraries such as cuDNN and Caffe, which explicitly specify the number of padded pixels and always pad the same number of pixels on both sides. -For the `'VALID`' scheme, the output height and width are computed as: +For the `'VALID'` scheme, the output height and width are computed as: out_height = ceil(float(in_height - filter_height + 1) / float(strides[1])) out_width = ceil(float(in_width - filter_width + 1) / float(strides[2])) @@ -98,10 +98,10 @@ and no padding is used. Given the output size and the padding, the output can be computed as - output[b, i, j, :] = - sum_{di, dj} input[b, strides[1] * i + di - pad_top, - strides[2] * j + dj - pad_left, ...] * - filter[di, dj, ...] +$$ output[b, i, j, :] = + sum_{d_i, d_j} input[b, strides[1] * i + d_i - pad_{top},\ + strides[2] * j + d_j - pad_{left}, ...] * + filter[d_i, d_j,\ ...]$$ where any value outside the original input image region are considered zero ( i.e. we pad zero values around the border of the image). @@ -161,12 +161,12 @@ Morphological operators are non-linear filters used in image processing. ](https://en.wikipedia.org/wiki/Dilation_(morphology)) is the max-sum counterpart of standard sum-product convolution: - output[b, y, x, c] = +$$ output[b, y, x, c] = max_{dy, dx} input[b, strides[1] * y + rates[1] * dy, strides[2] * x + rates[2] * dx, c] + - filter[dy, dx, c] + filter[dy, dx, c]$$ The `filter` is usually called structuring function. Max-pooling is a special case of greyscale morphological dilation when the filter assumes all-zero @@ -176,12 +176,12 @@ values (a.k.a. flat structuring function). ](https://en.wikipedia.org/wiki/Erosion_(morphology)) is the min-sum counterpart of standard sum-product convolution: - output[b, y, x, c] = +$$ output[b, y, x, c] = min_{dy, dx} input[b, strides[1] * y - rates[1] * dy, strides[2] * x - rates[2] * dx, c] - - filter[dy, dx, c] + filter[dy, dx, c]$$ Dilation and erosion are dual to each other. The dilation of the input signal `f` by the structuring signal `g` is equal to the negation of the erosion of diff --git a/tensorflow/docs_src/get_started/index.md b/tensorflow/docs_src/get_started/index.md index 9c58b3b900..b28cb9df75 100644 --- a/tensorflow/docs_src/get_started/index.md +++ b/tensorflow/docs_src/get_started/index.md @@ -10,15 +10,18 @@ course prior to diving into TensorFlow documentation: TensorFlow is a tool for machine learning. While it contains a wide range of functionality, TensorFlow is mainly designed for deep neural network models. -TensorFlow provides many APIs. This section focuses on the high-level APIs. -If you are new to TensorFlow, begin by reading one of the following documents: - - * @{$get_started/eager} is for machine learning beginners and uses - @{$programmers_guide/eager}. - * @{$get_started/get_started_for_beginners} is also for machine learning - beginners and uses @{$programmers_guide/graphs}. - * @{$get_started/premade_estimators} assumes some machine learning background - and uses an @{tf.estimator.Estimator$Estimator}. +The easiest way to get started with tensorflow is using Eager Execution. + + * @{$get_started/eager}, is for anyone new to machine learning or TensorFlow. + +TensorFlow provides many APIs. The remainder of this section focuses on the +Estimator API which provide scalable, high-performance models. +To get started with Estimators begin by reading one of the following documents: + + * @{$get_started/get_started_for_beginners}, which is aimed at readers + new to machine learning. + * @{$get_started/premade_estimators}, which is aimed at readers who have + experience in machine learning. Then, read the following documents, which demonstrate the key features in the high-level APIs: diff --git a/tensorflow/docs_src/get_started/leftnav_files b/tensorflow/docs_src/get_started/leftnav_files index 17bc209e46..4c12f0d84b 100644 --- a/tensorflow/docs_src/get_started/leftnav_files +++ b/tensorflow/docs_src/get_started/leftnav_files @@ -5,7 +5,10 @@ eager.md get_started_for_beginners.md premade_estimators.md -### Details +### Estimators +get_started_for_beginners.md: For Beginners +premade_estimators.md: Premade Estimators +>>> checkpoints.md feature_columns.md datasets_quickstart.md diff --git a/tensorflow/docs_src/get_started/premade_estimators.md b/tensorflow/docs_src/get_started/premade_estimators.md index aa4f85f6ce..4be7e508f9 100644 --- a/tensorflow/docs_src/get_started/premade_estimators.md +++ b/tensorflow/docs_src/get_started/premade_estimators.md @@ -1,4 +1,4 @@ -# Get Started with Estimators +# Premade Estimators This document introduces the TensorFlow programming environment and shows you how to solve the Iris classification problem in TensorFlow. diff --git a/tensorflow/docs_src/install/install_c.md b/tensorflow/docs_src/install/install_c.md index 9059b3f3b6..a3eca4bf37 100644 --- a/tensorflow/docs_src/install/install_c.md +++ b/tensorflow/docs_src/install/install_c.md @@ -38,7 +38,7 @@ enable TensorFlow for C: OS="linux" # Change to "darwin" for macOS TARGET_DIRECTORY="/usr/local" curl -L \ - "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.7.0-rc1.tar.gz" | + "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.7.0.tar.gz" | sudo tar -C $TARGET_DIRECTORY -xz The `tar` command extracts the TensorFlow C library into the `lib` diff --git a/tensorflow/docs_src/install/install_go.md b/tensorflow/docs_src/install/install_go.md index 2e47a6d212..1a0956634d 100644 --- a/tensorflow/docs_src/install/install_go.md +++ b/tensorflow/docs_src/install/install_go.md @@ -38,7 +38,7 @@ steps to install this library and enable TensorFlow for Go: TF_TYPE="cpu" # Change to "gpu" for GPU support TARGET_DIRECTORY='/usr/local' curl -L \ - "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.7.0-rc1.tar.gz" | + "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.7.0.tar.gz" | sudo tar -C $TARGET_DIRECTORY -xz The `tar` command extracts the TensorFlow C library into the `lib` diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md index eff066d200..cdde45a6f4 100644 --- a/tensorflow/docs_src/install/install_java.md +++ b/tensorflow/docs_src/install/install_java.md @@ -36,7 +36,7 @@ following to the project's `pom.xml` to use the TensorFlow Java APIs: org.tensorflow tensorflow - 1.7.0-rc1 + 1.7.0 ``` @@ -65,7 +65,7 @@ As an example, these steps will create a Maven project that uses TensorFlow: org.tensorflow tensorflow - 1.7.0-rc1 + 1.7.0 @@ -123,12 +123,12 @@ instead: org.tensorflow libtensorflow - 1.7.0-rc1 + 1.7.0 org.tensorflow libtensorflow_jni_gpu - 1.7.0-rc1 + 1.7.0 ``` @@ -147,7 +147,7 @@ refer to the simpler instructions above instead. Take the following steps to install TensorFlow for Java on Linux or macOS: 1. Download - [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.7.0-rc1.jar), + [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.7.0.jar), which is the TensorFlow Java Archive (JAR). 2. Decide whether you will run TensorFlow for Java on CPU(s) only or with @@ -166,7 +166,7 @@ Take the following steps to install TensorFlow for Java on Linux or macOS: OS=$(uname -s | tr '[:upper:]' '[:lower:]') mkdir -p ./jni curl -L \ - "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.7.0-rc1.tar.gz" | + "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.7.0.tar.gz" | tar -xz -C ./jni ### Install on Windows @@ -174,10 +174,10 @@ Take the following steps to install TensorFlow for Java on Linux or macOS: Take the following steps to install TensorFlow for Java on Windows: 1. Download - [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.7.0-rc1.jar), + [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.7.0.jar), which is the TensorFlow Java Archive (JAR). 2. Download the following Java Native Interface (JNI) file appropriate for - [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.7.0-rc1.zip). + [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.7.0.zip). 3. Extract this .zip file. @@ -225,7 +225,7 @@ must be part of your `classpath`. For example, you can include the downloaded `.jar` in your `classpath` by using the `-cp` compilation flag as follows: -
javac -cp libtensorflow-1.7.0-rc1.jar HelloTF.java
+
javac -cp libtensorflow-1.7.0.jar HelloTF.java
### Running @@ -239,11 +239,11 @@ two files are available to the JVM: For example, the following command line executes the `HelloTF` program on Linux and macOS X: -
java -cp libtensorflow-1.7.0-rc1.jar:. -Djava.library.path=./jni HelloTF
+
java -cp libtensorflow-1.7.0.jar:. -Djava.library.path=./jni HelloTF
And the following command line executes the `HelloTF` program on Windows: -
java -cp libtensorflow-1.7.0-rc1.jar;. -Djava.library.path=jni HelloTF
+
java -cp libtensorflow-1.7.0.jar;. -Djava.library.path=jni HelloTF
If the program prints Hello from version, you've successfully installed TensorFlow for Java and are ready to use the API. If the program diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md index 27b696696d..04e4242b0f 100644 --- a/tensorflow/docs_src/install/install_linux.md +++ b/tensorflow/docs_src/install/install_linux.md @@ -46,6 +46,35 @@ must be installed on your system: a list of supported GPU cards. * [GPU drivers](http://nvidia.com/driver) supporting your version of the CUDA Toolkit. + * The libcupti-dev library, which is the NVIDIA CUDA Profile Tools Interface. + This library provides advanced profiling support. To install this library, + issue the following command for CUDA Toolkit >= 8.0: + + 
+    $ sudo apt-get install cuda-command-line-tools
+
+ + and add its path to your `LD_LIBRARY_PATH` environment variable: + + 
+    $ export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}/usr/local/cuda/extras/CUPTI/lib64
+
+ + For CUDA Toolkit <= 7.5 do: + + 
+    $ sudo apt-get install libcupti-dev
+
+ * **[OPTIONAL]** For optimized inferencing performance, you can also install + NVIDIA TensorRT 3.0. For details, see + [NVIDIA's TensorRT documentation](http://docs.nvidia.com/deeplearning/sdk/tensorrt-install-guide/index.html#installing-tar). + Only steps 1-4 in the TensorRT Tar File installation instructions are + required for compatibility with TensorFlow; the Python package installation + in steps 5 and 6 can be omitted. Detailed installation instructions can be found at [package documentataion](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/tensorrt#installing-tensorrt-304) + + **IMPORTANT:** For compatibility with the pre-built `tensorflow-gpu` + package, please use the Ubuntu **14.04** tar file package of TensorRT + even when installing onto an Ubuntu 16.04 system. If you have an earlier version of the preceding packages, please upgrade to the specified versions. If upgrading is not possible, then you may still run @@ -165,7 +194,7 @@ Take the following steps to install TensorFlow with Virtualenv: Virtualenv environment: 
(tensorflow)$ pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp34-cp34m-linux_x86_64.whl
+ https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0-cp34-cp34m-linux_x86_64.whl If you encounter installation problems, see [Common Installation Problems](#common_installation_problems). @@ -270,7 +299,7 @@ take the following steps: 
      $ sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp34-cp34m-linux_x86_64.whl
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0-cp34-cp34m-linux_x86_64.whl
If this step fails, see @@ -456,7 +485,7 @@ Take the following steps to install TensorFlow in an Anaconda environment: 
      (tensorflow)$ pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp34-cp34m-linux_x86_64.whl
+ https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0-cp34-cp34m-linux_x86_64.whl ## Validate your installation @@ -630,14 +659,14 @@ This section documents the relevant values for Linux installations. CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0-cp27-none-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc1-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0-cp27-none-linux_x86_64.whl
Note that GPU support requires the NVIDIA hardware and software described in @@ -649,14 +678,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0-cp34-cp34m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc1-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0-cp34-cp34m-linux_x86_64.whl
Note that GPU support requires the NVIDIA hardware and software described in @@ -668,14 +697,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0-cp35-cp35m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc1-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0-cp35-cp35m-linux_x86_64.whl
@@ -687,14 +716,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0-cp36-cp36m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc1-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0-cp36-cp36m-linux_x86_64.whl
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md index 7060ef43da..b3e9616a05 100644 --- a/tensorflow/docs_src/install/install_mac.md +++ b/tensorflow/docs_src/install/install_mac.md @@ -119,7 +119,7 @@ Take the following steps to install TensorFlow with Virtualenv: TensorFlow in the active Virtualenv is as follows: 
 $ pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py3-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0-py3-none-any.whl If you encounter installation problems, see [Common Installation Problems](#common-installation-problems). @@ -242,7 +242,7 @@ take the following steps: issue the following command: 
 $ sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py3-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0-py3-none-any.whl If the preceding command fails, see [installation problems](#common-installation-problems). @@ -350,7 +350,7 @@ Take the following steps to install TensorFlow in an Anaconda environment: TensorFlow for Python 2.7: 
 (targetDirectory)$ pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py2-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0-py2-none-any.whl @@ -524,7 +524,7 @@ The value you specify depends on your Python version. 
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py2-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0-py2-none-any.whl
@@ -532,5 +532,5 @@ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py2-none-a 
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py3-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0-py3-none-any.whl
diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md index 148f80efe2..7d7c2aa75a 100644 --- a/tensorflow/docs_src/install/install_sources.md +++ b/tensorflow/docs_src/install/install_sources.md @@ -350,10 +350,10 @@ Invoke `pip install` to install that pip package. The filename of the `.whl` file depends on your platform. For example, the following command will install the pip package -for TensorFlow 1.7.0rc1 on Linux: +for TensorFlow 1.7.0 on Linux: 
-$ sudo pip install /tmp/tensorflow_pkg/tensorflow-1.7.0rc1-py2-none-any.whl
+$ sudo pip install /tmp/tensorflow_pkg/tensorflow-1.7.0-py2-none-any.whl
## Validate your installation @@ -450,8 +450,8 @@ Stack Overflow and specify the `tensorflow` tag. **Linux** - - + + @@ -471,7 +471,7 @@ Stack Overflow and specify the `tensorflow` tag. **Mac**
Version:CPU/GPU:Python Version:Compiler:Build Tools:cuDNN:CUDA:
tensorflow-1.7.0rc1CPU2.7, 3.3-3.6GCC 4.8Bazel 0.10.0N/AN/A
tensorflow_gpu-1.7.0rc1GPU2.7, 3.3-3.6GCC 4.8Bazel 0.9.079
tensorflow-1.7.0CPU2.7, 3.3-3.6GCC 4.8Bazel 0.10.0N/AN/A
tensorflow_gpu-1.7.0GPU2.7, 3.3-3.6GCC 4.8Bazel 0.9.079
tensorflow-1.6.0CPU2.7, 3.3-3.6GCC 4.8Bazel 0.9.0N/AN/A
tensorflow_gpu-1.6.0GPU2.7, 3.3-3.6GCC 4.8Bazel 0.9.079
tensorflow-1.5.0CPU2.7, 3.3-3.6GCC 4.8Bazel 0.8.0N/AN/A
- + @@ -486,8 +486,8 @@ Stack Overflow and specify the `tensorflow` tag. **Windows**
Version:CPU/GPU:Python Version:Compiler:Build Tools:cuDNN:CUDA:
tensorflow-1.7.0rc1CPU2.7, 3.3-3.6Clang from xcodeBazel 0.10.1N/AN/A
tensorflow-1.7.0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.10.1N/AN/A
tensorflow-1.6.0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.8.1N/AN/A
tensorflow-1.5.0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.8.1N/AN/A
tensorflow-1.4.0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.5.4N/AN/A
- - + + diff --git a/tensorflow/docs_src/programmers_guide/using_tpu.md b/tensorflow/docs_src/programmers_guide/using_tpu.md index a9c2cb3e33..cb0d86fc4c 100644 --- a/tensorflow/docs_src/programmers_guide/using_tpu.md +++ b/tensorflow/docs_src/programmers_guide/using_tpu.md @@ -11,7 +11,7 @@ This doc is aimed at users who: using an existing model. * Have, perhaps, skimmed the code of an example TPU model [[1]](https://github.com/tensorflow/models/blob/master/official/mnist/mnist_tpu.py) - [[2]](https://github.com/tensorflow/tpu-demos/tree/master/cloud_tpu/models). + [[2]](https://github.com/tensorflow/tpu/tree/master/models). * Are interested in porting an existing `Estimator` model to run on Cloud TPUs @@ -288,7 +288,7 @@ If shape inference has failed, but the shape is known it is possible to impose the correct shape using `tf.set_shape()`. In the example below the shape -inference algorithm fails, but it is corrected using `set_shape`: +inference algorithm fails, but it is correctly using `set_shape`: ``` >>> x = tf.zeros(tf.constant([1,2,3])+1) @@ -371,10 +371,10 @@ in bytes. A minimum of a few MB (`buffer_size=8*1024*1024`) is recommended so that data is available when needed. The TPU-demos repo includes -[a script](https://github.com/tensorflow/tpu-demos/blob/master/cloud_tpu/datasets/imagenet_to_gcs.py) +[a script](https://github.com/tensorflow/tpu/blob/master/tools/datasets/imagenet_to_gcs.py) for downloading the imagenet dataset and converting it to an appropriate format. This together with the imagenet -[models](https://github.com/tensorflow/tpu-demos/tree/master/cloud_tpu/models) +[models](https://github.com/tensorflow/tpu/tree/master/models) included in the repo demonstrate all of these best-practices. @@ -387,7 +387,7 @@ For details on how to actually set up and run a Cloud TPU see: This document is by no means exhaustive. The best source of more detail on how to make a Cloud TPU compatible model are the example models published in: - * The [TPU Demos Repository.](https://github.com/tensorflow/tpu-demos/) + * The [TPU Demos Repository.](https://github.com/tensorflow/tpu) For more information about tuning TensorFlow code for performance see: diff --git a/tensorflow/docs_src/tutorials/layers.md b/tensorflow/docs_src/tutorials/layers.md index aeb746f29c..cadaec391d 100644 --- a/tensorflow/docs_src/tutorials/layers.md +++ b/tensorflow/docs_src/tutorials/layers.md @@ -198,17 +198,23 @@ Classifier"](#training_and_evaluating_the_cnn_mnist_classifier). ### Input Layer The methods in the `layers` module for creating convolutional and pooling layers -for two-dimensional image data expect input tensors to have a `channels_last` shape of -[batch_size, image_height, image_width, channels] -or a `channels_first` shape of [batch_size, channels, image_height, image_width], defined as follows: +for two-dimensional image data expect input tensors to have a shape of +[batch_size, image_height, image_width, +channels] by default. This behavior can be changed using the data_format parameter; defined as follows: + * _`batch_size`_. Size of the subset of examples to use when performing gradient descent during training. -* _`image_width`_. Width of the example images. * _`image_height`_. Height of the example images. +* _`image_width`_. Width of the example images. * _`channels`_. Number of color channels in the example images. For color images, the number of channels is 3 (red, green, blue). For monochrome images, there is just 1 channel (black). +* _`image_height`_. Height of the example images. +* _`data_format`_. A string, one of `channels_last` (default) or `channels_first`. + `channels_last` corresponds to inputs with shape + `(batch, ..., channels)` while `channels_first` corresponds to + inputs with shape `(batch, channels, ...)`. Here, our MNIST dataset is composed of monochrome 28x28 pixel images, so the desired shape for our input layer is [batch_size, 28, 28, @@ -247,28 +253,27 @@ conv1 = tf.layers.conv2d( ``` The `inputs` argument specifies our input tensor, which must have the shape -[batch_size, image_width, image_height, +[batch_size, image_height, image_width, channels]. Here, we're connecting our first convolutional layer to `input_layer`, which has the shape [batch_size, 28, 28, 1]. > Note: conv2d() will instead accept a shape of -> [channels, batch_size, image_width, -> image_height] when passed the argument +> [batch_size, channels, image_height, image_width] when passed the argument > data_format=channels_first. The `filters` argument specifies the number of filters to apply (here, 32), and -`kernel_size` specifies the dimensions of the filters as [width, -height] (here, [5, 5]). +`kernel_size` specifies the dimensions of the filters as [height, +width] (here, [5, 5]). -

TIP: If filter width and height have the same value, you can instead specify a +

TIP: If filter height and width have the same value, you can instead specify a single integer for kernel_size—e.g., kernel_size=5.

The `padding` argument specifies one of two enumerated values (case-insensitive): `valid` (default value) or `same`. To specify that the -output tensor should have the same width and height values as the input tensor, +output tensor should have the same height and width values as the input tensor, we set `padding=same` here, which instructs TensorFlow to add 0 values to the -edges of the input tensor to preserve width and height of 28. (Without padding, +edges of the input tensor to preserve height and width of 28. (Without padding, a 5x5 convolution over a 28x28 tensor will produce a 24x24 tensor, as there are 24x24 locations to extract a 5x5 tile from a 28x28 grid.) @@ -277,7 +282,7 @@ output of the convolution. Here, we specify ReLU activation with @{tf.nn.relu}. Our output tensor produced by `conv2d()` has a shape of -[batch_size, 28, 28, 32]: the same width and height +[batch_size, 28, 28, 32]: the same height and width dimensions as the input, but now with 32 channels holding the output from each of the filters. @@ -292,31 +297,30 @@ pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2) ``` Again, `inputs` specifies the input tensor, with a shape of -[batch_size, image_width, image_height, +[batch_size, image_height, image_width, channels]. Here, our input tensor is `conv1`, the output from the first convolutional layer, which has a shape of [batch_size, 28, 28, 32]. > Note: As with conv2d(), max_pooling2d() will instead -> accept a shape of [channels, batch_size, -> image_width, image_height] when passed the argument +> accept a shape of [batch_size, channels, +> image_height, image_width] when passed the argument > data_format=channels_first. The `pool_size` argument specifies the size of the max pooling filter as -[width, height] (here, `[2, 2]`). If both +[height, width] (here, `[2, 2]`). If both dimensions have the same value, you can instead specify a single integer (e.g., `pool_size=2`). The `strides` argument specifies the size of the stride. Here, we set a stride of 2, which indicates that the subregions extracted by the filter should be -separated by 2 pixels in both the width and height dimensions (for a 2x2 filter, +separated by 2 pixels in both the height and width dimensions (for a 2x2 filter, this means that none of the regions extracted will overlap). If you want to set -different stride values for width and height, you can instead specify a tuple or +different stride values for height and width, you can instead specify a tuple or list (e.g., `stride=[3, 6]`). Our output tensor produced by `max_pooling2d()` (`pool1`) has a shape of -[batch_size, 14, 14, 32]: the 2x2 filter reduces width and -height by 50% each. +[batch_size, 14, 14, 32]: the 2x2 filter reduces height and width by 50% each. ### Convolutional Layer #2 and Pooling Layer #2 @@ -338,13 +342,11 @@ pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2) Note that convolutional layer #2 takes the output tensor of our first pooling layer (`pool1`) as input, and produces the tensor `conv2` as output. `conv2` -has a shape of [batch_size, 14, 14, 64], the same width -and height as `pool1` (due to `padding="same"`), and 64 channels for the 64 +has a shape of [batch_size, 14, 14, 64], the same height and width as `pool1` (due to `padding="same"`), and 64 channels for the 64 filters applied. Pooling layer #2 takes `conv2` as input, producing `pool2` as output. `pool2` -has shape [batch_size, 7, 7, 64] (50% reduction of width -and height from `conv2`). +has shape [batch_size, 7, 7, 64] (50% reduction of height and width from `conv2`). ### Dense Layer @@ -360,7 +362,7 @@ pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 64]) In the `reshape()` operation above, the `-1` signifies that the *`batch_size`* dimension will be dynamically calculated based on the number of examples in our -input data. Each example has 7 (`pool2` width) * 7 (`pool2` height) * 64 +input data. Each example has 7 (`pool2` height) * 7 (`pool2` width) * 64 (`pool2` channels) features, so we want the `features` dimension to have a value of 7 * 7 * 64 (3136 in total). The output tensor, `pool2_flat`, has shape [batch_size, 3136]. diff --git a/tensorflow/java/BUILD b/tensorflow/java/BUILD index acaf1a44eb..565c1cb8e0 100644 --- a/tensorflow/java/BUILD +++ b/tensorflow/java/BUILD @@ -314,6 +314,9 @@ tf_cc_test( srcs = [ "src/gen/cc/source_writer_test.cc", ], + data = [ + "src/gen/resources/test.java.snippet", + ], deps = [ ":java_op_gen_lib", "//tensorflow/core:lib", diff --git a/tensorflow/java/src/gen/cc/java_defs.h b/tensorflow/java/src/gen/cc/java_defs.h index 615cdc165b..59f8beaee7 100644 --- a/tensorflow/java/src/gen/cc/java_defs.h +++ b/tensorflow/java/src/gen/cc/java_defs.h @@ -17,10 +17,7 @@ limitations under the License. #define TENSORFLOW_JAVA_SRC_GEN_CC_JAVA_DEFS_H_ #include -#include -#include - -#include "tensorflow/core/platform/env.h" +#include namespace tensorflow { namespace java { @@ -104,17 +101,17 @@ class Type { description_ = description; return *this; } - const std::vector& parameters() const { return parameters_; } + const std::list& parameters() const { return parameters_; } Type& add_parameter(const Type& parameter) { parameters_.push_back(parameter); return *this; } - const std::vector& annotations() const { return annotations_; } + const std::list& annotations() const { return annotations_; } Type& add_annotation(const Annotation& annotation) { annotations_.push_back(annotation); return *this; } - const std::deque& supertypes() const { return supertypes_; } + const std::list& supertypes() const { return supertypes_; } Type& add_supertype(const Type& type) { if (type.kind_ == CLASS) { supertypes_.push_front(type); // keep superclass at the front of the list @@ -141,9 +138,9 @@ class Type { string name_; string package_; string description_; - std::vector parameters_; - std::vector annotations_; - std::deque supertypes_; + std::list parameters_; + std::list annotations_; + std::list supertypes_; }; // Definition of a Java annotation @@ -223,16 +220,12 @@ class Method { return_description_ = description; return *this; } - const std::vector& arguments() const { return arguments_; } - Method& add_arguments(const std::vector& args) { - arguments_.insert(arguments_.cend(), args.cbegin(), args.cend()); - return *this; - } + const std::list& arguments() const { return arguments_; } Method& add_argument(const Variable& var) { arguments_.push_back(var); return *this; } - const std::vector& annotations() const { return annotations_; } + const std::list& annotations() const { return annotations_; } Method& add_annotation(const Annotation& annotation) { annotations_.push_back(annotation); return *this; @@ -244,29 +237,13 @@ class Method { bool constructor_; string description_; string return_description_; - std::vector arguments_; - std::vector annotations_; + std::list arguments_; + std::list annotations_; Method(const string& name, const Type& return_type, bool constructor) : name_(name), return_type_(return_type), constructor_(constructor) {} }; -// A piece of code to read from a file. -class Snippet { - public: - static Snippet Create(const string& fname, Env* env = Env::Default()) { - return Snippet(fname, env); - } - const string& data() const { return data_; } - - private: - string data_; - - Snippet(const string& fname, Env* env) { - TF_CHECK_OK(ReadFileToString(env, fname, &data_)); - } -}; - } // namespace java } // namespace tensorflow diff --git a/tensorflow/java/src/gen/cc/source_writer.cc b/tensorflow/java/src/gen/cc/source_writer.cc index 2da81f2911..a02f75ad6e 100644 --- a/tensorflow/java/src/gen/cc/source_writer.cc +++ b/tensorflow/java/src/gen/cc/source_writer.cc @@ -14,49 +14,328 @@ limitations under the License. ==============================================================================*/ #include +#include +#include #include "tensorflow/java/src/gen/cc/source_writer.h" namespace tensorflow { +namespace java { -SourceWriter& SourceWriter::Append(const StringPiece& str) { - if (!str.empty()) { - if (newline_) { - DoAppend(left_margin_ + line_prefix_); - newline_ = false; - } - DoAppend(str); +SourceWriter::SourceWriter() { + // Push an empty generic namespace at start, for simplification. + generic_namespaces_.push(new GenericNamespace()); +} + +SourceWriter::~SourceWriter() { + // Remove empty generic namespace added at start as well as any other + // namespace objects that haven't been removed. + while (!generic_namespaces_.empty()) { + GenericNamespace* generic_namespace = generic_namespaces_.top(); + generic_namespaces_.pop(); + delete generic_namespace; } +} + +SourceWriter& SourceWriter::Indent(int tab) { + left_margin_.resize( + std::max(static_cast(left_margin_.size() + tab), 0), ' '); + return *this; +} + +SourceWriter& SourceWriter::Prefix(const char* line_prefix) { + line_prefix_ = line_prefix; return *this; } -SourceWriter& SourceWriter::Write(const string& str) { +SourceWriter& SourceWriter::Write(const StringPiece& str) { size_t line_pos = 0; do { size_t start_pos = line_pos; line_pos = str.find('\n', start_pos); if (line_pos != string::npos) { ++line_pos; - Append(StringPiece(str.data() + start_pos, line_pos - start_pos)); + Append(str.substr(start_pos, line_pos - start_pos)); newline_ = true; } else { - Append(StringPiece(str.data() + start_pos, str.size() - start_pos)); + Append(str.substr(start_pos, str.size() - start_pos)); } } while (line_pos != string::npos && line_pos < str.size()); return *this; } +SourceWriter& SourceWriter::WriteFromFile(const string& fname, Env* env) { + string data_; + TF_CHECK_OK(ReadFileToString(env, fname, &data_)); + return Write(data_); +} + +SourceWriter& SourceWriter::Append(const StringPiece& str) { + if (!str.empty()) { + if (newline_) { + DoAppend(left_margin_ + line_prefix_); + newline_ = false; + } + DoAppend(str); + } + return *this; +} + +SourceWriter& SourceWriter::AppendType(const Type& type) { + if (type.kind() == Type::Kind::GENERIC && type.name().empty()) { + Append("?"); + } else { + Append(type.name()); + } + if (!type.parameters().empty()) { + Append("<"); + for (const Type& t : type.parameters()) { + if (&t != &type.parameters().front()) { + Append(", "); + } + AppendType(t); + } + Append(">"); + } + return *this; +} + SourceWriter& SourceWriter::EndLine() { Append("\n"); newline_ = true; return *this; } -SourceWriter& SourceWriter::Indent(int tab) { - left_margin_.resize(std::max(static_cast(left_margin_.size() + tab), 0), - ' '); +SourceWriter& SourceWriter::BeginMethod(const Method& method, int modifiers) { + GenericNamespace* generic_namespace = PushGenericNamespace(modifiers); + if (!method.constructor()) { + generic_namespace->Visit(method.return_type()); + } + for (const Variable& v : method.arguments()) { + generic_namespace->Visit(v.type()); + } + EndLine(); + WriteDoc(method.description(), method.return_description(), + &method.arguments()); + if (!method.annotations().empty()) { + WriteAnnotations(method.annotations()); + } + WriteModifiers(modifiers); + if (!generic_namespace->declared_types().empty()) { + WriteGenerics(generic_namespace->declared_types()); + Append(" "); + } + if (!method.constructor()) { + AppendType(method.return_type()).Append(" "); + } + Append(method.name()).Append("("); + for (const Variable& v : method.arguments()) { + if (&v != &method.arguments().front()) { + Append(", "); + } + AppendType(v.type()).Append(v.variadic() ? "... " : " ").Append(v.name()); + } + return Append(")").BeginBlock(); +} + +SourceWriter& SourceWriter::EndMethod() { + EndBlock(); + PopGenericNamespace(); + return *this; +} + +SourceWriter& SourceWriter::BeginType(const Type& type, + const std::list* dependencies, int modifiers) { + if (!type.package().empty()) { + Append("package ").Append(type.package()).Append(";").EndLine(); + } + if (dependencies != nullptr && !dependencies->empty()) { + TypeImporter type_importer(type.package()); + for (const Type& t : *dependencies) { + type_importer.Visit(t); + } + EndLine(); + for (const string& s : type_importer.imports()) { + Append("import ").Append(s).Append(";").EndLine(); + } + } + return BeginInnerType(type, modifiers); +} + +SourceWriter& SourceWriter::BeginInnerType(const Type& type, int modifiers) { + GenericNamespace* generic_namespace = PushGenericNamespace(modifiers); + generic_namespace->Visit(type); + EndLine(); + WriteDoc(type.description()); + if (!type.annotations().empty()) { + WriteAnnotations(type.annotations()); + } + WriteModifiers(modifiers); + CHECK_EQ(Type::Kind::CLASS, type.kind()) << ": Not supported yet"; + Append("class ").Append(type.name()); + if (!generic_namespace->declared_types().empty()) { + WriteGenerics(generic_namespace->declared_types()); + } + if (!type.supertypes().empty()) { + bool first_interface = true; + for (const Type& t : type.supertypes()) { + if (t.kind() == Type::CLASS) { // superclass is always first in list + Append(" extends "); + } else if (first_interface) { + Append(" implements "); + first_interface = false; + } else { + Append(", "); + } + AppendType(t); + } + } + return BeginBlock(); +} + +SourceWriter& SourceWriter::EndType() { + EndBlock(); + PopGenericNamespace(); + return *this; +} + +SourceWriter& SourceWriter::WriteFields(const std::list& fields, + int modifiers) { + EndLine(); + for (const Variable& v : fields) { + WriteModifiers(modifiers); + AppendType(v.type()).Append(" ").Append(v.name()).Append(";"); + EndLine(); + } + return *this; +} + +SourceWriter& SourceWriter::WriteModifiers(int modifiers) { + if (modifiers & PUBLIC) { + Append("public "); + } else if (modifiers & PROTECTED) { + Append("protected "); + } else if (modifiers & PRIVATE) { + Append("private "); + } + if (modifiers & STATIC) { + Append("static "); + } + if (modifiers & FINAL) { + Append("final "); + } + return *this; +} + +SourceWriter& SourceWriter::WriteDoc(const string& description, + const string& return_description, const std::list* parameters) { + if (description.empty() && return_description.empty() + && (parameters == nullptr || parameters->empty())) { + return *this; // no doc to write + } + bool do_line_break = false; + Append("/**").EndLine().Prefix(" * "); + if (!description.empty()) { + Write(description).EndLine(); + do_line_break = true; + } + if (parameters != nullptr && !parameters->empty()) { + if (do_line_break) { + EndLine(); + do_line_break = false; + } + for (const Variable& v : *parameters) { + Append("@param ").Append(v.name()); + if (!v.description().empty()) { + Append(" ").Write(v.description()); + } + EndLine(); + } + } + if (!return_description.empty()) { + if (do_line_break) { + EndLine(); + do_line_break = false; + } + Append("@return ").Write(return_description).EndLine(); + } + return Prefix("").Append(" **/").EndLine(); +} + +SourceWriter& SourceWriter::WriteAnnotations( + const std::list& annotations) { + for (const Annotation& a : annotations) { + Append("@" + a.name()); + if (!a.attributes().empty()) { + Append("(").Append(a.attributes()).Append(")"); + } + EndLine(); + } return *this; } +SourceWriter& SourceWriter::WriteGenerics( + const std::list& generics) { + Append("<"); + for (const Type* pt : generics) { + if (pt != generics.front()) { + Append(", "); + } + Append(pt->name()); + if (!pt->supertypes().empty()) { + Append(" extends ").AppendType(pt->supertypes().front()); + } + } + return Append(">"); +} + +SourceWriter::GenericNamespace* SourceWriter::PushGenericNamespace( + int modifiers) { + GenericNamespace* generic_namespace; + if (modifiers & STATIC) { + generic_namespace = new GenericNamespace(); + } else { + generic_namespace = new GenericNamespace(generic_namespaces_.top()); + } + generic_namespaces_.push(generic_namespace); + return generic_namespace; +} + +void SourceWriter::PopGenericNamespace() { + GenericNamespace* generic_namespace = generic_namespaces_.top(); + generic_namespaces_.pop(); + delete generic_namespace; +} + +void SourceWriter::TypeVisitor::Visit(const Type& type) { + DoVisit(type); + for (const Type& t : type.parameters()) { + DoVisit(t); + } + for (const Annotation& t : type.annotations()) { + DoVisit(t); + } + for (const Type& t : type.supertypes()) { + DoVisit(t); + } +} + +void SourceWriter::GenericNamespace::DoVisit(const Type& type) { + // ignore non-generic parameters, wildcards and generics already declared + if (type.kind() == Type::GENERIC + && !type.IsWildcard() + && generic_names_.find(type.name()) == generic_names_.end()) { + declared_types_.push_back(&type); + generic_names_.insert(type.name()); + } +} + +void SourceWriter::TypeImporter::DoVisit(const Type& type) { + if (!type.package().empty() && type.package() != current_package_) { + imports_.insert(type.package() + '.' + type.name()); + } +} + +} // namespace java } // namespace tensorflow diff --git a/tensorflow/java/src/gen/cc/source_writer.h b/tensorflow/java/src/gen/cc/source_writer.h index bff26eb185..637072c0df 100644 --- a/tensorflow/java/src/gen/cc/source_writer.h +++ b/tensorflow/java/src/gen/cc/source_writer.h @@ -17,44 +17,23 @@ limitations under the License. #define TENSORFLOW_JAVA_SRC_GEN_CC_SOURCE_WRITER_H_ #include +#include +#include +#include #include "tensorflow/core/lib/core/stringpiece.h" #include "tensorflow/core/platform/env.h" +#include "tensorflow/java/src/gen/cc/java_defs.h" namespace tensorflow { +namespace java { -// A utility class for writing source code, normally generated at -// compile-time. -// -// Source writers are language-agnostic and therefore only expose generic -// methods common to most languages. Extend or wrap this class to implement -// language-specific features. -// -// Note: if you are looking to reuse this class for generating code in another -// language than Java, please do by moving it at the '//tensorflow/core/lib/io' -// level. +// A class for writing Java source code. class SourceWriter { public: - virtual ~SourceWriter() = default; - - // Returns true if the writer is at the beginnig of a new line - bool newline() const { return newline_; } - - // Appends a piece of code or text. - // - // It is expected that no newline character is present in the data provided, - // otherwise Write() must be used. - SourceWriter& Append(const StringPiece& str); + SourceWriter(); - // Writes a block of code or text. - // - // The data might potentially contain newline characters, therefore it will - // be scanned to ensure that each line is indented and prefixed properly, - // making it a bit slower than Append(). - SourceWriter& Write(const string& text); - - // Appends a newline character and start writing on a new line. - SourceWriter& EndLine(); + virtual ~SourceWriter(); // Indents following lines with white spaces. // @@ -75,18 +54,166 @@ class SourceWriter { // Indent(2)->Prefix("//") will result in prefixing lines with " //". // // An empty value ("") will remove any line prefix that was previously set. - SourceWriter& Prefix(const char* line_prefix) { - line_prefix_ = line_prefix; - return *this; + SourceWriter& Prefix(const char* line_prefix); + + // Writes a source code snippet. + // + // The data might potentially contain newline characters, therefore it will + // be scanned to ensure that each line is indented and prefixed properly, + // making it a bit slower than Append(). + SourceWriter& Write(const StringPiece& text); + + // Writes a source code snippet read from a file. + // + // All lines of the file at the provided path will be read and written back + // to the output of this writer in regard of its current attributes (e.g. + // the indentation, prefix, etc.) + SourceWriter& WriteFromFile(const string& fname, Env* env = Env::Default()); + + // Appends a piece of source code. + // + // It is expected that no newline character is present in the data provided, + // otherwise Write() must be used. + SourceWriter& Append(const StringPiece& str); + + // Appends a type to the current line. + // + // The type is written in its simple form (i.e. not prefixed by its package) + // and followed by any parameter types it has enclosed in brackets (<>). + SourceWriter& AppendType(const Type& type); + + // Appends a newline character. + // + // Data written after calling this method will start on a new line, in respect + // of the current indentation. + SourceWriter& EndLine(); + + // Begins a block of source code. + // + // This method appends a new opening brace to the current data and indent the + // next lines according to Google Java Style Guide. The block can optionally + // be preceded by an expression (e.g. Append("if(true)").BeginBlock();) + SourceWriter& BeginBlock() { + return Append(newline_ ? "{" : " {").EndLine().Indent(2); + } + + // Ends the current block of source code. + // + // This method appends a new closing brace to the current data and outdent the + // next lines back to the margin used before BeginBlock() was invoked. + SourceWriter& EndBlock() { + return Indent(-2).Append("}").EndLine(); } + // Begins to write a method. + // + // This method outputs the signature of the Java method from the data passed + // in the 'method' parameter and starts a new block. Additionnal modifiers can + // also be passed in parameter to define the accesses and the scope of this + // method. + SourceWriter& BeginMethod(const Method& method, int modifiers = 0); + + // Ends the current method. + // + // This method ends the block of code that has begun when invoking + // BeginMethod() prior to this. + SourceWriter& EndMethod(); + + // Begins to write the main type of a source file. + // + // This method outputs the declaration of the Java type from the data passed + // in the 'type' parameter and starts a new block. Additionnal modifiers can + // also be passed in parameter to define the accesses and the scope of this + // type. + // + // If not null, all types found in the 'dependencies' list will be imported + // before declaring the new type. + SourceWriter& BeginType(const Type& clazz, + const std::list* dependencies, int modifiers = 0); + + // Begins to write a new inner type. + // + // This method outputs the declaration of the Java type from the data passed + // in the 'type' parameter and starts a new block. Additionnal modifiers can + // also be passed in parameter to define the accesses and the scope of this + // type. + SourceWriter& BeginInnerType(const Type& type, int modifiers = 0); + + // Ends the current type. + // + // This method ends the block of code that has begun when invoking + // BeginType() or BeginInnerType() prior to this. + SourceWriter& EndType(); + + // Writes a list of variables as fields of a type. + // + // This method must be called within the definition of a type (see BeginType() + // or BeginInnerType()). Additional modifiers can also be passed in parameter + // to define the accesses and the scope of those fields. + SourceWriter& WriteFields(const std::list& fields, + int modifiers = 0); + protected: virtual void DoAppend(const StringPiece& str) = 0; private: + // A utility base class for visiting elements of a type. + class TypeVisitor { + public: + virtual ~TypeVisitor() = default; + void Visit(const Type& type); + + protected: + virtual void DoVisit(const Type& type) = 0; + }; + + // A utility class for keeping track of declared generics in a given scope. + class GenericNamespace : public TypeVisitor { + public: + GenericNamespace() = default; + explicit GenericNamespace(const GenericNamespace* parent) + : generic_names_(parent->generic_names_) {} + std::list declared_types() { + return declared_types_; + } + protected: + virtual void DoVisit(const Type& type); + + private: + std::list declared_types_; + std::set generic_names_; + }; + + // A utility class for collecting a list of import statements to declare. + class TypeImporter : public TypeVisitor { + public: + explicit TypeImporter(const string& current_package) + : current_package_(current_package) {} + virtual ~TypeImporter() = default; + const std::set imports() { + return imports_; + } + protected: + virtual void DoVisit(const Type& type); + + private: + string current_package_; + std::set imports_; + }; + string left_margin_; string line_prefix_; bool newline_ = true; + std::stack generic_namespaces_; + + SourceWriter& WriteModifiers(int modifiers); + SourceWriter& WriteDoc(const string& description, + const string& return_description = "", + const std::list* parameters = nullptr); + SourceWriter& WriteAnnotations(const std::list& annotations); + SourceWriter& WriteGenerics(const std::list& generics); + GenericNamespace* PushGenericNamespace(int modifiers); + void PopGenericNamespace(); }; // A writer that outputs source code into a file. @@ -128,6 +255,7 @@ class SourceBufferWriter : public SourceWriter { string* buffer_; }; +} // namespace java } // namespace tensorflow #endif // TENSORFLOW_JAVA_SRC_GEN_CC_SOURCE_WRITER_H_ diff --git a/tensorflow/java/src/gen/cc/source_writer_test.cc b/tensorflow/java/src/gen/cc/source_writer_test.cc index e973895754..4bce2fea70 100644 --- a/tensorflow/java/src/gen/cc/source_writer_test.cc +++ b/tensorflow/java/src/gen/cc/source_writer_test.cc @@ -13,11 +13,15 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/java/src/gen/cc/source_writer.h" +#include + #include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/platform/test.h" +#include "tensorflow/java/src/gen/cc/java_defs.h" +#include "tensorflow/java/src/gen/cc/source_writer.h" namespace tensorflow { +namespace java { namespace { TEST(AppendTest, SingleLineText) { @@ -211,5 +215,368 @@ TEST(MarginTest, EmptyPrefix) { ASSERT_STREQ(expected, writer.str().data()); } +TEST(StreamTest, BlocksAndLines) { + SourceBufferWriter writer; + + writer.Append("int i = 0;").EndLine() + .Append("int j = 10;").EndLine() + .Append("if (true)") + .BeginBlock() + .Append("int aLongWayToTen = 0;").EndLine() + .Append("while (++i <= j)") + .BeginBlock() + .Append("++aLongWayToTen;").EndLine() + .EndBlock() + .EndBlock(); + + const char* expected = + "int i = 0;\n" + "int j = 10;\n" + "if (true) {\n" + " int aLongWayToTen = 0;\n" + " while (++i <= j) {\n" + " ++aLongWayToTen;\n" + " }\n" + "}\n"; + ASSERT_STREQ(expected, writer.str().data()); +} + +TEST(StreamTest, Types) { + SourceBufferWriter writer; + Type generic = Type::Generic("T").add_supertype(Type::Class("Number")); + + writer.AppendType(Type::Int()).Append(", ") + .AppendType(Type::Class("String")).Append(", ") + .AppendType(generic).Append(", ") + .AppendType(Type::ListOf(generic)).Append(", ") + .AppendType(Type::ListOf(Type::IterableOf(generic))).Append(", ") + .AppendType(Type::ListOf(Type::Generic())); + + const char* expected = + "int, String, T, List, List>, List"; + ASSERT_STREQ(expected, writer.str().data()); +} + +TEST(StreamTest, FileSnippet) { + SourceBufferWriter writer; + const string fname = tensorflow::io::JoinPath( + tensorflow::testing::TensorFlowSrcRoot(), + "java/src/gen/resources/test.java.snippet"); + + writer.WriteFromFile(fname) + .BeginBlock() + .WriteFromFile(fname) + .EndBlock(); + + const char* expected = + "// Here is a little snippet\n" + "System.out.println(\"Hello!\");\n" + "{\n" + " // Here is a little snippet\n" + " System.out.println(\"Hello!\");\n" + "}\n"; + ASSERT_STREQ(expected, writer.str().data()); +} + +TEST(WriteType, SimpleClass) { + SourceBufferWriter writer; + Type clazz = Type::Class("Test", "org.tensorflow"); + + writer.BeginType(clazz, nullptr, PUBLIC).EndType(); + + const char* expected = + "package org.tensorflow;\n\n" + "public class Test {\n}\n"; + ASSERT_STREQ(expected, writer.str().data()); +} + +TEST(WriteType, SimpleClassWithDependencies) { + SourceBufferWriter writer; + Type clazz = Type::Class("Test", "org.tensorflow"); + std::list deps; + deps.push_back(Type::Class("TypeA", "org.test.sub")); + deps.push_back(Type::Class("TypeA", "org.test.sub")); // a second time + deps.push_back(Type::Class("TypeB", "org.other")); + deps.push_back(Type::Class("SamePackageType", "org.tensorflow")); + deps.push_back(Type::Class("NoPackageType")); + + writer.BeginType(clazz, &deps, PUBLIC).EndType(); + + const char* expected = + "package org.tensorflow;\n\n" + "import org.other.TypeB;\n" + "import org.test.sub.TypeA;\n\n" + "public class Test {\n}\n"; + ASSERT_STREQ(expected, writer.str().data()); +} + +TEST(WriteType, AnnotatedAndDocumentedClass) { + SourceBufferWriter writer; + Type clazz = Type::Class("Test", "org.tensorflow"); + clazz.description("This class has a\n

\nmultiline description."); + clazz.add_annotation(Annotation::Create("Bean")); + clazz.add_annotation(Annotation::Create("SuppressWarnings") + .attributes("\"rawtypes\"")); + + writer.BeginType(clazz, nullptr, PUBLIC).EndType(); + + const char* expected = + "package org.tensorflow;\n\n" + "/**\n" + " * This class has a\n" + " *

\n" + " * multiline description.\n" + " **/\n" + "@Bean\n" + "@SuppressWarnings(\"rawtypes\")\n" + "public class Test {\n}\n"; + ASSERT_STREQ(expected, writer.str().data()); +} + +TEST(WriteType, ParameterizedClass) { + SourceBufferWriter writer; + Type clazz = Type::Class("Test", "org.tensorflow"); + clazz.add_parameter(Type::Generic("T")); + clazz.add_parameter(Type::Generic("U").add_supertype(Type::Class("Number"))); + + writer.BeginType(clazz, nullptr, PUBLIC).EndType(); + + const char* expected = + "package org.tensorflow;\n\n" + "public class Test {\n}\n"; + ASSERT_STREQ(expected, writer.str().data()); +} + +TEST(WriteType, ParameterizedClassAndSupertypes) { + SourceBufferWriter writer; + Type clazz = Type::Class("Test", "org.tensorflow"); + Type type_t = Type::Generic("T"); + clazz.add_parameter(type_t); + Type type_u = Type::Generic("U").add_supertype(Type::Class("Number")); + clazz.add_parameter(type_u); + clazz.add_supertype(Type::Interface("Parametrizable").add_parameter(type_u)); + clazz.add_supertype(Type::Interface("Runnable")); + clazz.add_supertype(Type::Class("SuperTest").add_parameter(type_t)); + + writer.BeginType(clazz, nullptr, PUBLIC).EndType(); + + const char* expected = + "package org.tensorflow;\n\n" + "public class Test" + " extends SuperTest implements Parametrizable, Runnable {\n}\n"; + ASSERT_STREQ(expected, writer.str().data()); +} + +TEST(WriteType, ParameterizedClassFields) { + SourceBufferWriter writer; + Type clazz = Type::Class("Test", "org.tensorflow"); + Type type_t = Type::Generic("T").add_supertype(Type::Class("Number")); + clazz.add_parameter(type_t); + std::list static_fields; + static_fields.push_back(Variable::Create("field1", Type::Class("String"))); + std::list member_fields; + member_fields.push_back(Variable::Create("field2", Type::Class("String"))); + member_fields.push_back(Variable::Create("field3", type_t)); + + writer.BeginType(clazz, nullptr, PUBLIC) + .WriteFields(static_fields, STATIC | PUBLIC | FINAL) + .WriteFields(member_fields, PRIVATE) + .EndType(); + + const char* expected = + "package org.tensorflow;\n\n" + "public class Test {\n" + " \n" + " public static final String field1;\n" + " \n" + " private String field2;\n" + " private T field3;\n" + "}\n"; + ASSERT_STREQ(expected, writer.str().data()); +} + +TEST(WriteType, SimpleInnerClass) { + SourceBufferWriter writer; + Type clazz = Type::Class("Test", "org.tensorflow"); + Type inner_class = Type::Class("InnerTest"); + + writer.BeginType(clazz, nullptr, PUBLIC) + .BeginInnerType(inner_class, PUBLIC) + .EndType() + .EndType(); + + const char* expected = + "package org.tensorflow;\n\n" + "public class Test {\n" + " \n" + " public class InnerTest {\n" + " }\n" + "}\n"; + ASSERT_STREQ(expected, writer.str().data()); +} + +TEST(WriteType, StaticParameterizedInnerClass) { + SourceBufferWriter writer; + Type clazz = Type::Class("Test", "org.tensorflow"); + Type type_t = Type::Generic("T").add_supertype(Type::Class("Number")); + clazz.add_parameter(type_t); + Type inner_class = Type::Class("InnerTest"); + inner_class.add_parameter(type_t); + + writer.BeginType(clazz, nullptr, PUBLIC) + .BeginInnerType(inner_class, PUBLIC | STATIC) + .EndType() + .EndType(); + + const char* expected = + "package org.tensorflow;\n\n" + "public class Test {\n" + " \n" + " public static class InnerTest {\n" + " }\n" + "}\n"; + ASSERT_STREQ(expected, writer.str().data()); +} + +TEST(WriteMethod, SimpleMethod) { + SourceBufferWriter writer; + Type clazz = Type::Class("Test", "org.tensorflow"); + Method method = Method::Create("doNothing", Type::Void()); + + writer.BeginType(clazz, nullptr, PUBLIC) + .BeginMethod(method, PUBLIC).EndMethod() + .EndType(); + + const char* expected = + "package org.tensorflow;\n\n" + "public class Test {\n" + " \n" + " public void doNothing() {\n" + " }\n" + "}\n"; + ASSERT_STREQ(expected, writer.str().data()); +} + +TEST(WriteMethod, AnnotatedAndDocumentedMethod) { + SourceBufferWriter writer; + Type clazz = Type::Class("Test", "org.tensorflow"); + Method method = Method::Create("doNothing", Type::Void()); + method.description("This method has a\n

\nmultiline description."); + method.add_annotation(Annotation::Create("Override")); + method.add_annotation(Annotation::Create("SuppressWarnings") + .attributes("\"rawtypes\"")); + + writer.BeginType(clazz, nullptr, PUBLIC) + .BeginMethod(method, PUBLIC).EndMethod() + .EndType(); + + const char* expected = + "package org.tensorflow;\n\n" + "public class Test {\n" + " \n" + " /**\n" + " * This method has a\n" + " *

\n" + " * multiline description.\n" + " **/\n" + " @Override\n" + " @SuppressWarnings(\"rawtypes\")\n" + " public void doNothing() {\n" + " }\n" + "}\n"; + ASSERT_STREQ(expected, writer.str().data()); +} + +TEST(WriteMethod, DocumentedMethodWithArguments) { + SourceBufferWriter writer; + Type clazz = Type::Class("Test", "org.tensorflow"); + Method method = Method::Create("boolToInt", Type::Int()); + method.description("Converts a boolean to an int"); + method.return_description("int value for this boolean"); + method.add_argument(Variable::Create("b", Type::Boolean())); + Variable reverse = Variable::Create("reverse", Type::Boolean()); + reverse.description("if true, value is reversed"); + method.add_argument(reverse); + + writer.BeginType(clazz, nullptr, PUBLIC) + .BeginMethod(method, PUBLIC) + .Append("if (b && !reverse)") + .BeginBlock() + .Append("return 1;").EndLine() + .EndBlock() + .Append("return 0;").EndLine() + .EndMethod() + .EndType(); + + const char* expected = + "package org.tensorflow;\n\n" + "public class Test {\n" + " \n" + " /**\n" + " * Converts a boolean to an int\n" + " * \n" + " * @param b\n" + " * @param reverse if true, value is reversed\n" + " * @return int value for this boolean\n" + " **/\n" + " public int boolToInt(boolean b, boolean reverse) {\n" + " if (b && !reverse) {\n" + " return 1;\n" + " }\n" + " return 0;\n" + " }\n" + "}\n"; + ASSERT_STREQ(expected, writer.str().data()); +} + +TEST(WriteMethod, ParameterizedMethod) { + SourceBufferWriter writer; + Type clazz = Type::Class("Test", "org.tensorflow"); + Type type_t = Type::Generic("T").add_supertype(Type::Class("Number")); + clazz.add_parameter(type_t); + Method method = Method::Create("doNothing", type_t); + + writer.BeginType(clazz, nullptr, PUBLIC) + .BeginMethod(method, PUBLIC) + .Append("return null;").EndLine() + .EndMethod() + .EndType(); + + const char* expected = + "package org.tensorflow;\n\n" + "public class Test {\n" + " \n" + " public T doNothing() {\n" + " return null;\n" + " }\n" + "}\n"; + ASSERT_STREQ(expected, writer.str().data()); +} + +TEST(WriteMethod, StaticParameterizedMethod) { + SourceBufferWriter writer; + Type clazz = Type::Class("Test", "org.tensorflow"); + Type type_t = Type::Generic("T").add_supertype(Type::Class("Number")); + clazz.add_parameter(type_t); + Method method = Method::Create("doNothing", type_t); + + writer.BeginType(clazz, nullptr, PUBLIC) + .BeginMethod(method, PUBLIC | STATIC) + .Append("return null;").EndLine() + .EndMethod() + .EndType(); + + const char* expected = + "package org.tensorflow;\n\n" + "public class Test {\n" + " \n" + " public static T doNothing() {\n" + " return null;\n" + " }\n" + "}\n"; + ASSERT_STREQ(expected, writer.str().data()); +} + } // namespace +} // namespace java } // namespace tensorflow diff --git a/tensorflow/java/src/gen/resources/test.java.snippet b/tensorflow/java/src/gen/resources/test.java.snippet new file mode 100644 index 0000000000..5e412a9aef --- /dev/null +++ b/tensorflow/java/src/gen/resources/test.java.snippet @@ -0,0 +1,2 @@ +// Here is a little snippet +System.out.println("Hello!"); diff --git a/tensorflow/python/client/timeline_test.py b/tensorflow/python/client/timeline_test.py index 5e6b5acdb0..c046e9cfd4 100644 --- a/tensorflow/python/client/timeline_test.py +++ b/tensorflow/python/client/timeline_test.py @@ -24,6 +24,7 @@ from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session from tensorflow.python.client import timeline from tensorflow.python.framework import constant_op +from tensorflow.python.framework import test_util from tensorflow.python.framework import ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import variables @@ -155,9 +156,7 @@ class TimelineTest(test.TestCase): ctf = step_analysis.chrome_trace.format_to_string() self._validateTrace(ctf) maximums = step_analysis.allocator_maximums - cpuname = 'cpu' - if 'mklcpu' in maximums: - cpuname = 'mkl' + cpuname + cpuname = 'mklcpu' if test_util.IsMklEnabled() else 'cpu' self.assertTrue(cpuname in maximums) cpu_max = maximums[ 'cuda_host_bfc'] if 'cuda_host_bfc' in maximums else maximums[cpuname] diff --git a/tensorflow/python/eager/execution_callbacks.py b/tensorflow/python/eager/execution_callbacks.py index 535361498a..9a08259653 100644 --- a/tensorflow/python/eager/execution_callbacks.py +++ b/tensorflow/python/eager/execution_callbacks.py @@ -253,7 +253,7 @@ def add_execution_callback(callback): `f(op_type, op_name, attrs, inputs, outputs)`. `op_type` is the type of the operation that was just executed (e.g., `MatMul`). - `op_name` is the name of the operation that has was just executed. This + `op_name` is the name of the operation that was just executed. This name is set by the client who created the operation and can be `None` if it is unset. `attrs` contains the attributes of the operation as a `tuple` of diff --git a/tensorflow/python/kernel_tests/init_ops_test.py b/tensorflow/python/kernel_tests/init_ops_test.py index 36a86a25cc..1e5c118cbc 100644 --- a/tensorflow/python/kernel_tests/init_ops_test.py +++ b/tensorflow/python/kernel_tests/init_ops_test.py @@ -618,7 +618,7 @@ class ConvolutionDeltaOrthogonalInitializerTest(test.TestCase): for dtype in [dtypes.float32]: for kernel_size in [[3], [8], [3, 5], [2, 4], [3, 3, 3], [2, 2, 2]]: tol = 1e-2 - # Check orthogonality by computing the 2-norms of the inputs and ouputs. + # Check orthogonality by computing the 2-norms of the inputs and outputs. if len(kernel_size) == 1: shape = [4, 32, 64] convolution = convolutional.conv1d diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py index 7be8628073..fb53d9ffea 100644 --- a/tensorflow/python/ops/control_flow_ops.py +++ b/tensorflow/python/ops/control_flow_ops.py @@ -833,6 +833,9 @@ class GradLoopState(object): if outer_grad_state: outer_forward_ctxt = outer_grad_state.forward_context else: + if not hasattr(forward_ctxt, 'outer_context'): + raise ValueError("Failed to call gradients on a while loop without" + "properly serializing graph via MetaGraphDef") outer_forward_ctxt = forward_ctxt.outer_context # Add the forward loop counter. diff --git a/tensorflow/python/ops/ctc_ops.py b/tensorflow/python/ops/ctc_ops.py index 4b57e2de79..908e793902 100644 --- a/tensorflow/python/ops/ctc_ops.py +++ b/tensorflow/python/ops/ctc_ops.py @@ -218,7 +218,7 @@ def ctc_greedy_decoder(inputs, sequence_length, merge_repeated=True): The rows store: `[batch, time]`. `decoded.values`: Values vector, size `(total_decoded_outputs)`. The vector stores the decoded classes. - `decoded.shape`: Shape vector, size `(2)`. + `decoded.dense_shape`: Shape vector, size `(2)`. The shape values are: `[batch_size, max_decoded_length]` neg_sum_logits: A `float` matrix `(batch_size x 1)` containing, for the sequence found, the negative of the sum of the greatest logit at each @@ -265,7 +265,7 @@ def ctc_beam_search_decoder(inputs, sequence_length, beam_width=100, The rows store: [batch, time]. `decoded[j].values`: Values vector, size `(total_decoded_outputs[j])`. The vector stores the decoded classes for beam j. - `decoded[j].shape`: Shape vector, size `(2)`. + `decoded[j].dense_shape`: Shape vector, size `(2)`. The shape values are: `[batch_size, max_decoded_length[j]]`. log_probability: A `float` matrix `(batch_size x top_paths)` containing sequence log-probabilities. diff --git a/tensorflow/python/ops/custom_gradient.py b/tensorflow/python/ops/custom_gradient.py index 9eacac1b37..dfa07abfc6 100644 --- a/tensorflow/python/ops/custom_gradient.py +++ b/tensorflow/python/ops/custom_gradient.py @@ -95,7 +95,7 @@ def custom_gradient(f): if not context.executing_eagerly(): if kwargs: raise ValueError( - "The custom_gradient decorator currently suports keywords " + "The custom_gradient decorator currently supports keywords " "arguments only when eager execution is enabled.") name = "CustomGradient-%s" % ops.uid() args = [ops.convert_to_tensor(x) for x in args] diff --git a/tensorflow/python/ops/data_flow_ops.py b/tensorflow/python/ops/data_flow_ops.py index d2cc87555f..cb725199a8 100644 --- a/tensorflow/python/ops/data_flow_ops.py +++ b/tensorflow/python/ops/data_flow_ops.py @@ -1769,7 +1769,9 @@ class StagingArea(BaseStagingArea): its capacity. Args: - values: Tensor (or a tuple of Tensors) to place into the staging area. + values: A single tensor, a list or tuple of tensors, or a dictionary with + tensor values. The number of elements must match the length of the + list provided to the dtypes argument when creating the StagingArea. name: A name for the operation (optional). Returns: @@ -1780,11 +1782,12 @@ class StagingArea(BaseStagingArea): """ with ops.name_scope(name, "%s_put" % self._name, self._scope_vals(values)) as scope: + + if not isinstance(values, (list, tuple, dict)): + values = [values] # Hard-code indices for this staging area - indices = ( - list(six.moves.range(len(values))) - if isinstance(values, (list, tuple)) else None) + indices = list(six.moves.range(len(values))) vals, _ = self._check_put_dtypes(values, indices) with ops.colocate_with(self._coloc_op): diff --git a/tensorflow/python/ops/linalg/linear_operator.py b/tensorflow/python/ops/linalg/linear_operator.py index c7513d5b40..193c787baa 100644 --- a/tensorflow/python/ops/linalg/linear_operator.py +++ b/tensorflow/python/ops/linalg/linear_operator.py @@ -166,8 +166,7 @@ class LinearOperator(object): meaning the quadratic form `x^H A x` has positive real part for all nonzero `x`. Note that we do not require the operator to be self-adjoint to be positive-definite. See: - https://en.wikipedia.org/wiki/Positive-definite_matrix\ - #Extension_for_non_symmetric_matrices + https://en.wikipedia.org/wiki/Positive-definite_matrix#Extension_for_non-symmetric_matrices is_square: Expect that this operator acts like square [batch] matrices. name: A name for this `LinearOperator`. diff --git a/tensorflow/python/ops/linalg/linear_operator_composition.py b/tensorflow/python/ops/linalg/linear_operator_composition.py index ecd30e4d7e..0292bc51dc 100644 --- a/tensorflow/python/ops/linalg/linear_operator_composition.py +++ b/tensorflow/python/ops/linalg/linear_operator_composition.py @@ -134,8 +134,7 @@ class LinearOperatorComposition(linear_operator.LinearOperator): meaning the quadratic form `x^H A x` has positive real part for all nonzero `x`. Note that we do not require the operator to be self-adjoint to be positive-definite. See: - https://en.wikipedia.org/wiki/Positive-definite_matrix\ - #Extension_for_non_symmetric_matrices + https://en.wikipedia.org/wiki/Positive-definite_matrix#Extension_for_non-symmetric_matrices is_square: Expect that this operator acts like square [batch] matrices. name: A name for this `LinearOperator`. Default is the individual operators names joined with `_o_`. diff --git a/tensorflow/python/ops/linalg/linear_operator_diag.py b/tensorflow/python/ops/linalg/linear_operator_diag.py index e180e83026..5beaea65a5 100644 --- a/tensorflow/python/ops/linalg/linear_operator_diag.py +++ b/tensorflow/python/ops/linalg/linear_operator_diag.py @@ -132,8 +132,7 @@ class LinearOperatorDiag(linear_operator.LinearOperator): meaning the quadratic form `x^H A x` has positive real part for all nonzero `x`. Note that we do not require the operator to be self-adjoint to be positive-definite. See: - https://en.wikipedia.org/wiki/Positive-definite_matrix\ - #Extension_for_non_symmetric_matrices + https://en.wikipedia.org/wiki/Positive-definite_matrix#Extension_for_non-symmetric_matrices is_square: Expect that this operator acts like square [batch] matrices. name: A name for this `LinearOperator`. diff --git a/tensorflow/python/ops/linalg/linear_operator_full_matrix.py b/tensorflow/python/ops/linalg/linear_operator_full_matrix.py index f979fb37d6..5ba3b090ae 100644 --- a/tensorflow/python/ops/linalg/linear_operator_full_matrix.py +++ b/tensorflow/python/ops/linalg/linear_operator_full_matrix.py @@ -125,8 +125,7 @@ class LinearOperatorFullMatrix(linear_operator.LinearOperator): meaning the quadratic form `x^H A x` has positive real part for all nonzero `x`. Note that we do not require the operator to be self-adjoint to be positive-definite. See: - https://en.wikipedia.org/wiki/Positive-definite_matrix\ - #Extension_for_non_symmetric_matrices + https://en.wikipedia.org/wiki/Positive-definite_matrix#Extension_for_non-symmetric_matrices is_square: Expect that this operator acts like square [batch] matrices. name: A name for this `LinearOperator`. diff --git a/tensorflow/python/ops/linalg/linear_operator_identity.py b/tensorflow/python/ops/linalg/linear_operator_identity.py index 50f3d407e8..45929eb4e2 100644 --- a/tensorflow/python/ops/linalg/linear_operator_identity.py +++ b/tensorflow/python/ops/linalg/linear_operator_identity.py @@ -236,8 +236,7 @@ class LinearOperatorIdentity(BaseLinearOperatorIdentity): meaning the quadratic form `x^H A x` has positive real part for all nonzero `x`. Note that we do not require the operator to be self-adjoint to be positive-definite. See: - https://en.wikipedia.org/wiki/Positive-definite_matrix\ - #Extension_for_non_symmetric_matrices + https://en.wikipedia.org/wiki/Positive-definite_matrix#Extension_for_non-symmetric_matrices is_square: Expect that this operator acts like square [batch] matrices. assert_proper_shapes: Python `bool`. If `False`, only perform static checks that initialization and method arguments have proper shape. @@ -576,8 +575,7 @@ class LinearOperatorScaledIdentity(BaseLinearOperatorIdentity): meaning the quadratic form `x^H A x` has positive real part for all nonzero `x`. Note that we do not require the operator to be self-adjoint to be positive-definite. See: - https://en.wikipedia.org/wiki/Positive-definite_matrix\ - #Extension_for_non_symmetric_matrices + https://en.wikipedia.org/wiki/Positive-definite_matrix#Extension_for_non-symmetric_matrices is_square: Expect that this operator acts like square [batch] matrices. assert_proper_shapes: Python `bool`. If `False`, only perform static checks that initialization and method arguments have proper shape. diff --git a/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py b/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py index a5130188b6..c4d386ccb4 100644 --- a/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py +++ b/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py @@ -133,8 +133,7 @@ class LinearOperatorLowerTriangular(linear_operator.LinearOperator): meaning the quadratic form `x^H A x` has positive real part for all nonzero `x`. Note that we do not require the operator to be self-adjoint to be positive-definite. See: - https://en.wikipedia.org/wiki/Positive-definite_matrix\ - #Extension_for_non_symmetric_matrices + https://en.wikipedia.org/wiki/Positive-definite_matrix#Extension_for_non-symmetric_matrices is_square: Expect that this operator acts like square [batch] matrices. name: A name for this `LinearOperator`. diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py index 78bc024c0d..c6b2dcdf98 100644 --- a/tensorflow/python/training/distribute.py +++ b/tensorflow/python/training/distribute.py @@ -538,7 +538,7 @@ class DistributionStrategy(object): in the distributed vs. single tower cases. """ - # TODO(josh11b): Raise an exception if variable paritioning requested before + # TODO(josh11b): Raise an exception if variable partitioning requested before # we add support. # TODO(josh11b): Also `parameter_device_index` property? # TODO(josh11b): `map()` diff --git a/tensorflow/python/training/session_manager.py b/tensorflow/python/training/session_manager.py index 360e02fb44..a00ceb9021 100644 --- a/tensorflow/python/training/session_manager.py +++ b/tensorflow/python/training/session_manager.py @@ -229,10 +229,14 @@ class SessionManager(object): up to `max_wait_secs`, for recovery to succeed. If the model cannot be recovered successfully then it is initialized by - either running the provided `init_op`, or calling the provided `init_fn`. - The local_init_op is also run after init_op and init_fn, regardless of + running the `init_op` and calling `init_fn` if they are provided. + The `local_init_op` is also run after init_op and init_fn, regardless of whether the model was recovered successfully, but only if - ready_for_local_init_op passes. + `ready_for_local_init_op` passes. + + If the model is recovered from a checkpoint it is assumed that all + global variables have been initialized, in particular neither `init_op` + nor `init_fn` will be executed. It is an error if the model cannot be recovered and no `init_op` or `init_fn` or `local_init_op` are passed. diff --git a/tensorflow/tools/ci_build/install/install_golang.sh b/tensorflow/tools/ci_build/install/install_golang.sh index e1edd62cc5..124ad82e91 100755 --- a/tensorflow/tools/ci_build/install/install_golang.sh +++ b/tensorflow/tools/ci_build/install/install_golang.sh @@ -16,7 +16,7 @@ set -ex -GOLANG_URL="https://storage.googleapis.com/golang/go1.9.2.linux-amd64.tar.gz" +GOLANG_URL="https://storage.googleapis.com/golang/go1.10.linux-amd64.tar.gz" sudo mkdir -p /usr/local wget -q -O - "${GOLANG_URL}" | sudo tar -C /usr/local -xz diff --git a/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh b/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh index 7b2d7e1a56..d654b433e7 100644 --- a/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh +++ b/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh @@ -120,7 +120,9 @@ function run_configure_for_gpu_build { export TF_CUDA_VERSION=9.0 export CUDA_TOOLKIT_PATH="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v9.0" export TF_CUDNN_VERSION=7.0 - export CUDNN_INSTALL_PATH="C:/tools/cuda" + if [ -z "$CUDNN_INSTALL_PATH" ]; then + export CUDNN_INSTALL_PATH="C:/tools/cuda" + fi export TF_CUDA_COMPUTE_CAPABILITIES="3.7" if [ -z "$TF_ENABLE_XLA" ]; then export TF_ENABLE_XLA=0 diff --git a/tensorflow/tools/pip_package/build_pip_package.sh b/tensorflow/tools/pip_package/build_pip_package.sh index e2d212a0db..8f0cf8c3d1 100755 --- a/tensorflow/tools/pip_package/build_pip_package.sh +++ b/tensorflow/tools/pip_package/build_pip_package.sh @@ -139,7 +139,9 @@ function main() { fi mkdir "${TMPDIR}/tensorflow/aux-bin" # Install toco as a binary in aux-bin. - cp bazel-bin/tensorflow/contrib/lite/toco/toco ${TMPDIR}/tensorflow/aux-bin/ + # TODO(aselle): Re-enable this when we find a way to do it without doubling + # the whl size (over the limit). + # cp bazel-bin/tensorflow/contrib/lite/toco/toco ${TMPDIR}/tensorflow/aux-bin/ fi # protobuf pip package doesn't ship with header files. Copy the headers diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index cfad0f70c9..6511a50b3b 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -29,7 +29,7 @@ from setuptools.dist import Distribution # This version string is semver compatible, but incompatible with pip. # For pip, we will remove all '-' characters from this string, and use the # result for pip. -_VERSION = '1.7.0-rc1' +_VERSION = '1.7.0' REQUIRED_PACKAGES = [ 'absl-py >= 0.1.6', -- GitLab From f9c5e71104cb30583127fdc918591cc7604f17ca Mon Sep 17 00:00:00 2001 From: Loo Rong Jie Date: Wed, 11 Apr 2018 09:51:10 +0800 Subject: [PATCH 142/791] Add missing TF_ATTRIBUTE_WEAK for MSVC (#18303) --- tensorflow/core/platform/macros.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tensorflow/core/platform/macros.h b/tensorflow/core/platform/macros.h index 1b1faed703..3723968175 100644 --- a/tensorflow/core/platform/macros.h +++ b/tensorflow/core/platform/macros.h @@ -31,13 +31,14 @@ limitations under the License. __attribute__((__format__(__printf__, string_index, first_to_check))) #define TF_SCANF_ATTRIBUTE(string_index, first_to_check) \ __attribute__((__format__(__scanf__, string_index, first_to_check))) -#elif defined(COMPILER_MSVC) +#elif defined(_MSC_VER) // Non-GCC equivalents #define TF_ATTRIBUTE_NORETURN __declspec(noreturn) -#define TF_ATTRIBUTE_ALWAYS_INLINE +#define TF_ATTRIBUTE_ALWAYS_INLINE __forceinline #define TF_ATTRIBUTE_NOINLINE #define TF_ATTRIBUTE_UNUSED #define TF_ATTRIBUTE_COLD +#define TF_ATTRIBUTE_WEAK #define TF_MUST_USE_RESULT #define TF_PACKED #define TF_PRINTF_ATTRIBUTE(string_index, first_to_check) @@ -57,7 +58,7 @@ limitations under the License. #endif // Control visiblity outside .so -#if defined(COMPILER_MSVC) +#if defined(_WIN32) #ifdef TF_COMPILE_LIBRARY #define TF_EXPORT __declspec(dllexport) #else @@ -65,7 +66,7 @@ limitations under the License. #endif // TF_COMPILE_LIBRARY #else #define TF_EXPORT __attribute__((visibility("default"))) -#endif // COMPILER_MSVC +#endif // _WIN32 #ifdef __has_builtin #define TF_HAS_BUILTIN(x) __has_builtin(x) -- GitLab From 963ad0ff75d880861df20266652b263a9e32f0c7 Mon Sep 17 00:00:00 2001 From: Francois Chollet Date: Tue, 10 Apr 2018 18:50:58 -0700 Subject: [PATCH 143/791] Remove BN workaround for resource variable gradients bug that was recently fixed. PiperOrigin-RevId: 192388867 --- .../keras/_impl/keras/layers/normalization.py | 33 +------------------ 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/tensorflow/python/keras/_impl/keras/layers/normalization.py b/tensorflow/python/keras/_impl/keras/layers/normalization.py index b60d864ae5..b73025a5a8 100644 --- a/tensorflow/python/keras/_impl/keras/layers/normalization.py +++ b/tensorflow/python/keras/_impl/keras/layers/normalization.py @@ -33,7 +33,6 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import init_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn -from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import state_ops from tensorflow.python.platform import tf_logging as logging from tensorflow.python.training import distribute as distribute_lib @@ -171,7 +170,6 @@ class BatchNormalization(Layer): self.fused = fused self._bessels_correction_test_only = True - self._use_resource_variables = None if renorm: renorm_clipping = renorm_clipping or {} @@ -277,27 +275,6 @@ class BatchNormalization(Layer): for idx, x in enumerate(self.axis): self.axis[idx] = x + 1 # Account for added dimension - # BUG: when using fused BN with Resource Variables with a dynamic - # `training` argument in call, the cond - # `smart_cond( - # training, - # _fused_batch_norm_training, - # _fused_batch_norm_inference)` triggers None gradients for the - # variables gamma and beta. - # In this case we choose to force normal variables when possible. - # The bug will not occur of `training` is static, or when - # not using fused BN, or when in eager execution. - # TODO(fchollet): remove code below when bug is fixed. - use_resource = False - if context.executing_eagerly(): - use_resource = True # Eager execution requires resource variables. - elif not self.fused: - use_resource = True # Issue only exists with fused BN. - elif self._use_resource_variables is True: - use_resource = True # Case of a subclassed model, always use RVs. - if hasattr(self, '_scope'): - use_resource = None # Legacy layers, leave it to `add_weight`. - if self.scale: self.gamma = self.add_variable( name='gamma', @@ -306,7 +283,6 @@ class BatchNormalization(Layer): initializer=self.gamma_initializer, regularizer=self.gamma_regularizer, constraint=self.gamma_constraint, - use_resource=use_resource, trainable=True) else: self.gamma = None @@ -322,7 +298,6 @@ class BatchNormalization(Layer): initializer=self.beta_initializer, regularizer=self.beta_regularizer, constraint=self.beta_constraint, - use_resource=use_resource, trainable=True) else: self.beta = None @@ -531,13 +506,7 @@ class BatchNormalization(Layer): outputs = array_ops.reshape(outputs, original_shape) return outputs - # Gradient bug when using fused BN with dynamic `training` and resource - # variables. TODO(fchollet): remove workaround when bug fixed. - use_fused_bn = ( - self.fused and - (tf_utils.constant_value(training) is not None or - not isinstance(self.gamma, resource_variable_ops.ResourceVariable))) - if use_fused_bn: + if self.fused: outputs = self._fused_batch_norm(inputs, training=training) if self.virtual_batch_size is not None: # Currently never reaches here since fused_batch_norm does not support -- GitLab From b675450000753ff77e7a39a9ea84a59210781ea7 Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Tue, 10 Apr 2018 19:01:33 -0700 Subject: [PATCH 144/791] Checkpointable: remove colocation constraints from restore ops Mystery solved thanks to log_device_placement. PiperOrigin-RevId: 192389574 --- .../eager/python/checkpointable_utils_test.py | 10 ++++------ tensorflow/python/training/optimizer.py | 14 +++++++------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/tensorflow/contrib/eager/python/checkpointable_utils_test.py b/tensorflow/contrib/eager/python/checkpointable_utils_test.py index 36670aa210..b344d50e7f 100644 --- a/tensorflow/contrib/eager/python/checkpointable_utils_test.py +++ b/tensorflow/contrib/eager/python/checkpointable_utils_test.py @@ -764,9 +764,8 @@ class CheckpointingTests(test.TestCase): checkpoint_directory = self.get_temp_dir() root = checkpointable.Checkpointable() - with ops.device("/cpu:0"): - root.var = checkpointable_utils.add_variable( - root, name="var", initializer=0.) + root.var = checkpointable_utils.add_variable( + root, name="var", initializer=0.) optimizer = adam.AdamOptimizer(0.1) if context.executing_eagerly(): optimizer.minimize(root.var.read_value) @@ -796,9 +795,8 @@ class CheckpointingTests(test.TestCase): new_root).restore(no_slots_path) with self.assertRaises(AssertionError): no_slot_status.assert_consumed() - with ops.device("/cpu:0"): - new_root.var = checkpointable_utils.add_variable( - new_root, name="var", shape=[]) + new_root.var = checkpointable_utils.add_variable( + new_root, name="var", shape=[]) no_slot_status.assert_consumed() no_slot_status.run_restore_ops() self.assertEqual(12., self.evaluate(new_root.var)) diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py index 46a58a9adf..f126d3847b 100644 --- a/tensorflow/python/training/optimizer.py +++ b/tensorflow/python/training/optimizer.py @@ -818,13 +818,13 @@ class Optimizer( if restored_initial_value is not None: initial_value = restored_initial_value v = variable_scope.variable(initial_value, name=name, trainable=False) - # Restore this variable by name if necessary, but don't add a - # Checkpointable dependency. Optimizers return the current graph's - # non-slot variables from _checkpoint_dependencies explicitly rather - # than unconditionally adding dependencies (since there may be multiple - # non-slot variables with the same name in different graphs, trying to - # save all of them would result in errors). - self._handle_deferred_dependencies(name=name, checkpointable=v) + # Restore this variable by name if necessary, but don't add a + # Checkpointable dependency. Optimizers return the current graph's + # non-slot variables from _checkpoint_dependencies explicitly rather + # than unconditionally adding dependencies (since there may be multiple + # non-slot variables with the same name in different graphs, trying to + # save all of them would result in errors). + self._handle_deferred_dependencies(name=name, checkpointable=v) self._non_slot_dict[key] = v return v -- GitLab From 531e71b799bb8803d7357a501f38bed5c7141921 Mon Sep 17 00:00:00 2001 From: Asim Shankar Date: Tue, 10 Apr 2018 19:20:58 -0700 Subject: [PATCH 145/791] experimental C API: Fix compilation failure in Windows. The functions added in https://github.com/tensorflow/tensorflow/commit/be917027e37c5e8f21f6ba07f24bdbf072cf6dfd are temporary, and their existence breaks compilation in MSVC because of https://docs.microsoft.com/en-us/cpp/c-language/maximum-string-length and https://docs.microsoft.com/en-us/cpp/error-messages/compiler-errors-1/compiler-error-c2026 So just disabling it in Windows for now. PiperOrigin-RevId: 192391164 --- tensorflow/c/BUILD | 1 + tensorflow/c/c_api_experimental.cc | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD index 2367014cd0..8a9301d584 100644 --- a/tensorflow/c/BUILD +++ b/tensorflow/c/BUILD @@ -122,6 +122,7 @@ tf_cuda_library( "//tensorflow/core:core_cpu", "//tensorflow/core:framework", "//tensorflow/core:lib", + "//tensorflow/core:lib_platform", "//tensorflow/core:protos_all_cc", ], ) diff --git a/tensorflow/c/c_api_experimental.cc b/tensorflow/c/c_api_experimental.cc index e82a546092..9678ee926f 100644 --- a/tensorflow/c/c_api_experimental.cc +++ b/tensorflow/c/c_api_experimental.cc @@ -20,6 +20,7 @@ limitations under the License. #include "tensorflow/core/graph/graph.h" #include "tensorflow/core/graph/node_builder.h" #include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/platform.h" #include "tensorflow/core/protobuf/config.pb.h" using tensorflow::FunctionDef; @@ -189,6 +190,12 @@ library { // be deleted by calling TF_DeleteFunction. static std::vector CreateImagenetDatasetFunctions( const char* file_path, std::string* dataset_name, TF_Status* status) { +#if defined(PLATFORM_WINDOWS) + status->status = tensorflow::errors::Unimplemented( + "TF_MakeFileBasedIteratorGetNextWithDatasets in the experimental C API " + "is not implemented for Windows"); + return std::vector(); +#else const char* func_def = R"PREFIX( library { function { @@ -7067,6 +7074,7 @@ library { DCHECK(found); }; return CreateFunctionsFromTextProto(func_def, &mutate_proto_func, status); +#endif } // On success, returns a set of TF_Function instances encoding a dataset @@ -7076,6 +7084,12 @@ library { static std::vector CreateMNISTDatasetFunctions( const char* file_path, int batch_size, std::string* dataset_name, TF_Status* status) { +#if defined(PLATFORM_WINDOWS) + status->status = tensorflow::errors::Unimplemented( + "TF_MakeFileBasedIteratorGetNextWithDatasets in the experimental C API " + "is not implemented for Windows"); + return nullptr; +#else const char* func_def = R"PREFIX( library { function { @@ -8205,6 +8219,7 @@ library { DCHECK(found_batch_size); }; return CreateFunctionsFromTextProto(func_def, &mutate_proto_func, status); +#endif } // Adds the input functions to `graph`. On success, returns the created -- GitLab From 44adf97426c6e1f218010a4a16190b5ec0a9f4df Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 19:31:05 -0700 Subject: [PATCH 146/791] [XLA] Redesign: implement and test BatchNormXXX. PiperOrigin-RevId: 192391748 --- .../xla/client/xla_client/xla_builder.cc | 58 +++++++++++- tensorflow/compiler/xla/tests/BUILD | 4 +- .../xla/tests/batch_normalization_test.cc | 93 +++++++++---------- 3 files changed, 102 insertions(+), 53 deletions(-) diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc index ba76001c78..40bafdb5c1 100644 --- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc +++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc @@ -1489,21 +1489,73 @@ XlaOp XlaBuilder::ReduceWindowWithGeneralPadding( XlaOp XlaBuilder::BatchNormTraining(const XlaOp& operand, const XlaOp& scale, const XlaOp& offset, float epsilon, int64 feature_index) { - return UnimplementedOp(); + return NoteErrorOrReturn([&]() -> StatusOr { + HloInstructionProto instr; + + TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); + TF_ASSIGN_OR_RETURN(const Shape& scale_shape, GetShape(scale)); + TF_ASSIGN_OR_RETURN(const Shape& offset_shape, GetShape(offset)); + TF_ASSIGN_OR_RETURN( + *instr.mutable_shape(), + ShapeInference::InferBatchNormTrainingShape( + operand_shape, scale_shape, offset_shape, feature_index)); + + instr.set_epsilon(epsilon); + instr.set_feature_index(feature_index); + + return AddInstruction(std::move(instr), HloOpcode::kBatchNormTraining, + {operand, scale, offset}); + }); } XlaOp XlaBuilder::BatchNormInference(const XlaOp& operand, const XlaOp& scale, const XlaOp& offset, const XlaOp& mean, const XlaOp& variance, float epsilon, int64 feature_index) { - return UnimplementedOp(); + return NoteErrorOrReturn([&]() -> StatusOr { + HloInstructionProto instr; + + TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); + TF_ASSIGN_OR_RETURN(const Shape& scale_shape, GetShape(scale)); + TF_ASSIGN_OR_RETURN(const Shape& offset_shape, GetShape(offset)); + TF_ASSIGN_OR_RETURN(const Shape& mean_shape, GetShape(mean)); + TF_ASSIGN_OR_RETURN(const Shape& variance_shape, GetShape(variance)); + TF_ASSIGN_OR_RETURN(*instr.mutable_shape(), + ShapeInference::InferBatchNormInferenceShape( + operand_shape, scale_shape, offset_shape, + mean_shape, variance_shape, feature_index)); + + instr.set_epsilon(epsilon); + instr.set_feature_index(feature_index); + + return AddInstruction(std::move(instr), HloOpcode::kBatchNormInference, + {operand, scale, offset, mean, variance}); + }); } XlaOp XlaBuilder::BatchNormGrad(const XlaOp& operand, const XlaOp& scale, const XlaOp& batch_mean, const XlaOp& batch_var, const XlaOp& grad_output, float epsilon, int64 feature_index) { - return UnimplementedOp(); + return NoteErrorOrReturn([&]() -> StatusOr { + HloInstructionProto instr; + + TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); + TF_ASSIGN_OR_RETURN(const Shape& scale_shape, GetShape(scale)); + TF_ASSIGN_OR_RETURN(const Shape& batch_mean_shape, GetShape(batch_mean)); + TF_ASSIGN_OR_RETURN(const Shape& batch_var_shape, GetShape(batch_var)); + TF_ASSIGN_OR_RETURN(const Shape& grad_output_shape, GetShape(grad_output)); + TF_ASSIGN_OR_RETURN(*instr.mutable_shape(), + ShapeInference::InferBatchNormGradShape( + operand_shape, scale_shape, batch_mean_shape, + batch_var_shape, grad_output_shape, feature_index)); + + instr.set_epsilon(epsilon); + instr.set_feature_index(feature_index); + + return AddInstruction(std::move(instr), HloOpcode::kBatchNormGrad, + {operand, scale, batch_mean, batch_var, grad_output}); + }); } XlaOp XlaBuilder::CrossReplicaSum(const XlaOp& operand) { diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD index 2a2ef229ed..74ea1a0f39 100644 --- a/tensorflow/compiler/xla/tests/BUILD +++ b/tensorflow/compiler/xla/tests/BUILD @@ -860,11 +860,11 @@ xla_test( "//tensorflow/compiler/xla:test_helpers", "//tensorflow/compiler/xla:util", "//tensorflow/compiler/xla:xla_data_proto", - "//tensorflow/compiler/xla/client:computation", - "//tensorflow/compiler/xla/client:computation_builder", "//tensorflow/compiler/xla/client:global_data", "//tensorflow/compiler/xla/client:local_client", "//tensorflow/compiler/xla/client/lib:arithmetic", + "//tensorflow/compiler/xla/client/xla_client:xla_builder", + "//tensorflow/compiler/xla/client/xla_client:xla_computation", "//tensorflow/compiler/xla/service:hlo", "//tensorflow/compiler/xla/tests:client_library_test_base", "//tensorflow/compiler/xla/tests:hlo_test_base", diff --git a/tensorflow/compiler/xla/tests/batch_normalization_test.cc b/tensorflow/compiler/xla/tests/batch_normalization_test.cc index af8af99c79..f3dac75a44 100644 --- a/tensorflow/compiler/xla/tests/batch_normalization_test.cc +++ b/tensorflow/compiler/xla/tests/batch_normalization_test.cc @@ -19,10 +19,10 @@ limitations under the License. #include "tensorflow/compiler/xla/array2d.h" #include "tensorflow/compiler/xla/array4d.h" -#include "tensorflow/compiler/xla/client/computation.h" -#include "tensorflow/compiler/xla/client/computation_builder.h" #include "tensorflow/compiler/xla/client/lib/arithmetic.h" #include "tensorflow/compiler/xla/client/local_client.h" +#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h" +#include "tensorflow/compiler/xla/client/xla_client/xla_computation.h" #include "tensorflow/compiler/xla/literal_util.h" #include "tensorflow/compiler/xla/reference_util.h" #include "tensorflow/compiler/xla/service/hlo_computation.h" @@ -69,14 +69,12 @@ class BatchNormalizationTest CHECK_EQ(kY, input_array_.width()); } - ComputationDataHandle CheckShape(ComputationBuilder* b, - const ComputationDataHandle& operand, - const Shape& expected_shape) const { - std::unique_ptr actual_shape = - b->GetShape(operand).ConsumeValueOrDie(); - CHECK(ShapeUtil::Equal(expected_shape, *actual_shape)) + XlaOp CheckShape(XlaBuilder* b, const XlaOp& operand, + const Shape& expected_shape) const { + Shape actual_shape = b->GetShape(operand).ConsumeValueOrDie(); + CHECK(ShapeUtil::Equal(expected_shape, actual_shape)) << "want " << ShapeUtil::HumanString(expected_shape) << " got " - << ShapeUtil::HumanString(*actual_shape); + << ShapeUtil::HumanString(actual_shape); return operand; } @@ -102,7 +100,7 @@ INSTANTIATE_TEST_CASE_P(BatchNormalizationTestInstance, BatchNormalizationTest, #endif XLA_TEST_P(BatchNormalizationTest, SubtractInZ) { - ComputationBuilder builder(client_, "subtract_in_z_one_sample"); + XlaBuilder builder("subtract_in_z_one_sample"); auto x = builder.ConstantLiteral(input_literal_); auto y = builder.ConstantR1({3.14, 4.25}); builder.Sub(x, y, /*broadcast_dimensions=*/{1}); @@ -118,7 +116,7 @@ XLA_TEST_P(BatchNormalizationTest, SubtractInZ) { } XLA_TEST_P(BatchNormalizationTest, SquareTesseractElementwise) { - ComputationBuilder builder(client_, "square_tesseract_elementwise"); + XlaBuilder builder("square_tesseract_elementwise"); auto x = builder.ConstantLiteral(input_literal_); builder.SquareF32(x); @@ -135,9 +133,9 @@ XLA_TEST_P(BatchNormalizationTest, SquareTesseractElementwise) { } XLA_TEST_P(BatchNormalizationTest, SumToZ) { - ComputationBuilder builder(client_, "sum_to_z"); + XlaBuilder builder("sum_to_z"); auto input_activations = builder.ConstantLiteral(input_literal_); - Computation add = CreateScalarAddComputation(F32, &builder); + XlaComputation add = CreateScalarAddComputation(F32, &builder); // Reduce all but the Z dimension. builder.Reduce(input_activations, builder.ConstantR0(0.0f), add, {0, 2, 3}); @@ -147,24 +145,23 @@ XLA_TEST_P(BatchNormalizationTest, SumToZ) { } XLA_TEST_P(BatchNormalizationTest, SquareAndReduce) { - ComputationBuilder builder(client_, "square_and_reduce"); + XlaBuilder builder("square_and_reduce"); auto input_activations = builder.ConstantLiteral(input_literal_); auto set_means = builder.ConstantR1({2.f, 4.2f}); auto activation_deviations = builder.Sub(input_activations, set_means, /*broadcast_dimensions=*/{1}); - Computation add = CreateScalarAddComputation(F32, &builder); + XlaComputation add = CreateScalarAddComputation(F32, &builder); auto dev_squares = builder.SquareF32(activation_deviations); - auto sum_of_squares = builder.Reduce( - dev_squares, builder.ConstantR0(0.0f), add, {0, 2, 3}); + builder.Reduce(dev_squares, builder.ConstantR0(0.0f), add, {0, 2, 3}); std::vector expected = {18, 0.06}; ComputeAndCompareR1(&builder, expected, {}, error_spec_); } XLA_TEST_P(BatchNormalizationTest, VarianceToStddev) { - ComputationBuilder builder(client_, "variance_to_stddev"); + XlaBuilder builder("variance_to_stddev"); auto variance = builder.ConstantR1({6.f, .02f}); - auto sqrt = builder.SqrtF32(variance); + builder.SqrtF32(variance); std::vector expected = {2.44948974f, 0.14142136f}; ComputeAndCompareR1(&builder, expected, {}, error_spec_); @@ -173,13 +170,13 @@ XLA_TEST_P(BatchNormalizationTest, VarianceToStddev) { // Compare against a forward batch normalization example in the NN spec // reference. XLA_TEST_P(BatchNormalizationTest, SpecComparisonForward) { - ComputationBuilder builder(client_, "batch_normalize_per_spec"); + XlaBuilder builder("batch_normalize_per_spec"); auto input_activations = CheckShape(&builder, builder.ConstantLiteral(input_literal_), ShapeUtil::MakeShape(F32, {3, 2, 1, 1})); auto gamma = builder.ConstantR1({1.0, 1.0}); auto beta = builder.ConstantR1({0.0, 0.0}); - Computation add = CreateScalarAddComputation(F32, &builder); + XlaComputation add = CreateScalarAddComputation(F32, &builder); // Reduce all dimensions except dimension 1. Shape TwoElementVectorF32 = ShapeUtil::MakeShape(F32, {2}); auto sum = CheckShape( @@ -189,8 +186,8 @@ XLA_TEST_P(BatchNormalizationTest, SpecComparisonForward) { TwoElementVectorF32); auto input_shape = builder.GetShape(input_activations).ConsumeValueOrDie(); auto sum_shape = builder.GetShape(sum).ConsumeValueOrDie(); - auto count = builder.ConstantR0(ShapeUtil::ElementsIn(*input_shape) / - ShapeUtil::ElementsIn(*sum_shape)); + auto count = builder.ConstantR0(ShapeUtil::ElementsIn(input_shape) / + ShapeUtil::ElementsIn(sum_shape)); auto set_means = builder.Div(sum, count); const float kEpsilon = 1e-9f; @@ -233,7 +230,7 @@ XLA_TEST_P(BatchNormalizationTest, SpecComparisonForward) { XLA_TEST_P(BatchNormalizationTest, BasicTraining) { const int kFeatureIndex = 3; - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto operand = builder.ConstantR4FromArray4D( {{{{1.f, 2.f}}, {{3.f, 4.f}}}, {{{5.f, 6.f}}, {{7.f, 8.f}}}}); @@ -242,8 +239,8 @@ XLA_TEST_P(BatchNormalizationTest, BasicTraining) { auto offset = builder.ConstantR1({1.0f, 2.0f}); - auto tuple = builder.BatchNormTraining(operand, scale, offset, - /*epsilon=*/0.001, kFeatureIndex); + builder.BatchNormTraining(operand, scale, offset, + /*epsilon=*/0.001, kFeatureIndex); auto expected = Literal::MakeTuple( {Literal::CreateR4({{{{-1.6f, -2.0f}}, {{0.1f, 0.6f}}}, @@ -257,7 +254,7 @@ XLA_TEST_P(BatchNormalizationTest, BasicTraining) { XLA_TEST_P(BatchNormalizationTest, BasicTrainingOnSublane) { const int kFeatureIndex = 2; - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto operand = builder.ConstantR4FromArray4D( {{{{1.f}, {2.f}}, {{3.f}, {4.f}}}, {{{5.f}, {6.f}}, {{7.f}, {8.f}}}}); @@ -266,8 +263,8 @@ XLA_TEST_P(BatchNormalizationTest, BasicTrainingOnSublane) { auto offset = builder.ConstantR1({1.0f, 2.0f}); - auto tuple = builder.BatchNormTraining(operand, scale, offset, - /*epsilon=*/0.001, kFeatureIndex); + builder.BatchNormTraining(operand, scale, offset, + /*epsilon=*/0.001, kFeatureIndex); auto expected = Literal::MakeTuple( {Literal::CreateR4({{{{-1.6f}, {-2.0f}}, {{0.1f}, {0.6f}}}, @@ -282,23 +279,23 @@ XLA_TEST_P(BatchNormalizationTest, BasicTrainingOnSublane) { XLA_TEST_P(BatchNormalizationTest, TrainingWithFeatureOnLowDimension) { // Use 0 dimension as feature, tests layout analyzer. const int kFeatureIndex = 0; - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); - ComputationDataHandle h0; + XlaOp h0; auto operand = CreateR3Parameter(Array3D(260, 2, 2, 1.0f), /*parameter_number=*/0, "operand", &builder, &h0); - ComputationDataHandle h1; + XlaOp h1; auto scale = CreateR1Parameter(std::vector(260, 1.0f), /*parameter_number=*/1, "scale", &builder, &h1); - ComputationDataHandle h2; + XlaOp h2; auto offset = CreateR1Parameter(std::vector(260, 1.0f), /*parameter_number=*/2, "offset", &builder, &h2); - auto tuple = builder.BatchNormTraining(h0, h1, h2, - /*epsilon=*/1, kFeatureIndex); + builder.BatchNormTraining(h0, h1, h2, + /*epsilon=*/1, kFeatureIndex); auto expected = Literal::MakeTuple( {Literal::CreateR3FromArray3D(Array3D(260, 2, 2, 1.0f)) @@ -314,24 +311,24 @@ XLA_TEST_P(BatchNormalizationTest, TrainingWithFeatureOnLowDimension) { XLA_TEST_P(BatchNormalizationTest, LargeEpsilonTest) { // Test the correctness of choosing a large epsilon value. const int kFeatureIndex = 2; - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); - ComputationDataHandle h0; + XlaOp h0; auto operand = CreateR3Parameter({{{0.0f}, {10.0f}, {20.0f}, {30.0f}}}, /*parameter_number=*/0, "operand", &builder, &h0); - ComputationDataHandle h1; + XlaOp h1; auto scale = CreateR1Parameter(std::vector(1, 1.0f), /*parameter_number=*/1, "scale", &builder, &h1); - ComputationDataHandle h2; + XlaOp h2; auto offset = CreateR1Parameter(std::vector(1, 0.0f), /*parameter_number=*/2, "offset", &builder, &h2); // var = 125, mean = 15, epsilon = -100 - auto tuple = builder.BatchNormTraining(h0, h1, h2, - /*epsilon=*/-100, kFeatureIndex); + builder.BatchNormTraining(h0, h1, h2, + /*epsilon=*/-100, kFeatureIndex); auto expected = Literal::MakeTuple( {Literal::CreateR3FromArray3D({{{-3.0f}, {-1.0f}, {1.0f}, {3.0f}}}) @@ -346,7 +343,7 @@ XLA_TEST_P(BatchNormalizationTest, LargeEpsilonTest) { XLA_TEST_P(BatchNormalizationTest, BatchNormGradBasic) { const int kFeatureIndex = 2; - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto operand = builder.ConstantR4FromArray4D(Array4D(2, 2, 2, 1, 0.0f)); @@ -453,7 +450,7 @@ INSTANTIATE_TEST_CASE_P(BatchNormTest_Instantiation, BatchNormTestManySizes, XLA_TEST_P(BatchNormTestManySizes, RandomizedTrainingTests) { float epsilon = 0.001; - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); const std::vector& bounds = GetParam().bounds; Array4D input_array(bounds[0], bounds[1], bounds[2], bounds[3]); input_array.FillRandom(GetParam().random_value_var, @@ -553,7 +550,7 @@ XLA_TEST_P(BatchNormTestManySizes, RandomizedTrainingTests) { XLA_TEST_P(BatchNormTestManySizes, RandomizedInferencingTests) { float epsilon = 0.001; - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); const std::vector& bounds = GetParam().bounds; Array4D input_array(bounds[0], bounds[1], bounds[2], bounds[3]); input_array.FillRandom(GetParam().random_value_var, @@ -661,7 +658,7 @@ XLA_TEST_P(BatchNormTestManySizes, RandomizedInferencingTests) { XLA_TEST_P(BatchNormTestManySizes, RandomizedGradTests) { float epsilon = 0.001; - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); const std::vector& bounds = GetParam().bounds; Array4D input_array(bounds[0], bounds[1], bounds[2], bounds[3]); input_array.FillRandom(GetParam().random_value_var, @@ -828,9 +825,9 @@ XLA_TEST_P(BatchNormTestManySizes, RandomizedGradTests) { std::unique_ptr grad_output_data = client_->TransferToServer(*grad_output_literal).ConsumeValueOrDie(); - auto t = builder.BatchNormGrad(input_parameter, scale_parameter, - mean_parameter, var_parameter, - grad_output_parameter, epsilon, feature_index); + builder.BatchNormGrad(input_parameter, scale_parameter, mean_parameter, + var_parameter, grad_output_parameter, epsilon, + feature_index); auto expected = Literal::MakeTuple({expected_grad_activation.get(), -- GitLab From e7b1ab049d22119c7b649046be853ea88120f27a Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Tue, 10 Apr 2018 19:34:54 -0700 Subject: [PATCH 147/791] [StreamExecutor] Merge StreamExecutor's and XLA's StatusOr classes. StatusOr is a...complicated class to write. It's really not good to have two copies of it. They've diverged (the XLA one is more sophisticated), and this may be causing upstream build problems with gcc6. PiperOrigin-RevId: 192392111 --- tensorflow/stream_executor/BUILD | 2 + tensorflow/stream_executor/lib/statusor.h | 225 +--------------------- 2 files changed, 5 insertions(+), 222 deletions(-) diff --git a/tensorflow/stream_executor/BUILD b/tensorflow/stream_executor/BUILD index 1913fc20ee..80fc9ff292 100644 --- a/tensorflow/stream_executor/BUILD +++ b/tensorflow/stream_executor/BUILD @@ -33,6 +33,7 @@ cc_library( }), visibility = ["//visibility:public"], deps = [ + "//tensorflow/compiler/xla:statusor", "//tensorflow/core:lib", "@local_config_cuda//cuda:cuda_headers", ], @@ -45,6 +46,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ "//tensorflow/core:lib", + "//tensorflow/compiler/xla:statusor", "@local_config_cuda//cuda:cuda_headers", ] + if_static([":stream_executor_impl"]), ) diff --git a/tensorflow/stream_executor/lib/statusor.h b/tensorflow/stream_executor/lib/statusor.h index 138738ecab..3b97929b37 100644 --- a/tensorflow/stream_executor/lib/statusor.h +++ b/tensorflow/stream_executor/lib/statusor.h @@ -14,238 +14,19 @@ limitations under the License. ==============================================================================*/ // IWYU pragma: private, include "perftools/gputools/executor/stream_executor.h" -// -// StatusOr is the union of a Status object and a T -// object. StatusOr models the concept of an object that is either a -// usable value, or an error Status explaining why such a value is -// not present. To this end, StatusOr does not allow its Status -// value to be Status::OK. Further, StatusOr does not allow the -// contained pointer to be NULL. -// -// The primary use-case for StatusOr is as the return value of a -// function which may fail. -// -// Example client usage for a StatusOr, where T is not a pointer: -// -// StatusOr result = DoBigCalculationThatCouldFail(); -// if (result.ok()) { -// float answer = result.ValueOrDie(); -// printf("Big calculation yielded: %f", answer); -// } else { -// LOG(ERROR) << result.status(); -// } -// -// Example client usage for a StatusOr: -// -// StatusOr result = FooFactory::MakeNewFoo(arg); -// if (result.ok()) { -// std::unique_ptr foo(result.ValueOrDie()); -// foo->DoSomethingCool(); -// } else { -// LOG(ERROR) << result.status(); -// } -// -// Example client usage for a StatusOr>: -// -// StatusOr> result = FooFactory::MakeNewFoo(arg); -// if (result.ok()) { -// std::unique_ptr foo = std::move(result.ValueOrDie()); -// foo->DoSomethingCool(); -// } else { -// LOG(ERROR) << result.status(); -// } -// -// Example factory implementation returning StatusOr: -// -// StatusOr FooFactory::MakeNewFoo(int arg) { -// if (arg <= 0) { -// return Status(port::error::INVALID_ARGUMENT, -// "Arg must be positive"); -// } else { -// return new Foo(arg); -// } -// } -// #ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_STATUSOR_H_ #define TENSORFLOW_STREAM_EXECUTOR_LIB_STATUSOR_H_ -#include -#include "tensorflow/stream_executor/platform/port.h" -#include -#include - -#include "tensorflow/stream_executor/lib/error.h" -#include "tensorflow/stream_executor/lib/status.h" -#include "tensorflow/stream_executor/platform/logging.h" -#include "tensorflow/stream_executor/platform/port.h" +#include "tensorflow/compiler/xla/statusor.h" namespace perftools { namespace gputools { namespace port { -template -class StatusOr { - template friend class StatusOr; - - public: - // Construct a new StatusOr with Status::UNKNOWN status - StatusOr() : status_(error::UNKNOWN, "") {} - - // Construct a new StatusOr with the given non-ok status. After calling - // this constructor, calls to ValueOrDie() is invalid. - // - // NOTE: Not explicit - we want to use StatusOr as a return - // value, so it is convenient and sensible to be able to do 'return - // Status()' when the return type is StatusOr. - // - // REQUIRES: status != Status::OK. - // In optimized builds, passing Status::OK here will have the effect - // of passing PosixErrorSpace::EINVAL as a fallback. - StatusOr(const Status& status); // NOLINT - - // Construct a new StatusOr with the given value. If T is a plain pointer, - // value must not be NULL. After calling this constructor, calls to - // ValueOrDie() will succeed, and calls to status() will return OK. - // - // NOTE: Not explicit - we want to use StatusOr as a return type - // so it is convenient and sensible to be able to do 'return T()' - // when the return type is StatusOr. - // - // REQUIRES: if T is a plain pointer, value != NULL. - // In optimized builds, passing a NULL pointer here will have - // the effect of passing PosixErrorSpace::EINVAL as a fallback. - StatusOr(const T& value); // NOLINT - - // Conversion copy constructor, T must be copy constructible from U - template - StatusOr(const StatusOr& other) // NOLINT - : status_(other.status_), - value_(other.value_) {} - - // Conversion assignment operator, T must be assignable from U - template - StatusOr& operator=(const StatusOr& other) { - status_ = other.status_; - value_ = other.value_; - return *this; - } - - // Rvalue-reference overloads of the other constructors and assignment - // operators, to support move-only types and avoid unnecessary copying. - StatusOr(T&& value); // NOLINT - - // Move conversion operator to avoid unnecessary copy. - // T must be assignable from U. - // Not marked with explicit so the implicit conversion can happen. - template - StatusOr(StatusOr&& other) // NOLINT - : status_(std::move(other.status_)), - value_(std::move(other.value_)) {} - - // Move assignment operator to avoid unnecessary copy. - // T must be assignable from U - template - StatusOr& operator=(StatusOr&& other) { - status_ = std::move(other.status_); - value_ = std::move(other.value_); - return *this; - } - - // Returns a reference to our status. If this contains a T, then - // returns Status::OK. - const Status& status() const { return status_; } - - // Returns this->status().ok() - bool ok() const { return status_.ok(); } - - // Returns a reference to our current value, requires that this->ok(). - // If you need to initialize a T object from the stored value, - // ConsumeValueOrDie() may be more efficient. - const T& ValueOrDie() const; - T& ValueOrDie(); - - // Returns our current value, requires this->ok(). Use this if - // you would otherwise want to say std::move(s.ValueOrDie()), for example - // if you need to initialize a T object from the stored value and you don't - // need subsequent access to the stored value. It uses T's move constructor, - // if it has one, so it will work with move-only types, and will often be - // more efficient than ValueOrDie, but may leave the stored value - // in an arbitrary valid state. - T ConsumeValueOrDie(); - - private: - Status status_; - T value_; - - void CheckValueNotNull(const T& value); - - template - struct IsNull { - // For non-pointer U, a reference can never be NULL. - static inline bool IsValueNull(const U& t) { return false; } - }; - - template - struct IsNull { - static inline bool IsValueNull(const U* t) { return t == NULL; } - }; -}; - -//////////////////////////////////////////////////////////////////////////////// -// Implementation details for StatusOr - -template -StatusOr::StatusOr(const T& value) - : status_(), value_(value) { - CheckValueNotNull(value); -} - -template -const T& StatusOr::ValueOrDie() const { - TF_CHECK_OK(status_); - return value_; -} - -template -T& StatusOr::ValueOrDie() { - TF_CHECK_OK(status_); - return value_; -} - -template -T StatusOr::ConsumeValueOrDie() { - TF_CHECK_OK(status_); - return std::move(value_); -} - -template -StatusOr::StatusOr(const Status& status) - : status_(status) { - assert(!status.ok()); - if (status.ok()) { - status_ = - Status(error::INTERNAL, - "Status::OK is not a valid constructor argument to StatusOr"); - } -} - -template -StatusOr::StatusOr(T&& value) - : status_() { - CheckValueNotNull(value); - value_ = std::move(value); -} - +// Use XLA's StatusOr so we don't duplicate code. template -void StatusOr::CheckValueNotNull(const T& value) { - assert(!IsNull::IsValueNull(value)); - if (IsNull::IsValueNull(value)) { - status_ = - Status(error::INTERNAL, - "NULL is not a valid constructor argument to StatusOr"); - } -} +using StatusOr = ::xla::StatusOr; } // namespace port } // namespace gputools -- GitLab From f3180f3827ef1340f51408385f139143da55f07f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 19:44:00 -0700 Subject: [PATCH 148/791] Update ops-related pbtxt files. PiperOrigin-RevId: 192392702 --- .../core/ops/compat/ops_history.v1.pbtxt | 399 ++++++++++++++++++ tensorflow/core/ops/ops.pbtxt | 7 + 2 files changed, 406 insertions(+) diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt index fe4b7a7be0..12df60a2ae 100644 --- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt +++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt @@ -7610,6 +7610,111 @@ op { } } } +op { + name: "AvgPool3D" + input_arg { + name: "input" + type_attr: "T" + } + output_arg { + name: "output" + type_attr: "T" + } + attr { + name: "ksize" + type: "list(int)" + has_minimum: true + minimum: 5 + } + attr { + name: "strides" + type: "list(int)" + has_minimum: true + minimum: 5 + } + attr { + name: "padding" + type: "string" + allowed_values { + list { + s: "SAME" + s: "VALID" + } + } + } + attr { + name: "data_format" + type: "string" + default_value { + s: "NDHWC" + } + allowed_values { + list { + s: "NDHWC" + s: "NCDHW" + } + } + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_HALF + type: DT_BFLOAT16 + type: DT_FLOAT + type: DT_DOUBLE + } + } + } +} +op { + name: "AvgPool3DGrad" + input_arg { + name: "orig_input_shape" + type: DT_INT32 + } + input_arg { + name: "grad" + type_attr: "T" + } + output_arg { + name: "output" + type_attr: "T" + } + attr { + name: "ksize" + type: "list(int)" + has_minimum: true + minimum: 5 + } + attr { + name: "strides" + type: "list(int)" + has_minimum: true + minimum: 5 + } + attr { + name: "padding" + type: "string" + allowed_values { + list { + s: "SAME" + s: "VALID" + } + } + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + } + } + } +} op { name: "AvgPool3DGrad" input_arg { @@ -7646,6 +7751,19 @@ op { } } } + attr { + name: "data_format" + type: "string" + default_value { + s: "NDHWC" + } + allowed_values { + list { + s: "NDHWC" + s: "NCDHW" + } + } + } attr { name: "T" type: "type" @@ -7711,6 +7829,7 @@ op { type: "type" allowed_values { list { + type: DT_BFLOAT16 type: DT_FLOAT type: DT_DOUBLE } @@ -7771,6 +7890,7 @@ op { type: "type" allowed_values { list { + type: DT_HALF type: DT_BFLOAT16 type: DT_FLOAT type: DT_DOUBLE @@ -17318,6 +17438,76 @@ op { } } } +op { + name: "DepthwiseConv2dNativeBackpropFilter" + input_arg { + name: "input" + type_attr: "T" + } + input_arg { + name: "filter_sizes" + type: DT_INT32 + } + input_arg { + name: "out_backprop" + type_attr: "T" + } + output_arg { + name: "output" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_HALF + type: DT_BFLOAT16 + type: DT_FLOAT + type: DT_DOUBLE + } + } + } + attr { + name: "strides" + type: "list(int)" + } + attr { + name: "padding" + type: "string" + allowed_values { + list { + s: "SAME" + s: "VALID" + } + } + } + attr { + name: "data_format" + type: "string" + default_value { + s: "NHWC" + } + allowed_values { + list { + s: "NHWC" + s: "NCHW" + } + } + } + attr { + name: "dilations" + type: "list(int)" + default_value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} op { name: "DepthwiseConv2dNativeBackpropInput" input_arg { @@ -17486,6 +17676,76 @@ op { } } } +op { + name: "DepthwiseConv2dNativeBackpropInput" + input_arg { + name: "input_sizes" + type: DT_INT32 + } + input_arg { + name: "filter" + type_attr: "T" + } + input_arg { + name: "out_backprop" + type_attr: "T" + } + output_arg { + name: "output" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_HALF + type: DT_BFLOAT16 + type: DT_FLOAT + type: DT_DOUBLE + } + } + } + attr { + name: "strides" + type: "list(int)" + } + attr { + name: "padding" + type: "string" + allowed_values { + list { + s: "SAME" + s: "VALID" + } + } + } + attr { + name: "data_format" + type: "string" + default_value { + s: "NHWC" + } + allowed_values { + list { + s: "NHWC" + s: "NCHW" + } + } + } + attr { + name: "dilations" + type: "list(int)" + default_value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} op { name: "Dequantize" input_arg { @@ -28687,6 +28947,63 @@ op { } } } +op { + name: "MaxPool3D" + input_arg { + name: "input" + type_attr: "T" + } + output_arg { + name: "output" + type_attr: "T" + } + attr { + name: "ksize" + type: "list(int)" + has_minimum: true + minimum: 5 + } + attr { + name: "strides" + type: "list(int)" + has_minimum: true + minimum: 5 + } + attr { + name: "padding" + type: "string" + allowed_values { + list { + s: "SAME" + s: "VALID" + } + } + } + attr { + name: "data_format" + type: "string" + default_value { + s: "NDHWC" + } + allowed_values { + list { + s: "NDHWC" + s: "NCDHW" + } + } + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_HALF + type: DT_BFLOAT16 + type: DT_FLOAT + } + } + } +} op { name: "MaxPool3DGrad" input_arg { @@ -28958,6 +29275,88 @@ op { } } } +op { + name: "MaxPool3DGrad" + input_arg { + name: "orig_input" + type_attr: "TInput" + } + input_arg { + name: "orig_output" + type_attr: "TInput" + } + input_arg { + name: "grad" + type_attr: "T" + } + output_arg { + name: "output" + type_attr: "T" + } + attr { + name: "ksize" + type: "list(int)" + has_minimum: true + minimum: 5 + } + attr { + name: "strides" + type: "list(int)" + has_minimum: true + minimum: 5 + } + attr { + name: "padding" + type: "string" + allowed_values { + list { + s: "SAME" + s: "VALID" + } + } + } + attr { + name: "data_format" + type: "string" + default_value { + s: "NDHWC" + } + allowed_values { + list { + s: "NDHWC" + s: "NCDHW" + } + } + } + attr { + name: "T" + type: "type" + default_value { + type: DT_FLOAT + } + allowed_values { + list { + type: DT_HALF + type: DT_BFLOAT16 + type: DT_FLOAT + } + } + } + attr { + name: "TInput" + type: "type" + default_value { + type: DT_FLOAT + } + allowed_values { + list { + type: DT_HALF + type: DT_BFLOAT16 + type: DT_FLOAT + } + } + } +} op { name: "MaxPool3DGradGrad" input_arg { diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index 9950388357..6af77be148 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -2449,6 +2449,7 @@ op { type: "type" allowed_values { list { + type: DT_HALF type: DT_BFLOAT16 type: DT_FLOAT type: DT_DOUBLE @@ -2510,6 +2511,7 @@ op { type: "type" allowed_values { list { + type: DT_HALF type: DT_BFLOAT16 type: DT_FLOAT type: DT_DOUBLE @@ -7892,6 +7894,7 @@ op { type: "type" allowed_values { list { + type: DT_HALF type: DT_BFLOAT16 type: DT_FLOAT type: DT_DOUBLE @@ -7961,6 +7964,7 @@ op { type: "type" allowed_values { list { + type: DT_HALF type: DT_BFLOAT16 type: DT_FLOAT type: DT_DOUBLE @@ -14232,6 +14236,7 @@ op { type: "type" allowed_values { list { + type: DT_HALF type: DT_BFLOAT16 type: DT_FLOAT } @@ -14299,6 +14304,7 @@ op { } allowed_values { list { + type: DT_HALF type: DT_BFLOAT16 type: DT_FLOAT } @@ -14312,6 +14318,7 @@ op { } allowed_values { list { + type: DT_HALF type: DT_BFLOAT16 type: DT_FLOAT } -- GitLab From 0c0428e41289392be095bb07f5daa1a0c4557c8c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 20:48:57 -0700 Subject: [PATCH 149/791] [XLA] Redesign: implment and test CrossReplicaSum. PiperOrigin-RevId: 192397189 --- .../compiler/xla/client/xla_client/xla_builder.cc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc index 40bafdb5c1..3b96bc72be 100644 --- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc +++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc @@ -1559,7 +1559,17 @@ XlaOp XlaBuilder::BatchNormGrad(const XlaOp& operand, const XlaOp& scale, } XlaOp XlaBuilder::CrossReplicaSum(const XlaOp& operand) { - return UnimplementedOp(); + return NoteErrorOrReturn([&]() -> StatusOr { + HloInstructionProto instr; + + TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); + TF_ASSIGN_OR_RETURN( + *instr.mutable_shape(), + ShapeInference::InferCrossReplicaSumShape({&operand_shape})); + + return AddInstruction(std::move(instr), HloOpcode::kCrossReplicaSum, + {operand}); + }); } XlaOp XlaBuilder::SelectAndScatter( -- GitLab From f22655d09820f83881b8a2170eb51407956864d6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 21:42:14 -0700 Subject: [PATCH 150/791] [XLA] Redesgin: implement and test Gather, Conditional. Also support convert from/to proto for Gather. PiperOrigin-RevId: 192400659 --- .../xla/client/xla_client/xla_builder.cc | 47 ++++- .../compiler/xla/service/hlo_instruction.cc | 15 ++ tensorflow/compiler/xla/tests/BUILD | 3 +- .../compiler/xla/tests/conditional_test.cc | 192 +++++++++--------- .../xla/tests/gather_operation_test.cc | 8 +- 5 files changed, 160 insertions(+), 105 deletions(-) diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc index 3b96bc72be..c3c824a231 100644 --- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc +++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc @@ -1390,14 +1390,57 @@ XlaOp XlaBuilder::While(const XlaComputation& condition, XlaOp XlaBuilder::Gather(const XlaOp& input, const XlaOp& gather_indices, const GatherDimensionNumbers& dimension_numbers, tensorflow::gtl::ArraySlice window_bounds) { - return UnimplementedOp(); + return NoteErrorOrReturn([&]() -> StatusOr { + HloInstructionProto instr; + + TF_ASSIGN_OR_RETURN(const Shape& input_shape, GetShape(input)); + TF_ASSIGN_OR_RETURN(const Shape& gather_indices_shape, + GetShape(gather_indices)); + TF_ASSIGN_OR_RETURN( + *instr.mutable_shape(), + ShapeInference::InferGatherShape(input_shape, gather_indices_shape, + dimension_numbers, window_bounds)); + + *instr.mutable_gather_dimension_numbers() = dimension_numbers; + for (int64 bound : window_bounds) { + instr.add_gather_window_bounds(bound); + } + + return AddInstruction(std::move(instr), HloOpcode::kGather, + {input, gather_indices}); + }); } XlaOp XlaBuilder::Conditional(const XlaOp& predicate, const XlaOp& true_operand, const XlaComputation& true_computation, const XlaOp& false_operand, const XlaComputation& false_computation) { - return UnimplementedOp(); + return NoteErrorOrReturn([&]() -> StatusOr { + HloInstructionProto instr; + + TF_ASSIGN_OR_RETURN(const Shape& predicate_shape, GetShape(predicate)); + TF_ASSIGN_OR_RETURN(const Shape& true_operand_shape, + GetShape(true_operand)); + TF_ASSIGN_OR_RETURN(const ProgramShape& true_computation_shape, + true_computation.GetProgramShape()); + TF_ASSIGN_OR_RETURN(const Shape& false_operand_shape, + GetShape(false_operand)); + TF_ASSIGN_OR_RETURN(const ProgramShape& false_computation_shape, + false_computation.GetProgramShape()); + TF_ASSIGN_OR_RETURN( + *instr.mutable_shape(), + ShapeInference::InferConditionalShape( + predicate_shape, true_operand_shape, false_operand_shape, + true_computation_shape, false_computation_shape)); + + // The index of true_computation must be 0 and that of false computation + // must be 1. + AddCalledComputation(true_computation, &instr); + AddCalledComputation(false_computation, &instr); + + return AddInstruction(std::move(instr), HloOpcode::kConditional, + {predicate, true_operand, false_operand}); + }); } XlaOp XlaBuilder::Reduce( diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index 8149e47cb5..3629106a25 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -159,6 +159,14 @@ StatusOr> HloInstruction::CreateFromProto( instruction->fft_length_.push_back(fft_len); } + if (proto.has_gather_dimension_numbers()) { + instruction->gather_dimension_numbers_ = + MakeUnique(proto.gather_dimension_numbers()); + } + for (int64 bound : proto.gather_window_bounds()) { + instruction->gather_window_bounds_.push_back(bound); + } + return std::move(instruction); } @@ -2416,6 +2424,13 @@ HloInstructionProto HloInstruction::ToProto() const { proto.add_fft_length(fft_len); } + if (gather_dimension_numbers_ != nullptr) { + *proto.mutable_gather_dimension_numbers() = *gather_dimension_numbers_; + } + for (int64 bound : gather_window_bounds_) { + proto.add_gather_window_bounds(bound); + } + return proto; } diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD index 74ea1a0f39..1f90a44d8b 100644 --- a/tensorflow/compiler/xla/tests/BUILD +++ b/tensorflow/compiler/xla/tests/BUILD @@ -492,9 +492,10 @@ xla_test( tags = ["enable_for_xla_interpreter"], deps = [ "//tensorflow/compiler/xla:xla_data_proto", - "//tensorflow/compiler/xla/client:computation_builder", "//tensorflow/compiler/xla/client:global_data", "//tensorflow/compiler/xla/client:local_client", + "//tensorflow/compiler/xla/client/xla_client:xla_builder", + "//tensorflow/compiler/xla/client/xla_client:xla_computation", "//tensorflow/compiler/xla/tests:client_library_test_base", "//tensorflow/compiler/xla/tests:literal_test_util", "//tensorflow/compiler/xla/tests:xla_internal_test_main", diff --git a/tensorflow/compiler/xla/tests/conditional_test.cc b/tensorflow/compiler/xla/tests/conditional_test.cc index b917dee77b..7ff6706935 100644 --- a/tensorflow/compiler/xla/tests/conditional_test.cc +++ b/tensorflow/compiler/xla/tests/conditional_test.cc @@ -13,7 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/compiler/xla/client/computation_builder.h" +#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h" +#include "tensorflow/compiler/xla/client/xla_client/xla_computation.h" #include "tensorflow/compiler/xla/tests/client_library_test_base.h" #include "tensorflow/compiler/xla/tests/literal_test_util.h" #include "tensorflow/compiler/xla/tests/test_macros.h" @@ -23,8 +24,8 @@ namespace { class ConditionalOpTest : public ClientLibraryTestBase { protected: - Computation CreateR0ConstantComputation(float value) { - ComputationBuilder builder(client_, "Constant"); + XlaComputation CreateR0ConstantComputation(float value) { + XlaBuilder builder("Constant"); builder.Parameter(0, empty_tuple_, "tuple"); builder.ConstantR0(value); auto build_status = builder.Build(); @@ -32,16 +33,16 @@ class ConditionalOpTest : public ClientLibraryTestBase { return build_status.ConsumeValueOrDie(); } - Computation CreateR0IdentityComputation() { - ComputationBuilder builder(client_, "Identity"); + XlaComputation CreateR0IdentityComputation() { + XlaBuilder builder("Identity"); builder.Parameter(0, r0f32_, "x"); auto build_status = builder.Build(); EXPECT_IS_OK(build_status.status()); return build_status.ConsumeValueOrDie(); } - Computation CreateCeilComputation(const Shape& shape) { - ComputationBuilder builder(client_, "Ceil"); + XlaComputation CreateCeilComputation(const Shape& shape) { + XlaBuilder builder("Ceil"); auto param = builder.Parameter(0, shape, "param"); builder.Ceil(param); auto build_status = builder.Build(); @@ -49,16 +50,16 @@ class ConditionalOpTest : public ClientLibraryTestBase { return build_status.ConsumeValueOrDie(); } - Computation CreateR0CeilComputation() { + XlaComputation CreateR0CeilComputation() { return CreateCeilComputation(r0f32_); } - Computation CreateR1CeilComputation() { + XlaComputation CreateR1CeilComputation() { return CreateCeilComputation(r1s2f32_); } - Computation CreateFloorComputation(const Shape& shape) { - ComputationBuilder builder(client_, "Floor"); + XlaComputation CreateFloorComputation(const Shape& shape) { + XlaBuilder builder("Floor"); auto param = builder.Parameter(0, shape, "param"); builder.Floor(param); auto build_status = builder.Build(); @@ -66,17 +67,17 @@ class ConditionalOpTest : public ClientLibraryTestBase { return build_status.ConsumeValueOrDie(); } - Computation CreateR0FloorComputation() { + XlaComputation CreateR0FloorComputation() { return CreateFloorComputation(r0f32_); } - Computation CreateR1FloorComputation() { + XlaComputation CreateR1FloorComputation() { return CreateFloorComputation(r1s2f32_); } - Computation CreateTupleCeilComputation(const string& computation_name, - const Shape& tuple_shape) { - ComputationBuilder builder(client_, computation_name); + XlaComputation CreateTupleCeilComputation(const string& computation_name, + const Shape& tuple_shape) { + XlaBuilder builder(computation_name); auto tuple = builder.Parameter(0, tuple_shape, "tuple"); auto x = builder.GetTupleElement(tuple, 0); auto y = builder.GetTupleElement(tuple, 1); @@ -88,17 +89,17 @@ class ConditionalOpTest : public ClientLibraryTestBase { return build_status.ConsumeValueOrDie(); } - Computation CreateR0TupleCeilComputation() { + XlaComputation CreateR0TupleCeilComputation() { return CreateTupleCeilComputation("CeilR0", tuple_2_r0f32_); } - Computation CreateR1TupleCeilComputation() { + XlaComputation CreateR1TupleCeilComputation() { return CreateTupleCeilComputation("CeilR1", tuple_2_r1s2f32_); } - Computation CreateTupleFloorComputation(const string& computation_name, - const Shape& tuple_shape) { - ComputationBuilder builder(client_, computation_name); + XlaComputation CreateTupleFloorComputation(const string& computation_name, + const Shape& tuple_shape) { + XlaBuilder builder(computation_name); auto tuple = builder.Parameter(0, tuple_shape, "tuple"); auto x = builder.GetTupleElement(tuple, 0); auto y = builder.GetTupleElement(tuple, 1); @@ -110,17 +111,17 @@ class ConditionalOpTest : public ClientLibraryTestBase { return build_status.ConsumeValueOrDie(); } - Computation CreateR0TupleFloorComputation() { + XlaComputation CreateR0TupleFloorComputation() { return CreateTupleFloorComputation("FloorR0", tuple_2_r0f32_); } - Computation CreateR1TupleFloorComputation() { + XlaComputation CreateR1TupleFloorComputation() { return CreateTupleFloorComputation("FloorR1", tuple_2_r1s2f32_); } - Computation CreateTupleAddComputation(const string& computation_name, - const Shape& tuple_shape) { - ComputationBuilder builder(client_, computation_name); + XlaComputation CreateTupleAddComputation(const string& computation_name, + const Shape& tuple_shape) { + XlaBuilder builder(computation_name); auto tuple = builder.Parameter(0, tuple_shape, "tuple"); auto x = builder.GetTupleElement(tuple, 0); auto y = builder.GetTupleElement(tuple, 1); @@ -130,17 +131,17 @@ class ConditionalOpTest : public ClientLibraryTestBase { return build_status.ConsumeValueOrDie(); } - Computation CreateR0TupleAddComputation() { + XlaComputation CreateR0TupleAddComputation() { return CreateTupleAddComputation("AddR0", tuple_2_r0f32_); } - Computation CreateR1TupleAddComputation() { + XlaComputation CreateR1TupleAddComputation() { return CreateTupleAddComputation("AddR1", tuple_2_r1s2f32_); } - Computation CreateTupleSubComputation(const string& computation_name, - const Shape& tuple_shape) { - ComputationBuilder builder(client_, computation_name); + XlaComputation CreateTupleSubComputation(const string& computation_name, + const Shape& tuple_shape) { + XlaBuilder builder(computation_name); auto tuple = builder.Parameter(0, tuple_shape, "tuple"); auto x = builder.GetTupleElement(tuple, 0); auto y = builder.GetTupleElement(tuple, 1); @@ -150,11 +151,11 @@ class ConditionalOpTest : public ClientLibraryTestBase { return build_status.ConsumeValueOrDie(); } - Computation CreateR0TupleSubComputation() { + XlaComputation CreateR0TupleSubComputation() { return CreateTupleSubComputation("SubR0", tuple_2_r0f32_); } - Computation CreateR1TupleSubComputation() { + XlaComputation CreateR1TupleSubComputation() { return CreateTupleSubComputation("SubR1", tuple_2_r1s2f32_); } @@ -170,26 +171,25 @@ class ConditionalOpTest : public ClientLibraryTestBase { // Test true and false computations that do not take any parameters. XLA_TEST_F(ConditionalOpTest, Parameters0) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto pred = builder.ConstantR0(true); auto operands = builder.Tuple({}); auto true_computation = CreateR0ConstantComputation(56.0f); auto false_computation = CreateR0ConstantComputation(12.0f); - auto result = builder.Conditional(pred, operands, true_computation, operands, - false_computation); + builder.Conditional(pred, operands, true_computation, operands, + false_computation); ComputeAndCompareR0(&builder, 56.0f, {}, error_spec_); } // Test true and false computations that take in 1 parameter. XLA_TEST_F(ConditionalOpTest, Parameters1) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto pred = builder.ConstantR0(false); auto operand1 = builder.ConstantR0(56.0f); auto operand2 = builder.ConstantR0(12.0f); auto identity = CreateR0IdentityComputation(); - auto result = - builder.Conditional(pred, operand1, identity, operand2, identity); + builder.Conditional(pred, operand1, identity, operand2, identity); ComputeAndCompareR0(&builder, 12.0f, {}, error_spec_); } @@ -197,12 +197,12 @@ XLA_TEST_F(ConditionalOpTest, Parameters1) { // Test conditional with two different computations in the true and false cases // that take in different arguments. XLA_TEST_F(ConditionalOpTest, DiffComputationsDiffArgs) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto pred = builder.ConstantR0(false); auto operand1 = builder.ConstantR0(56.4f); auto operand2 = builder.ConstantR0(12.6f); - auto result = builder.Conditional(pred, operand1, CreateR0CeilComputation(), - operand2, CreateR0FloorComputation()); + builder.Conditional(pred, operand1, CreateR0CeilComputation(), operand2, + CreateR0FloorComputation()); ComputeAndCompareR0(&builder, 12.0f, {}, error_spec_); } @@ -210,11 +210,11 @@ XLA_TEST_F(ConditionalOpTest, DiffComputationsDiffArgs) { // Test conditional with two different computations in the true and false cases // that take in the same arguments. XLA_TEST_F(ConditionalOpTest, DiffComputationsSameArg) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto pred = builder.ConstantR0(false); auto operand = builder.ConstantR0(12.6f); - auto result = builder.Conditional(pred, operand, CreateR0CeilComputation(), - operand, CreateR0FloorComputation()); + builder.Conditional(pred, operand, CreateR0CeilComputation(), operand, + CreateR0FloorComputation()); ComputeAndCompareR0(&builder, 12.0f, {}, error_spec_); } @@ -222,12 +222,12 @@ XLA_TEST_F(ConditionalOpTest, DiffComputationsSameArg) { // Test conditional with the same computation in the true and false cases but // take in different arguments. XLA_TEST_F(ConditionalOpTest, SameComputationDiffArgs) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto pred = builder.ConstantR0(false); auto operand1 = builder.ConstantR0(56.4f); auto operand2 = builder.ConstantR0(12.6f); auto floor = CreateR0FloorComputation(); - auto result = builder.Conditional(pred, operand1, floor, operand2, floor); + builder.Conditional(pred, operand1, floor, operand2, floor); ComputeAndCompareR0(&builder, 12.0f, {}, error_spec_); } @@ -235,11 +235,11 @@ XLA_TEST_F(ConditionalOpTest, SameComputationDiffArgs) { // Test conditional with the same computation in the true and false cases that // take in the same arguments. XLA_TEST_F(ConditionalOpTest, SameComputationSameArg) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto pred = builder.ConstantR0(false); auto operand = builder.ConstantR0(12.6f); auto floor = CreateR0FloorComputation(); - auto result = builder.Conditional(pred, operand, floor, operand, floor); + builder.Conditional(pred, operand, floor, operand, floor); ComputeAndCompareR0(&builder, 12.0f, {}, error_spec_); } @@ -247,12 +247,12 @@ XLA_TEST_F(ConditionalOpTest, SameComputationSameArg) { // Test conditional with different instances of the same computation in the true // and false cases. XLA_TEST_F(ConditionalOpTest, SameComputationDiffInstances) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto pred = builder.ConstantR0(false); auto operand1 = builder.ConstantR0(56.4f); auto operand2 = builder.ConstantR0(12.6f); - auto result = builder.Conditional(pred, operand1, CreateR0FloorComputation(), - operand2, CreateR0FloorComputation()); + builder.Conditional(pred, operand1, CreateR0FloorComputation(), operand2, + CreateR0FloorComputation()); ComputeAndCompareR0(&builder, 12.0f, {}, error_spec_); } @@ -260,7 +260,7 @@ XLA_TEST_F(ConditionalOpTest, SameComputationDiffInstances) { // Test the case when a call invokes a computation that contains a conditional. XLA_TEST_F(ConditionalOpTest, ConditionalWithCall) { Shape r0bool = ShapeUtil::MakeShape(PRED, {}); - ComputationBuilder inner_builder(client_, TestName() + ".inner_conditional"); + XlaBuilder inner_builder(TestName() + ".inner_conditional"); auto pred_cond = inner_builder.Parameter(0, r0bool, "param0"); auto true_operand = inner_builder.Parameter(1, r0f32_, "param1"); auto false_operand = inner_builder.Parameter(2, r0f32_, "param2"); @@ -268,7 +268,7 @@ XLA_TEST_F(ConditionalOpTest, ConditionalWithCall) { false_operand, CreateR0FloorComputation()); auto inner_builder_result = inner_builder.Build(); - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto pred = builder.ConstantR0(false); auto operand1 = builder.ConstantR0(56.4f); auto operand2 = builder.ConstantR0(12.6f); @@ -281,14 +281,13 @@ XLA_TEST_F(ConditionalOpTest, ConditionalWithCall) { // Test true and false computations that take in 2 parameters and predicate is // true. XLA_TEST_F(ConditionalOpTest, Parameters2TrueBranch) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto pred = builder.ConstantR0(true); auto operand1 = builder.ConstantR0(56.0f); auto operand2 = builder.ConstantR0(12.0f); auto operands = builder.Tuple({operand1, operand2}); - auto result = - builder.Conditional(pred, operands, CreateR0TupleAddComputation(), - operands, CreateR0TupleSubComputation()); + builder.Conditional(pred, operands, CreateR0TupleAddComputation(), operands, + CreateR0TupleSubComputation()); ComputeAndCompareR0(&builder, 68.0f, {}, error_spec_); } @@ -296,14 +295,13 @@ XLA_TEST_F(ConditionalOpTest, Parameters2TrueBranch) { // Test true and false computations that take in 2 parameters and predicate is // false. XLA_TEST_F(ConditionalOpTest, Parameters2FalseBranch) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto pred = builder.ConstantR0(false); auto operand1 = builder.ConstantR0(56.0f); auto operand2 = builder.ConstantR0(12.0f); auto operands = builder.Tuple({operand1, operand2}); - auto result = - builder.Conditional(pred, operands, CreateR0TupleAddComputation(), - operands, CreateR0TupleSubComputation()); + builder.Conditional(pred, operands, CreateR0TupleAddComputation(), operands, + CreateR0TupleSubComputation()); ComputeAndCompareR0(&builder, 44.0f, {}, error_spec_); } @@ -311,14 +309,13 @@ XLA_TEST_F(ConditionalOpTest, Parameters2FalseBranch) { // Test true and false computations that take in 2 array parameters and // predicate is true. XLA_TEST_F(ConditionalOpTest, Parameters2ArrayTrueBranch) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto pred = builder.ConstantR0(true); auto operand1 = builder.ConstantR1({24.0f, 56.0f}); auto operand2 = builder.ConstantR1({10.0f, 11.0f}); auto operands = builder.Tuple({operand1, operand2}); - auto result = - builder.Conditional(pred, operands, CreateR1TupleAddComputation(), - operands, CreateR1TupleSubComputation()); + builder.Conditional(pred, operands, CreateR1TupleAddComputation(), operands, + CreateR1TupleSubComputation()); ComputeAndCompareR1(&builder, {34.0f, 67.0f}, {}, error_spec_); } @@ -326,21 +323,20 @@ XLA_TEST_F(ConditionalOpTest, Parameters2ArrayTrueBranch) { // Test true and false computations that take in 2 array parameters and // predicate is false. XLA_TEST_F(ConditionalOpTest, Parameters2ArrayFalseBranch) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto pred = builder.ConstantR0(false); auto operand1 = builder.ConstantR1({24.0f, 56.0f}); auto operand2 = builder.ConstantR1({10.0f, 11.0f}); auto operands = builder.Tuple({operand1, operand2}); - auto result = - builder.Conditional(pred, operands, CreateR1TupleAddComputation(), - operands, CreateR1TupleSubComputation()); + builder.Conditional(pred, operands, CreateR1TupleAddComputation(), operands, + CreateR1TupleSubComputation()); ComputeAndCompareR1(&builder, {14.0f, 45.0f}, {}, error_spec_); } // Test true and false computations that return a tuple of scalars. XLA_TEST_F(ConditionalOpTest, ReturnTupleOfScalars) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto pred = builder.ConstantR0(false); auto operands = builder.Tuple( {builder.ConstantR0(12.2f), builder.ConstantR0(25.6f)}); @@ -356,7 +352,7 @@ XLA_TEST_F(ConditionalOpTest, ReturnTupleOfScalars) { // Test true and false computations that return a tuple of arrays. XLA_TEST_F(ConditionalOpTest, ReturnTupleOfArrays) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto pred = builder.ConstantR0(true); auto operands = builder.Tuple({builder.ConstantR1({12.2f, 15.8f}), builder.ConstantR1({25.6f, 29.2f})}); @@ -373,7 +369,7 @@ XLA_TEST_F(ConditionalOpTest, ReturnTupleOfArrays) { // Test true and false computations that return a tuple of a predicate, a // scalar, and an array. XLA_TEST_F(ConditionalOpTest, ReturnTupleofPredicateScalarArray) { - ComputationBuilder true_builder(client_, TestName() + ".true"); + XlaBuilder true_builder(TestName() + ".true"); { true_builder.Parameter(0, empty_tuple_, "tuple"); auto true_pred = true_builder.ConstantR0(true); @@ -384,7 +380,7 @@ XLA_TEST_F(ConditionalOpTest, ReturnTupleofPredicateScalarArray) { auto true_builder_result = true_builder.Build(); EXPECT_IS_OK(true_builder_result.status()); - ComputationBuilder false_builder(client_, TestName() + ".false"); + XlaBuilder false_builder(TestName() + ".false"); { false_builder.Parameter(0, empty_tuple_, "tuple"); auto false_pred = false_builder.ConstantR0(false); @@ -395,7 +391,7 @@ XLA_TEST_F(ConditionalOpTest, ReturnTupleofPredicateScalarArray) { auto false_builder_result = false_builder.Build(); EXPECT_IS_OK(false_builder_result.status()); - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto pred = builder.ConstantR0(true); auto operands = builder.Tuple({}); builder.Conditional(pred, operands, true_builder_result.ConsumeValueOrDie(), @@ -411,7 +407,7 @@ XLA_TEST_F(ConditionalOpTest, ReturnTupleofPredicateScalarArray) { // Test true and false computations that return a nested tuple. XLA_TEST_F(ConditionalOpTest, ReturnNestedTuple) { - ComputationBuilder true_builder(client_, TestName() + ".true"); + XlaBuilder true_builder(TestName() + ".true"); { true_builder.Parameter(0, empty_tuple_, "tuple"); auto true_constant1 = true_builder.ConstantR0(12.2f); @@ -424,7 +420,7 @@ XLA_TEST_F(ConditionalOpTest, ReturnNestedTuple) { auto true_builder_result = true_builder.Build(); EXPECT_IS_OK(true_builder_result.status()); - ComputationBuilder false_builder(client_, TestName() + ".false"); + XlaBuilder false_builder(TestName() + ".false"); { false_builder.Parameter(0, empty_tuple_, "tuple"); auto false_constant1 = false_builder.ConstantR0(46.6f); @@ -438,7 +434,7 @@ XLA_TEST_F(ConditionalOpTest, ReturnNestedTuple) { auto false_builder_result = false_builder.Build(); EXPECT_IS_OK(false_builder_result.status()); - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto pred = builder.ConstantR0(false); auto operands = builder.Tuple({}); builder.Conditional(pred, operands, true_builder_result.ConsumeValueOrDie(), @@ -460,16 +456,16 @@ XLA_TEST_F(ConditionalOpTest, ReturnNestedTuple) { // params. XLA_TEST_F(ConditionalOpTest, ScalarOperandsFromExternalParams) { Shape r0bool = ShapeUtil::MakeShape(PRED, {}); - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); - ComputationDataHandle pred, operand1, operand2; + XlaOp pred, operand1, operand2; auto pred_arg = CreateR0Parameter(true, 0, "pred", &builder, &pred); auto operand1_param = CreateR0Parameter(56.3f, 1, "operand1", &builder, &operand1); auto operand2_param = CreateR0Parameter(12.7f, 2, "operand2", &builder, &operand2); - auto result = builder.Conditional(pred, operand1, CreateR0CeilComputation(), - operand2, CreateR0FloorComputation()); + builder.Conditional(pred, operand1, CreateR0CeilComputation(), operand2, + CreateR0FloorComputation()); ComputeAndCompareR0( &builder, 57.0f, @@ -480,16 +476,16 @@ XLA_TEST_F(ConditionalOpTest, ScalarOperandsFromExternalParams) { // Test conditional that takes in array operands in the form of external params. XLA_TEST_F(ConditionalOpTest, ArrayOperandsFromExternalParams) { Shape r0bool = ShapeUtil::MakeShape(PRED, {}); - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); - ComputationDataHandle pred, operand1, operand2; + XlaOp pred, operand1, operand2; auto pred_arg = CreateR0Parameter(false, 0, "pred", &builder, &pred); auto operand1_param = CreateR1Parameter({24.3f, 56.7f}, 1, "operand1", &builder, &operand1); auto operand2_param = CreateR1Parameter({10.2f, 11.6f}, 2, "operand2", &builder, &operand2); - auto result = builder.Conditional(pred, operand1, CreateR1CeilComputation(), - operand2, CreateR1FloorComputation()); + builder.Conditional(pred, operand1, CreateR1CeilComputation(), operand2, + CreateR1FloorComputation()); ComputeAndCompareR1( &builder, {10.0f, 11.0f}, @@ -499,7 +495,7 @@ XLA_TEST_F(ConditionalOpTest, ArrayOperandsFromExternalParams) { // Test the case where one conditional is nested within another. XLA_TEST_F(ConditionalOpTest, NestedConditionals) { - ComputationBuilder inner_builder(client_, TestName() + ".inner_conditional"); + XlaBuilder inner_builder(TestName() + ".inner_conditional"); { Shape r0bool = ShapeUtil::MakeShape(PRED, {}); Shape tuple_shape = ShapeUtil::MakeTupleShape({r0bool, r0f32_, r0f32_}); @@ -514,7 +510,7 @@ XLA_TEST_F(ConditionalOpTest, NestedConditionals) { auto inner_builder_result = inner_builder.Build(); EXPECT_IS_OK(inner_builder_result.status()); - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto pred1 = builder.ConstantR0(true); auto pred2 = builder.ConstantR0(false); auto operand1 = builder.ConstantR0(1.1f); @@ -529,7 +525,7 @@ XLA_TEST_F(ConditionalOpTest, NestedConditionals) { } XLA_TEST_F(ConditionalOpTest, ConditionalInNestedComputation) { - ComputationBuilder inner_builder(client_, TestName() + ".inner_conditional"); + XlaBuilder inner_builder(TestName() + ".inner_conditional"); { Shape r0bool = ShapeUtil::MakeShape(PRED, {}); Shape tuple_shape = ShapeUtil::MakeTupleShape({r0bool, r0f32_, r0f32_}); @@ -544,7 +540,7 @@ XLA_TEST_F(ConditionalOpTest, ConditionalInNestedComputation) { auto inner_builder_result = inner_builder.Build(); EXPECT_IS_OK(inner_builder_result.status()); - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto pred2 = builder.ConstantR0(false); auto operand1 = builder.ConstantR0(1.1f); auto operand2 = builder.ConstantR0(12.2f); @@ -556,7 +552,7 @@ XLA_TEST_F(ConditionalOpTest, ConditionalInNestedComputation) { // Test a mismatch in the shape of the true operand and true computation. XLA_TEST_F(ConditionalOpTest, ShapeMismatch) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto pred = builder.ConstantR0(true); auto operand1 = builder.ConstantR0(56.0f); auto operand2 = builder.ConstantR0(12.0f); @@ -573,27 +569,27 @@ XLA_TEST_F(ConditionalOpTest, ShapeMismatch) { XLA_TEST_F(ConditionalOpTest, SwappedInputsInSequentialConditionals) { Shape tuple_shape = ShapeUtil::MakeTupleShape({r0f32_, r0f32_}); - Computation swapper; + XlaComputation swapper; { - ComputationBuilder builder(client_, TestName() + ".swapper"); + XlaBuilder builder(TestName() + ".swapper"); auto param0 = builder.Parameter(0, tuple_shape, "sp0"); auto x = builder.GetTupleElement(param0, 0); auto y = builder.GetTupleElement(param0, 1); builder.Tuple({y, x}); swapper = builder.Build().ConsumeValueOrDie(); } - Computation forwarder; + XlaComputation forwarder; { - ComputationBuilder builder(client_, TestName() + ".forwarder"); + XlaBuilder builder(TestName() + ".forwarder"); auto param0 = builder.Parameter(0, tuple_shape, "fp0"); auto x = builder.GetTupleElement(param0, 0); auto y = builder.GetTupleElement(param0, 1); builder.Tuple({x, y}); forwarder = builder.Build().ConsumeValueOrDie(); } - Computation main; + XlaComputation main; { - ComputationBuilder builder(client_, TestName() + ".main"); + XlaBuilder builder(TestName() + ".main"); auto param0 = builder.Parameter(0, tuple_shape, "mp0"); auto x = builder.GetTupleElement(param0, 0); auto y = builder.GetTupleElement(param0, 1); @@ -605,7 +601,7 @@ XLA_TEST_F(ConditionalOpTest, SwappedInputsInSequentialConditionals) { } auto test_swap = [&](float a, float b) { - ComputationBuilder builder(client_, TestName()); + XlaBuilder builder(TestName()); auto x = builder.ConstantR0(a); auto y = builder.ConstantR0(b); auto tuple_operand = builder.Tuple({x, y}); diff --git a/tensorflow/compiler/xla/tests/gather_operation_test.cc b/tensorflow/compiler/xla/tests/gather_operation_test.cc index 9db68ff7a6..90496d55e6 100644 --- a/tensorflow/compiler/xla/tests/gather_operation_test.cc +++ b/tensorflow/compiler/xla/tests/gather_operation_test.cc @@ -405,7 +405,7 @@ class GatherClientLibraryTest : public ClientLibraryTestBase {}; // GPU and CPU_PARALLEL. XLA_TEST_F(GatherClientLibraryTest, DISABLED_ON_CPU_PARALLEL(DISABLED_ON_GPU(Basic))) { - // We create this HLO, but using the ComputationBuilder API. + // We create this HLO, but using the XlaBuilder API. // // ENTRY main { // operand = s32[3,3] parameter(0) @@ -418,7 +418,7 @@ XLA_TEST_F(GatherClientLibraryTest, // window_bounds={1, 3} // } - ComputationBuilder builder(client_, "gather_basic"); + XlaBuilder builder("gather_basic"); Shape operand_shape = ShapeUtil::MakeShape(S32, {3, 3}); Shape indices_shape = ShapeUtil::MakeShape(S32, {2}); @@ -443,8 +443,8 @@ XLA_TEST_F(GatherClientLibraryTest, client_->GetDeviceHandles(1)); xla::ExecutionOptions execution_options = CreateDefaultExecutionOptions(); *execution_options.add_device_handles() = devices[0]; - TF_ASSERT_OK_AND_ASSIGN(Computation computation, builder.Build()); - std::vector computation_instances = { + TF_ASSERT_OK_AND_ASSIGN(XlaComputation computation, builder.Build()); + std::vector computation_instances = { {computation, {operand_arg.get(), indices_arg.get()}, execution_options, -- GitLab From 785c484288913ed7989881483aefa3bee0cec015 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 22:29:13 -0700 Subject: [PATCH 151/791] [XLA] Redesign: implement ComputeHost. Also support convert from/to proto for ComputeHost. PiperOrigin-RevId: 192403660 --- tensorflow/compiler/xla/client/xla_client/xla_builder.cc | 8 +++++++- tensorflow/compiler/xla/service/hlo.proto | 4 ++++ tensorflow/compiler/xla/service/hlo_instruction.cc | 5 +++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc index c3c824a231..7ccdc2ded2 100644 --- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc +++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc @@ -1074,7 +1074,13 @@ XlaOp XlaBuilder::CustomCall(const string& call_target_name, XlaOp XlaBuilder::HostCompute(tensorflow::gtl::ArraySlice operands, const string& channel_name, int64 cost_estimate_ns, const Shape& shape) { - return UnimplementedOp(); + return NoteErrorOrReturn([&]() -> StatusOr { + HloInstructionProto instr; + *instr.mutable_shape() = shape; + instr.set_channel_name(channel_name); + instr.set_cost_estimate_ns(cost_estimate_ns); + return AddInstruction(std::move(instr), HloOpcode::kHostCompute, operands); + }); } XlaOp XlaBuilder::Complex( diff --git a/tensorflow/compiler/xla/service/hlo.proto b/tensorflow/compiler/xla/service/hlo.proto index 0b446c6547..8fd7f8945c 100644 --- a/tensorflow/compiler/xla/service/hlo.proto +++ b/tensorflow/compiler/xla/service/hlo.proto @@ -135,6 +135,10 @@ message HloInstructionProto { xla.GatherDimensionNumbers gather_dimension_numbers = 33; repeated int64 gather_window_bounds = 34; + // Compute Host. + string channel_name = 41; + int64 cost_estimate_ns = 42; + // The id of this instruction. int64 id = 35; diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index 3629106a25..a986bbd511 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -167,6 +167,9 @@ StatusOr> HloInstruction::CreateFromProto( instruction->gather_window_bounds_.push_back(bound); } + instruction->channel_name_ = proto.channel_name(); + instruction->cost_estimate_ns_ = proto.cost_estimate_ns(); + return std::move(instruction); } @@ -2430,6 +2433,8 @@ HloInstructionProto HloInstruction::ToProto() const { for (int64 bound : gather_window_bounds_) { proto.add_gather_window_bounds(bound); } + proto.set_channel_name(channel_name_); + proto.set_cost_estimate_ns(cost_estimate_ns_); return proto; } -- GitLab From 231146433a45ca8135e132ee0b48469798ca0b1f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 10 Apr 2018 22:44:36 -0700 Subject: [PATCH 152/791] [XLA] Fix the size of data buffer for sparse literals. PiperOrigin-RevId: 192404543 --- tensorflow/compiler/xla/literal_util.cc | 13 ++++++++++--- tensorflow/compiler/xla/literal_util.h | 5 +++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/tensorflow/compiler/xla/literal_util.cc b/tensorflow/compiler/xla/literal_util.cc index c2950c1faa..c315b4ff30 100644 --- a/tensorflow/compiler/xla/literal_util.cc +++ b/tensorflow/compiler/xla/literal_util.cc @@ -97,11 +97,18 @@ Literal::Literal(const Shape& shape, bool allocate_arrays) const Shape& subshape = piece.subshape(); if (ShapeUtil::IsArray(subshape)) { if (allocate_arrays) { - piece.set_buffer(new char[piece.size_bytes()]); if (LayoutUtil::IsSparseArray(subshape)) { + // For sparse arrays, the buffer must be of the size of the maximum + // number of sparse elements possible. + const int64 max_sparse_elements = + LayoutUtil::MaxSparseElements(subshape.layout()); + piece.set_buffer( + new char[max_sparse_elements * ShapeUtil::ByteSizeOfPrimitiveType( + subshape.element_type())]); piece.set_sparse_indices(new SparseIndexArray( - LayoutUtil::MaxSparseElements(subshape.layout()), - ShapeUtil::Rank(subshape))); + max_sparse_elements, ShapeUtil::Rank(subshape))); + } else { + piece.set_buffer(new char[piece.size_bytes()]); } } else { piece.set_buffer(nullptr); diff --git a/tensorflow/compiler/xla/literal_util.h b/tensorflow/compiler/xla/literal_util.h index a6a3dffeb7..8aa19222dc 100644 --- a/tensorflow/compiler/xla/literal_util.h +++ b/tensorflow/compiler/xla/literal_util.h @@ -1287,12 +1287,13 @@ void Literal::PopulateSparse(SparseIndexArray indices, CHECK_LE(num_elements, max_elements); CHECK_EQ(num_elements, indices.index_count()); auto root_data = root_piece().data(); - root_data.remove_suffix(max_elements - values.size()); + // Piece::data() returns an ArraySlice of size equal to the number of indices + // in the SparseIndexArray. So there is no need to adjust the size of the data + // here. It is enough to just copy the incoming values into the data buffer. std::copy(values.begin(), values.end(), root_data.begin()); *this->root_piece().sparse_indices() = std::move(indices); if (sort) { auto root_data = this->root_piece().data(); - root_data.remove_suffix(root_data.size() - num_elements); this->root_piece().sparse_indices()->SortWithValues(root_data); } DCHECK(this->root_piece().sparse_indices()->Validate(shape())); -- GitLab From 6accb84d8437cb915e23d83673c233f5084aad68 Mon Sep 17 00:00:00 2001 From: Nick Felt Date: Tue, 10 Apr 2018 23:44:12 -0700 Subject: [PATCH 153/791] Create FileWriter <-> tf.contrib.summary compatibility layer This provides an implementation of FileWriter, activated by passing in a `session` parameter to the constructor, that is backed by session.run'ing graph ops that manipulate a tf.contrib.summary.create_file_writer() instance. Because tf.contrib.summary.SummaryWriters are backed by shared resources in the graph, this makes it possible to have a FileWriter and a tf.contrib.summary.SummaryWriter that both write to the same events file. This change includes some related smaller changes: - Factors out training_utils.py into a separate target to avoid a cyclic dep - Moves contrib/summary/summary_ops.py to python/ops/summary_ops_v2.py - Adds SummaryWriter.init(), .flush(), and .close() op-returning methods - Changes create_file_writer() `name` arg to default to logdir prefixed by `logdir:` so shared resources are scoped by logdir by default - Fixes a bug with tf.contrib.summary.flush() `writer` arg - Makes create_file_writer()'s max_queue arg behave as documented - Adds more testing for existing tf.contrib.summary API PiperOrigin-RevId: 192408079 --- tensorflow/contrib/eager/python/BUILD | 6 +- tensorflow/contrib/eager/python/evaluator.py | 2 +- .../contrib/eager/python/metrics_impl.py | 2 +- .../contrib/eager/python/metrics_test.py | 2 +- tensorflow/contrib/summary/BUILD | 33 +-- tensorflow/contrib/summary/summary.py | 40 +-- .../contrib/summary/summary_ops_graph_test.py | 197 ++++++++++++++- .../contrib/summary/summary_ops_test.py | 113 ++++++++- .../contrib/summary/summary_test_internal.py | 60 ----- .../contrib/summary/summary_test_util.py | 2 +- .../tensorboard/db/summary_file_writer.cc | 2 +- tensorflow/contrib/tpu/BUILD | 2 +- .../contrib/tpu/python/tpu/tpu_estimator.py | 2 +- tensorflow/python/BUILD | 54 +++- .../ops/summary_ops_v2.py} | 68 +++-- .../summary/writer/event_file_writer_v2.py | 140 +++++++++++ tensorflow/python/summary/writer/writer.py | 40 ++- .../python/summary/writer/writer_test.py | 233 ++++++++++++++---- .../tensorflow.summary.-file-writer.pbtxt | 2 +- 19 files changed, 797 insertions(+), 203 deletions(-) delete mode 100644 tensorflow/contrib/summary/summary_test_internal.py rename tensorflow/{contrib/summary/summary_ops.py => python/ops/summary_ops_v2.py} (90%) create mode 100644 tensorflow/python/summary/writer/event_file_writer_v2.py diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD index 4e088503bf..d97048405d 100644 --- a/tensorflow/contrib/eager/python/BUILD +++ b/tensorflow/contrib/eager/python/BUILD @@ -120,13 +120,13 @@ py_library( visibility = ["//tensorflow:internal"], deps = [ "//tensorflow/contrib/eager/python:checkpointable_utils", - "//tensorflow/contrib/summary:summary_ops", "//tensorflow/python:array_ops", "//tensorflow/python:control_flow_ops", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", "//tensorflow/python:init_ops", "//tensorflow/python:math_ops", + "//tensorflow/python:summary_ops_v2", "//tensorflow/python:util", "//tensorflow/python:variable_scope", "//tensorflow/python/eager:context", @@ -140,11 +140,11 @@ py_test( srcs_version = "PY2AND3", deps = [ ":metrics", - "//tensorflow/contrib/summary:summary_ops", "//tensorflow/contrib/summary:summary_test_util", "//tensorflow/python:array_ops", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", + "//tensorflow/python:summary_ops_v2", "//tensorflow/python:training", "//tensorflow/python/eager:context", "//tensorflow/python/eager:test", @@ -161,10 +161,10 @@ py_library( deps = [ ":datasets", ":metrics", - "//tensorflow/contrib/summary:summary_ops", "//tensorflow/python:control_flow_ops", "//tensorflow/python:errors", "//tensorflow/python:framework_ops", + "//tensorflow/python:summary_ops_v2", "//tensorflow/python/eager:context", "//tensorflow/python/eager:function", "@six_archive//:six", diff --git a/tensorflow/contrib/eager/python/evaluator.py b/tensorflow/contrib/eager/python/evaluator.py index 37c8f0d47a..7949a3f6da 100644 --- a/tensorflow/contrib/eager/python/evaluator.py +++ b/tensorflow/contrib/eager/python/evaluator.py @@ -22,12 +22,12 @@ import six from tensorflow.contrib.eager.python import datasets from tensorflow.contrib.eager.python import metrics -from tensorflow.contrib.summary import summary_ops from tensorflow.python.eager import context from tensorflow.python.eager import function from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import summary_ops_v2 as summary_ops class Evaluator(object): diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py index 2f2347736a..907f9204c2 100644 --- a/tensorflow/contrib/eager/python/metrics_impl.py +++ b/tensorflow/contrib/eager/python/metrics_impl.py @@ -20,7 +20,6 @@ from __future__ import print_function import re -from tensorflow.contrib.summary import summary_ops from tensorflow.python.eager import context from tensorflow.python.eager import function from tensorflow.python.framework import dtypes @@ -29,6 +28,7 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import init_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops import summary_ops_v2 as summary_ops from tensorflow.python.ops import variable_scope from tensorflow.python.training import checkpointable diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py index 15ac889191..28f5f286eb 100644 --- a/tensorflow/contrib/eager/python/metrics_test.py +++ b/tensorflow/contrib/eager/python/metrics_test.py @@ -23,7 +23,6 @@ import tempfile from tensorflow.contrib.eager.python import checkpointable_utils from tensorflow.contrib.eager.python import metrics -from tensorflow.contrib.summary import summary_ops from tensorflow.contrib.summary import summary_test_util from tensorflow.python.eager import context from tensorflow.python.eager import test @@ -31,6 +30,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops +from tensorflow.python.ops import summary_ops_v2 as summary_ops from tensorflow.python.training import training_util diff --git a/tensorflow/contrib/summary/BUILD b/tensorflow/contrib/summary/BUILD index fda1367b15..f88b03ec4c 100644 --- a/tensorflow/contrib/summary/BUILD +++ b/tensorflow/contrib/summary/BUILD @@ -15,7 +15,6 @@ py_test( srcs = ["summary_ops_test.py"], srcs_version = "PY2AND3", deps = [ - ":summary_ops", ":summary_test_util", "//tensorflow/python:array_ops", "//tensorflow/python:errors", @@ -23,6 +22,7 @@ py_test( "//tensorflow/python:framework_test_lib", "//tensorflow/python:platform", "//tensorflow/python:state_ops", + "//tensorflow/python:summary_ops_v2", "//tensorflow/python:training", "//tensorflow/python/eager:function", "//tensorflow/python/eager:test", @@ -35,7 +35,6 @@ py_test( srcs = ["summary_ops_graph_test.py"], srcs_version = "PY2AND3", deps = [ - ":summary_ops", ":summary_test_util", "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", @@ -44,31 +43,9 @@ py_test( "//tensorflow/python:control_flow_ops", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", + "//tensorflow/python:summary_ops_v2", "//tensorflow/python:training", - "@six_archive//:six", - ], -) - -py_library( - name = "summary_ops", - srcs = ["summary_ops.py"], - srcs_version = "PY2AND3", - visibility = ["//tensorflow:internal"], - deps = [ - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:constant_op", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:layers_base", - "//tensorflow/python:math_ops", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:summary_op_util", - "//tensorflow/python:summary_ops_gen", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python/eager:context", + "//tensorflow/python:variables", "@six_archive//:six", ], ) @@ -79,7 +56,7 @@ py_library( srcs_version = "PY2AND3", visibility = ["//tensorflow:internal"], deps = [ - ":summary_ops", + "//tensorflow/python:summary_ops_v2", ], ) @@ -92,8 +69,10 @@ py_library( visibility = ["//tensorflow:internal"], deps = [ "//tensorflow/core:protos_all_py", + "//tensorflow/python:framework_test_lib", "//tensorflow/python:lib", "//tensorflow/python:platform", + "//tensorflow/python:summary_ops_v2", "@org_sqlite//:python", ], ) diff --git a/tensorflow/contrib/summary/summary.py b/tensorflow/contrib/summary/summary.py index 2d6d7ea6a3..99ced53e11 100644 --- a/tensorflow/contrib/summary/summary.py +++ b/tensorflow/contrib/summary/summary.py @@ -61,23 +61,23 @@ from __future__ import division from __future__ import print_function # pylint: disable=unused-import -from tensorflow.contrib.summary.summary_ops import all_summary_ops -from tensorflow.contrib.summary.summary_ops import always_record_summaries -from tensorflow.contrib.summary.summary_ops import audio -from tensorflow.contrib.summary.summary_ops import create_db_writer -from tensorflow.contrib.summary.summary_ops import create_file_writer -from tensorflow.contrib.summary.summary_ops import create_summary_file_writer -from tensorflow.contrib.summary.summary_ops import eval_dir -from tensorflow.contrib.summary.summary_ops import flush -from tensorflow.contrib.summary.summary_ops import generic -from tensorflow.contrib.summary.summary_ops import graph -from tensorflow.contrib.summary.summary_ops import histogram -from tensorflow.contrib.summary.summary_ops import image -from tensorflow.contrib.summary.summary_ops import import_event -from tensorflow.contrib.summary.summary_ops import initialize -from tensorflow.contrib.summary.summary_ops import never_record_summaries -from tensorflow.contrib.summary.summary_ops import record_summaries_every_n_global_steps -from tensorflow.contrib.summary.summary_ops import scalar -from tensorflow.contrib.summary.summary_ops import should_record_summaries -from tensorflow.contrib.summary.summary_ops import summary_writer_initializer_op -from tensorflow.contrib.summary.summary_ops import SummaryWriter +from tensorflow.python.ops.summary_ops_v2 import all_summary_ops +from tensorflow.python.ops.summary_ops_v2 import always_record_summaries +from tensorflow.python.ops.summary_ops_v2 import audio +from tensorflow.python.ops.summary_ops_v2 import create_db_writer +from tensorflow.python.ops.summary_ops_v2 import create_file_writer +from tensorflow.python.ops.summary_ops_v2 import create_summary_file_writer +from tensorflow.python.ops.summary_ops_v2 import eval_dir +from tensorflow.python.ops.summary_ops_v2 import flush +from tensorflow.python.ops.summary_ops_v2 import generic +from tensorflow.python.ops.summary_ops_v2 import graph +from tensorflow.python.ops.summary_ops_v2 import histogram +from tensorflow.python.ops.summary_ops_v2 import image +from tensorflow.python.ops.summary_ops_v2 import import_event +from tensorflow.python.ops.summary_ops_v2 import initialize +from tensorflow.python.ops.summary_ops_v2 import never_record_summaries +from tensorflow.python.ops.summary_ops_v2 import record_summaries_every_n_global_steps +from tensorflow.python.ops.summary_ops_v2 import scalar +from tensorflow.python.ops.summary_ops_v2 import should_record_summaries +from tensorflow.python.ops.summary_ops_v2 import summary_writer_initializer_op +from tensorflow.python.ops.summary_ops_v2 import SummaryWriter diff --git a/tensorflow/contrib/summary/summary_ops_graph_test.py b/tensorflow/contrib/summary/summary_ops_graph_test.py index 3aba04540e..ae8336daaf 100644 --- a/tensorflow/contrib/summary/summary_ops_graph_test.py +++ b/tensorflow/contrib/summary/summary_ops_graph_test.py @@ -16,27 +16,220 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import os import tempfile +import time import six -from tensorflow.contrib.summary import summary_ops from tensorflow.contrib.summary import summary_test_util from tensorflow.core.framework import graph_pb2 from tensorflow.core.framework import node_def_pb2 from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import state_ops +from tensorflow.python.ops import summary_ops_v2 as summary_ops +from tensorflow.python.ops import variables +from tensorflow.python.platform import gfile from tensorflow.python.platform import test from tensorflow.python.training import training_util get_all = summary_test_util.get_all -class DbTest(summary_test_util.SummaryDbTest): +class GraphFileTest(test_util.TensorFlowTestCase): + + def testSummaryOps(self): + logdir = self.get_temp_dir() + writer = summary_ops.create_file_writer(logdir, max_queue=0) + with writer.as_default(), summary_ops.always_record_summaries(): + summary_ops.generic('tensor', 1, step=1) + summary_ops.scalar('scalar', 2.0, step=1) + summary_ops.histogram('histogram', [1.0], step=1) + summary_ops.image('image', [[[[1.0]]]], step=1) + summary_ops.audio('audio', [[1.0]], 1.0, 1, step=1) + with self.test_session() as sess: + sess.run(summary_ops.summary_writer_initializer_op()) + sess.run(summary_ops.all_summary_ops()) + # The working condition of the ops is tested in the C++ test so we just + # test here that we're calling them correctly. + self.assertTrue(gfile.Exists(logdir)) + + def testSummaryName(self): + logdir = self.get_temp_dir() + writer = summary_ops.create_file_writer(logdir, max_queue=0) + with writer.as_default(), summary_ops.always_record_summaries(): + summary_ops.scalar('scalar', 2.0, step=1) + with self.test_session() as sess: + sess.run(summary_ops.summary_writer_initializer_op()) + sess.run(summary_ops.all_summary_ops()) + events = summary_test_util.events_from_logdir(logdir) + self.assertEqual(2, len(events)) + self.assertEqual('scalar', events[1].summary.value[0].tag) + + def testSummaryNameScope(self): + logdir = self.get_temp_dir() + writer = summary_ops.create_file_writer(logdir, max_queue=0) + with writer.as_default(), summary_ops.always_record_summaries(): + with ops.name_scope('scope'): + summary_ops.scalar('scalar', 2.0, step=1) + with self.test_session() as sess: + sess.run(summary_ops.summary_writer_initializer_op()) + sess.run(summary_ops.all_summary_ops()) + events = summary_test_util.events_from_logdir(logdir) + self.assertEqual(2, len(events)) + self.assertEqual('scope/scalar', events[1].summary.value[0].tag) + + def testSummaryGlobalStep(self): + training_util.get_or_create_global_step() + logdir = self.get_temp_dir() + writer = summary_ops.create_file_writer(logdir, max_queue=0) + with writer.as_default(), summary_ops.always_record_summaries(): + summary_ops.scalar('scalar', 2.0) + with self.test_session() as sess: + sess.run(variables.global_variables_initializer()) + sess.run(summary_ops.summary_writer_initializer_op()) + step, _ = sess.run( + [training_util.get_global_step(), summary_ops.all_summary_ops()]) + events = summary_test_util.events_from_logdir(logdir) + self.assertEqual(2, len(events)) + self.assertEqual(step, events[1].step) + + def testMaxQueue(self): + logdir = self.get_temp_dir() + writer = summary_ops.create_file_writer( + logdir, max_queue=1, flush_millis=999999) + with writer.as_default(), summary_ops.always_record_summaries(): + summary_ops.scalar('scalar', 2.0, step=1) + with self.test_session() as sess: + sess.run(summary_ops.summary_writer_initializer_op()) + get_total = lambda: len(summary_test_util.events_from_logdir(logdir)) + # Note: First tf.Event is always file_version. + self.assertEqual(1, get_total()) + sess.run(summary_ops.all_summary_ops()) + self.assertEqual(1, get_total()) + # Should flush after second summary since max_queue = 1 + sess.run(summary_ops.all_summary_ops()) + self.assertEqual(3, get_total()) + + def testFlushFunction(self): + logdir = self.get_temp_dir() + writer = summary_ops.create_file_writer( + logdir, max_queue=999999, flush_millis=999999) + with writer.as_default(), summary_ops.always_record_summaries(): + summary_ops.scalar('scalar', 2.0, step=1) + flush_op = summary_ops.flush() + with self.test_session() as sess: + sess.run(summary_ops.summary_writer_initializer_op()) + get_total = lambda: len(summary_test_util.events_from_logdir(logdir)) + # Note: First tf.Event is always file_version. + self.assertEqual(1, get_total()) + sess.run(summary_ops.all_summary_ops()) + self.assertEqual(1, get_total()) + sess.run(flush_op) + self.assertEqual(2, get_total()) + # Test "writer" parameter + sess.run(summary_ops.all_summary_ops()) + sess.run(summary_ops.flush(writer=writer)) + self.assertEqual(3, get_total()) + sess.run(summary_ops.all_summary_ops()) + sess.run(summary_ops.flush(writer=writer._resource)) # pylint:disable=protected-access + self.assertEqual(4, get_total()) + + def testSharedName(self): + logdir = self.get_temp_dir() + with summary_ops.always_record_summaries(): + # Create with default shared name (should match logdir) + writer1 = summary_ops.create_file_writer(logdir) + with writer1.as_default(): + summary_ops.scalar('one', 1.0, step=1) + # Create with explicit logdir shared name (should be same resource/file) + shared_name = 'logdir:' + logdir + writer2 = summary_ops.create_file_writer(logdir, name=shared_name) + with writer2.as_default(): + summary_ops.scalar('two', 2.0, step=2) + # Create with different shared name (should be separate resource/file) + writer3 = summary_ops.create_file_writer(logdir, name='other') + with writer3.as_default(): + summary_ops.scalar('three', 3.0, step=3) + + with self.test_session() as sess: + # Run init ops across writers sequentially to avoid race condition. + # TODO(nickfelt): fix race condition in resource manager lookup or create + sess.run(writer1.init()) + sess.run(writer2.init()) + time.sleep(1.1) # Ensure filename has a different timestamp + sess.run(writer3.init()) + sess.run(summary_ops.all_summary_ops()) + sess.run([writer1.flush(), writer2.flush(), writer3.flush()]) + + event_files = iter(sorted(gfile.Glob(os.path.join(logdir, '*tfevents*')))) + + # First file has tags "one" and "two" + events = summary_test_util.events_from_file(next(event_files)) + self.assertEqual('brain.Event:2', events[0].file_version) + tags = [e.summary.value[0].tag for e in events[1:]] + self.assertItemsEqual(['one', 'two'], tags) + + # Second file has tag "three" + events = summary_test_util.events_from_file(next(event_files)) + self.assertEqual('brain.Event:2', events[0].file_version) + tags = [e.summary.value[0].tag for e in events[1:]] + self.assertItemsEqual(['three'], tags) + + # No more files + self.assertRaises(StopIteration, lambda: next(event_files)) + + def testWriterInitAndClose(self): + logdir = self.get_temp_dir() + with summary_ops.always_record_summaries(): + writer = summary_ops.create_file_writer( + logdir, max_queue=100, flush_millis=1000000) + with writer.as_default(): + summary_ops.scalar('one', 1.0, step=1) + with self.test_session() as sess: + sess.run(summary_ops.summary_writer_initializer_op()) + get_total = lambda: len(summary_test_util.events_from_logdir(logdir)) + self.assertEqual(1, get_total()) # file_version Event + # Running init() again while writer is open has no effect + sess.run(writer.init()) + self.assertEqual(1, get_total()) + sess.run(summary_ops.all_summary_ops()) + self.assertEqual(1, get_total()) + # Running close() should do an implicit flush + sess.run(writer.close()) + self.assertEqual(2, get_total()) + # Running init() on a closed writer should start a new file + time.sleep(1.1) # Ensure filename has a different timestamp + sess.run(writer.init()) + sess.run(summary_ops.all_summary_ops()) + sess.run(writer.close()) + files = sorted(gfile.Glob(os.path.join(logdir, '*tfevents*'))) + self.assertEqual(2, len(files)) + self.assertEqual(2, len(summary_test_util.events_from_file(files[1]))) + + def testWriterFlush(self): + logdir = self.get_temp_dir() + with summary_ops.always_record_summaries(): + writer = summary_ops.create_file_writer( + logdir, max_queue=100, flush_millis=1000000) + with writer.as_default(): + summary_ops.scalar('one', 1.0, step=1) + with self.test_session() as sess: + sess.run(summary_ops.summary_writer_initializer_op()) + get_total = lambda: len(summary_test_util.events_from_logdir(logdir)) + self.assertEqual(1, get_total()) # file_version Event + sess.run(summary_ops.all_summary_ops()) + self.assertEqual(1, get_total()) + sess.run(writer.flush()) + self.assertEqual(2, get_total()) + + +class GraphDbTest(summary_test_util.SummaryDbTest): def testGraphPassedToGraph_isForbiddenForThineOwnSafety(self): with self.assertRaises(TypeError): diff --git a/tensorflow/contrib/summary/summary_ops_test.py b/tensorflow/contrib/summary/summary_ops_test.py index c756f8b270..f1ef218e74 100644 --- a/tensorflow/contrib/summary/summary_ops_test.py +++ b/tensorflow/contrib/summary/summary_ops_test.py @@ -16,12 +16,13 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import os import tempfile +import time import numpy as np import six -from tensorflow.contrib.summary import summary_ops from tensorflow.contrib.summary import summary_test_util from tensorflow.core.framework import graph_pb2 from tensorflow.core.framework import node_def_pb2 @@ -33,6 +34,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import state_ops +from tensorflow.python.ops import summary_ops_v2 as summary_ops from tensorflow.python.platform import gfile from tensorflow.python.training import training_util @@ -57,7 +59,7 @@ _NUMPY_NUMERIC_TYPES = { } -class TargetTest(test_util.TensorFlowTestCase): +class EagerFileTest(test_util.TensorFlowTestCase): def testShouldRecordSummary(self): self.assertFalse(summary_ops.should_record_summaries()) @@ -138,21 +140,22 @@ class TargetTest(test_util.TensorFlowTestCase): def testMaxQueue(self): logs = tempfile.mkdtemp() with summary_ops.create_file_writer( - logs, max_queue=2, flush_millis=999999, + logs, max_queue=1, flush_millis=999999, name='lol').as_default(), summary_ops.always_record_summaries(): get_total = lambda: len(summary_test_util.events_from_logdir(logs)) # Note: First tf.Event is always file_version. self.assertEqual(1, get_total()) summary_ops.scalar('scalar', 2.0, step=1) self.assertEqual(1, get_total()) + # Should flush after second summary since max_queue = 1 summary_ops.scalar('scalar', 2.0, step=2) self.assertEqual(3, get_total()) - def testFlush(self): + def testFlushFunction(self): logs = tempfile.mkdtemp() - with summary_ops.create_file_writer( - logs, max_queue=999999, flush_millis=999999, - name='lol').as_default(), summary_ops.always_record_summaries(): + writer = summary_ops.create_file_writer( + logs, max_queue=999999, flush_millis=999999, name='lol') + with writer.as_default(), summary_ops.always_record_summaries(): get_total = lambda: len(summary_test_util.events_from_logdir(logs)) # Note: First tf.Event is always file_version. self.assertEqual(1, get_total()) @@ -161,9 +164,103 @@ class TargetTest(test_util.TensorFlowTestCase): self.assertEqual(1, get_total()) summary_ops.flush() self.assertEqual(3, get_total()) + # Test "writer" parameter + summary_ops.scalar('scalar', 2.0, step=3) + summary_ops.flush(writer=writer) + self.assertEqual(4, get_total()) + summary_ops.scalar('scalar', 2.0, step=4) + summary_ops.flush(writer=writer._resource) # pylint:disable=protected-access + self.assertEqual(5, get_total()) + + def testSharedName(self): + logdir = self.get_temp_dir() + with summary_ops.always_record_summaries(): + # Create with default shared name (should match logdir) + writer1 = summary_ops.create_file_writer(logdir) + with writer1.as_default(): + summary_ops.scalar('one', 1.0, step=1) + summary_ops.flush() + # Create with explicit logdir shared name (should be same resource/file) + shared_name = 'logdir:' + logdir + writer2 = summary_ops.create_file_writer(logdir, name=shared_name) + with writer2.as_default(): + summary_ops.scalar('two', 2.0, step=2) + summary_ops.flush() + # Create with different shared name (should be separate resource/file) + time.sleep(1.1) # Ensure filename has a different timestamp + writer3 = summary_ops.create_file_writer(logdir, name='other') + with writer3.as_default(): + summary_ops.scalar('three', 3.0, step=3) + summary_ops.flush() + + event_files = iter(sorted(gfile.Glob(os.path.join(logdir, '*tfevents*')))) + + # First file has tags "one" and "two" + events = iter(summary_test_util.events_from_file(next(event_files))) + self.assertEqual('brain.Event:2', next(events).file_version) + self.assertEqual('one', next(events).summary.value[0].tag) + self.assertEqual('two', next(events).summary.value[0].tag) + self.assertRaises(StopIteration, lambda: next(events)) + + # Second file has tag "three" + events = iter(summary_test_util.events_from_file(next(event_files))) + self.assertEqual('brain.Event:2', next(events).file_version) + self.assertEqual('three', next(events).summary.value[0].tag) + self.assertRaises(StopIteration, lambda: next(events)) + + # No more files + self.assertRaises(StopIteration, lambda: next(event_files)) + + def testWriterInitAndClose(self): + logdir = self.get_temp_dir() + get_total = lambda: len(summary_test_util.events_from_logdir(logdir)) + with summary_ops.always_record_summaries(): + writer = summary_ops.create_file_writer( + logdir, max_queue=100, flush_millis=1000000) + self.assertEqual(1, get_total()) # file_version Event + # Calling init() again while writer is open has no effect + writer.init() + self.assertEqual(1, get_total()) + try: + # Not using .as_default() to avoid implicit flush when exiting + writer.set_as_default() + summary_ops.scalar('one', 1.0, step=1) + self.assertEqual(1, get_total()) + # Calling .close() should do an implicit flush + writer.close() + self.assertEqual(2, get_total()) + # Calling init() on a closed writer should start a new file + time.sleep(1.1) # Ensure filename has a different timestamp + writer.init() + files = sorted(gfile.Glob(os.path.join(logdir, '*tfevents*'))) + self.assertEqual(2, len(files)) + get_total = lambda: len(summary_test_util.events_from_file(files[1])) + self.assertEqual(1, get_total()) # file_version Event + summary_ops.scalar('two', 2.0, step=2) + writer.close() + self.assertEqual(2, get_total()) + finally: + # Clean up by resetting default writer + summary_ops.create_file_writer(None).set_as_default() + + def testWriterFlush(self): + logdir = self.get_temp_dir() + get_total = lambda: len(summary_test_util.events_from_logdir(logdir)) + with summary_ops.always_record_summaries(): + writer = summary_ops.create_file_writer( + logdir, max_queue=100, flush_millis=1000000) + self.assertEqual(1, get_total()) # file_version Event + with writer.as_default(): + summary_ops.scalar('one', 1.0, step=1) + self.assertEqual(1, get_total()) + writer.flush() + self.assertEqual(2, get_total()) + summary_ops.scalar('two', 2.0, step=2) + # Exiting the "as_default()" should do an implicit flush of the "two" tag + self.assertEqual(3, get_total()) -class DbTest(summary_test_util.SummaryDbTest): +class EagerDbTest(summary_test_util.SummaryDbTest): def testIntegerSummaries(self): step = training_util.create_global_step() diff --git a/tensorflow/contrib/summary/summary_test_internal.py b/tensorflow/contrib/summary/summary_test_internal.py deleted file mode 100644 index d0d3384735..0000000000 --- a/tensorflow/contrib/summary/summary_test_internal.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Internal helpers for tests in this directory.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools -import os - -import sqlite3 - -from tensorflow.contrib.summary import summary_ops -from tensorflow.python.framework import test_util - - -class SummaryDbTest(test_util.TensorFlowTestCase): - """Helper for summary database testing.""" - - def setUp(self): - super(SummaryDbTest, self).setUp() - self.db_path = os.path.join(self.get_temp_dir(), 'DbTest.sqlite') - if os.path.exists(self.db_path): - os.unlink(self.db_path) - self.db = sqlite3.connect(self.db_path) - self.create_db_writer = functools.partial( - summary_ops.create_db_writer, - db_uri=self.db_path, - experiment_name='experiment', - run_name='run', - user_name='user') - - def tearDown(self): - self.db.close() - super(SummaryDbTest, self).tearDown() - - -def get_one(db, q, *p): - return db.execute(q, p).fetchone()[0] - - -def get_all(db, q, *p): - return unroll(db.execute(q, p).fetchall()) - - -def unroll(list_of_tuples): - return sum(list_of_tuples, ()) diff --git a/tensorflow/contrib/summary/summary_test_util.py b/tensorflow/contrib/summary/summary_test_util.py index 8506c4be9c..b4ae43302c 100644 --- a/tensorflow/contrib/summary/summary_test_util.py +++ b/tensorflow/contrib/summary/summary_test_util.py @@ -24,10 +24,10 @@ import os import sqlite3 -from tensorflow.contrib.summary import summary_ops from tensorflow.core.util import event_pb2 from tensorflow.python.framework import test_util from tensorflow.python.lib.io import tf_record +from tensorflow.python.ops import summary_ops_v2 as summary_ops from tensorflow.python.platform import gfile diff --git a/tensorflow/contrib/tensorboard/db/summary_file_writer.cc b/tensorflow/contrib/tensorboard/db/summary_file_writer.cc index 85b3e7231b..3f24f58f03 100644 --- a/tensorflow/contrib/tensorboard/db/summary_file_writer.cc +++ b/tensorflow/contrib/tensorboard/db/summary_file_writer.cc @@ -132,7 +132,7 @@ class SummaryFileWriter : public SummaryWriterInterface { Status WriteEvent(std::unique_ptr event) override { mutex_lock ml(mu_); queue_.emplace_back(std::move(event)); - if (queue_.size() >= max_queue_ || + if (queue_.size() > max_queue_ || env_->NowMicros() - last_flush_ > 1000 * flush_millis_) { return InternalFlush(); } diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD index 2f4a76720d..3e489d38b6 100644 --- a/tensorflow/contrib/tpu/BUILD +++ b/tensorflow/contrib/tpu/BUILD @@ -46,7 +46,6 @@ py_library( deps = [ ":tpu_lib", ":tpu_py", - "//tensorflow/contrib/summary:summary_ops", "//tensorflow/contrib/training:training_py", "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", @@ -57,6 +56,7 @@ py_library( "//tensorflow/python:platform", "//tensorflow/python:state_ops", "//tensorflow/python:summary", + "//tensorflow/python:summary_ops_v2", "//tensorflow/python:training", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py index 1332108d04..7fab19afee 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py @@ -30,7 +30,6 @@ import six from six.moves import queue as Queue # pylint: disable=redefined-builtin from six.moves import xrange # pylint: disable=redefined-builtin -from tensorflow.contrib.summary import summary_ops as contrib_summary from tensorflow.contrib.tpu.python.ops import tpu_ops from tensorflow.contrib.tpu.python.tpu import tpu from tensorflow.contrib.tpu.python.tpu import tpu_config @@ -57,6 +56,7 @@ from tensorflow.python.ops import init_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import state_ops +from tensorflow.python.ops import summary_ops_v2 as contrib_summary from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables from tensorflow.python.platform import tf_logging as logging diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 7b548d2c70..9707b370c0 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -2549,6 +2549,30 @@ py_library( ], ) +py_library( + name = "summary_ops_v2", + srcs = ["ops/summary_ops_v2.py"], + srcs_version = "PY2AND3", + visibility = ["//tensorflow:internal"], + deps = [ + ":array_ops", + ":constant_op", + ":control_flow_ops", + ":dtypes", + ":framework_ops", + ":math_ops", + ":resource_variable_ops", + ":smart_cond", + ":summary_op_util", + ":summary_ops_gen", + ":training_util", + ":util", + "//tensorflow/core:protos_all_py", + "//tensorflow/python/eager:context", + "@six_archive//:six", + ], +) + py_library( name = "template", srcs = ["ops/template.py"], @@ -2911,7 +2935,10 @@ py_library( name = "training", srcs = glob( ["training/**/*.py"], - exclude = ["**/*test*"], + exclude = [ + "**/*test*", + "training/training_util.py", # See :training_util + ], ), srcs_version = "PY2AND3", deps = [ @@ -2945,6 +2972,7 @@ py_library( ":string_ops", ":summary", ":training_ops_gen", + ":training_util", ":util", ":variable_scope", ":variables", @@ -4194,6 +4222,25 @@ py_test( ], ) +py_library( + name = "training_util", + srcs = ["training/training_util.py"], + srcs_version = "PY2AND3", + deps = [ + ":dtypes", + ":framework", + ":framework_ops", + ":init_ops", + ":platform", + ":resource_variable_ops", + ":state_ops", + ":util", + ":variable_scope", + ":variables", + "//tensorflow/python/eager:context", + ], +) + py_test( name = "training_util_test", size = "small", @@ -4204,6 +4251,7 @@ py_test( ":framework", ":platform", ":training", + ":training_util", ":variables", ], ) @@ -4248,6 +4296,7 @@ py_library( srcs_version = "PY2AND3", visibility = ["//visibility:public"], deps = [ + ":client", ":constant_op", ":errors", ":framework", @@ -4260,6 +4309,7 @@ py_library( ":summary_op_util", ":summary_ops", ":summary_ops_gen", + ":summary_ops_v2", ":util", "//tensorflow/python/eager:context", "//third_party/py/numpy", @@ -4286,7 +4336,7 @@ py_tests( ":platform", ":platform_test", ":summary", - ":training", + ":summary_ops_v2", "//tensorflow/core:protos_all_py", ], ) diff --git a/tensorflow/contrib/summary/summary_ops.py b/tensorflow/python/ops/summary_ops_v2.py similarity index 90% rename from tensorflow/contrib/summary/summary_ops.py rename to tensorflow/python/ops/summary_ops_v2.py index bc763fe655..12f361c513 100644 --- a/tensorflow/contrib/summary/summary_ops.py +++ b/tensorflow/python/ops/summary_ops_v2.py @@ -31,7 +31,7 @@ from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops -from tensorflow.python.layers import utils +from tensorflow.python.framework import smart_cond from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gen_summary_ops @@ -108,8 +108,10 @@ class SummaryWriter(object): - @{tf.contrib.summary.create_db_writer} """ - def __init__(self, resource): + def __init__(self, resource, init_op_fn): self._resource = resource + # TODO(nickfelt): cache constructed ops in graph mode + self._init_op_fn = init_op_fn if context.executing_eagerly() and self._resource is not None: self._resource_deleter = resource_variable_ops.EagerResourceDeleter( handle=self._resource, handle_device="cpu:0") @@ -129,10 +131,32 @@ class SummaryWriter(object): yield self # Flushes the summary writer in eager mode or in graph functions, but not # in legacy graph mode (you're on your own there). - with ops.device("cpu:0"): - gen_summary_ops.flush_summary_writer(self._resource) + self.flush() context.context().summary_writer_resource = old + def init(self): + """Operation to initialize the summary writer resource.""" + if self._resource is not None: + return self._init_op_fn() + + def _flush(self): + return _flush_fn(writer=self) + + def flush(self): + """Operation to force the summary writer to flush any buffered data.""" + if self._resource is not None: + return self._flush() + + def _close(self): + with ops.control_dependencies([self.flush()]): + with ops.device("cpu:0"): + return gen_summary_ops.close_summary_writer(self._resource) + + def close(self): + """Operation to flush and close the summary writer resource.""" + if self._resource is not None: + return self._close() + def initialize( graph=None, # pylint: disable=redefined-outer-name @@ -178,7 +202,7 @@ def create_file_writer(logdir, flush_millis=None, filename_suffix=None, name=None): - """Creates a summary file writer in the current context. + """Creates a summary file writer in the current context under the given name. Args: logdir: a string, or None. If a string, creates a summary file writer @@ -186,18 +210,20 @@ def create_file_writer(logdir, a mock object which acts like a summary writer but does nothing, useful to use as a context manager. max_queue: the largest number of summaries to keep in a queue; will - flush once the queue gets bigger than this. - flush_millis: the largest interval between flushes. - filename_suffix: optional suffix for the event file name. + flush once the queue gets bigger than this. Defaults to 10. + flush_millis: the largest interval between flushes. Defaults to 120,000. + filename_suffix: optional suffix for the event file name. Defaults to `.v2`. name: Shared name for this SummaryWriter resource stored to default - Graph. + Graph. Defaults to the provided logdir prefixed with `logdir:`. Note: if a + summary writer resource with this shared name already exists, the returned + SummaryWriter wraps that resource and the other arguments have no effect. Returns: Either a summary writer or an empty object which can be used as a summary writer. """ if logdir is None: - return SummaryWriter(None) + return SummaryWriter(None, None) with ops.device("cpu:0"): if max_queue is None: max_queue = constant_op.constant(10) @@ -205,6 +231,8 @@ def create_file_writer(logdir, flush_millis = constant_op.constant(2 * 60 * 1000) if filename_suffix is None: filename_suffix = constant_op.constant(".v2") + if name is None: + name = "logdir:" + logdir return _make_summary_writer( name, gen_summary_ops.create_summary_file_writer, @@ -267,13 +295,12 @@ def create_db_writer(db_uri, def _make_summary_writer(name, factory, **kwargs): resource = gen_summary_ops.summary_writer(shared_name=name) + init_op_fn = lambda: factory(resource, **kwargs) # TODO(apassos): Consider doing this instead. - # node = factory(resource, **kwargs) # if not context.executing_eagerly(): - # ops.get_default_session().run(node) - ops.add_to_collection(_SUMMARY_WRITER_INIT_COLLECTION_NAME, - factory(resource, **kwargs)) - return SummaryWriter(resource) + # ops.get_default_session().run(init_op) + ops.add_to_collection(_SUMMARY_WRITER_INIT_COLLECTION_NAME, init_op_fn()) + return SummaryWriter(resource, init_op_fn) def _cleanse_string(name, pattern, value): @@ -341,7 +368,7 @@ def summary_writer_function(name, tensor, function, family=None): if context.context().summary_writer_resource is None: return control_flow_ops.no_op() with ops.device("cpu:0"): - op = utils.smart_cond( + op = smart_cond.smart_cond( should_record_summaries(), record, _nothing, name="") ops.add_to_collection(ops.GraphKeys._SUMMARY_COLLECTION, op) # pylint: disable=protected-access return op @@ -538,7 +565,14 @@ def flush(writer=None, name=None): writer = context.context().summary_writer_resource if writer is None: return control_flow_ops.no_op() - return gen_summary_ops.flush_summary_writer(writer, name=name) + else: + if isinstance(writer, SummaryWriter): + writer = writer._resource # pylint: disable=protected-access + with ops.device("cpu:0"): + return gen_summary_ops.flush_summary_writer(writer, name=name) + + +_flush_fn = flush # for within SummaryWriter.flush() def eval_dir(model_dir, name=None): diff --git a/tensorflow/python/summary/writer/event_file_writer_v2.py b/tensorflow/python/summary/writer/event_file_writer_v2.py new file mode 100644 index 0000000000..5c66c0f7a8 --- /dev/null +++ b/tensorflow/python/summary/writer/event_file_writer_v2.py @@ -0,0 +1,140 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Writes events to disk in a logdir.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import summary_ops_v2 +from tensorflow.python.platform import gfile + + +class EventFileWriterV2(object): + """Writes `Event` protocol buffers to an event file via the graph. + + The `EventFileWriterV2` class is backed by the summary file writer in the v2 + summary API (currently in tf.contrib.summary), so it uses a shared summary + writer resource and graph ops to write events. + + As with the original EventFileWriter, this class will asynchronously write + Event protocol buffers to the backing file. The Event file is encoded using + the tfrecord format, which is similar to RecordIO. + """ + + def __init__(self, session, logdir, max_queue=10, flush_secs=120, + filename_suffix=''): + """Creates an `EventFileWriterV2` and an event file to write to. + + On construction, this calls `tf.contrib.summary.create_file_writer` within + the graph from `session.graph` to look up a shared summary writer resource + for `logdir` if one exists, and create one if not. Creating the summary + writer resource in turn creates a new event file in `logdir` to be filled + with `Event` protocol buffers passed to `add_event`. Graph ops to control + this writer resource are added to `session.graph` during this init call; + stateful methods on this class will call `session.run()` on these ops. + + Note that because the underlying resource is shared, it is possible that + other parts of the code using the same session may interact independently + with the resource, e.g. by flushing or even closing it. It is the caller's + responsibility to avoid any undesirable sharing in this regard. + + The remaining arguments to the constructor (`flush_secs`, `max_queue`, and + `filename_suffix`) control the construction of the shared writer resource + if one is created. If an existing resource is reused, these arguments have + no effect. See `tf.contrib.summary.create_file_writer` for details. + + Args: + session: A `tf.Session`. Session that will hold shared writer resource. + The writer ops will be added to session.graph during this init call. + logdir: A string. Directory where event file will be written. + max_queue: Integer. Size of the queue for pending events and summaries. + flush_secs: Number. How often, in seconds, to flush the + pending events and summaries to disk. + filename_suffix: A string. Every event file's name is suffixed with + `filename_suffix`. + """ + self._session = session + self._logdir = logdir + self._closed = False + if not gfile.IsDirectory(self._logdir): + gfile.MakeDirs(self._logdir) + + with self._session.graph.as_default(): + with ops.name_scope('filewriter'): + file_writer = summary_ops_v2.create_file_writer( + logdir=self._logdir, + max_queue=max_queue, + flush_millis=flush_secs * 1000, + filename_suffix=filename_suffix) + with summary_ops_v2.always_record_summaries(), file_writer.as_default(): + self._event_placeholder = array_ops.placeholder_with_default( + constant_op.constant('unused', dtypes.string), + shape=[]) + self._add_event_op = summary_ops_v2.import_event( + self._event_placeholder) + self._init_op = file_writer.init() + self._flush_op = file_writer.flush() + self._close_op = file_writer.close() + self._session.run(self._init_op) + + def get_logdir(self): + """Returns the directory where event file will be written.""" + return self._logdir + + def reopen(self): + """Reopens the EventFileWriter. + + Can be called after `close()` to add more events in the same directory. + The events will go into a new events file. + + Does nothing if the EventFileWriter was not closed. + """ + if self._closed: + self._closed = False + self._session.run(self._init_op) + + def add_event(self, event): + """Adds an event to the event file. + + Args: + event: An `Event` protocol buffer. + """ + if not self._closed: + event_pb = event.SerializeToString() + self._session.run( + self._add_event_op, feed_dict={self._event_placeholder: event_pb}) + + def flush(self): + """Flushes the event file to disk. + + Call this method to make sure that all pending events have been written to + disk. + """ + self._session.run(self._flush_op) + + def close(self): + """Flushes the event file to disk and close the file. + + Call this method when you do not need the summary writer anymore. + """ + if not self._closed: + self.flush() + self._session.run(self._close_op) + self._closed = True diff --git a/tensorflow/python/summary/writer/writer.py b/tensorflow/python/summary/writer/writer.py index 57f78c156b..aca084fc91 100644 --- a/tensorflow/python/summary/writer/writer.py +++ b/tensorflow/python/summary/writer/writer.py @@ -32,6 +32,7 @@ from tensorflow.python.platform import gfile from tensorflow.python.platform import tf_logging as logging from tensorflow.python.summary import plugin_asset from tensorflow.python.summary.writer.event_file_writer import EventFileWriter +from tensorflow.python.summary.writer.event_file_writer_v2 import EventFileWriterV2 from tensorflow.python.util.tf_export import tf_export _PLUGINS_DIR = "plugins" @@ -286,6 +287,11 @@ class FileWriter(SummaryToEventTransformer): file contents asynchronously. This allows a training program to call methods to add data to the file directly from the training loop, without slowing down training. + + When constructed with a `tf.Session` parameter, a `FileWriter` instead forms + a compatibility layer over new graph-based summaries (`tf.contrib.summary`) + to facilitate the use of new summary writing with pre-existing code that + expects a `FileWriter` instance. """ def __init__(self, @@ -294,10 +300,11 @@ class FileWriter(SummaryToEventTransformer): max_queue=10, flush_secs=120, graph_def=None, - filename_suffix=None): - """Creates a `FileWriter` and an event file. + filename_suffix=None, + session=None): + """Creates a `FileWriter`, optionally shared within the given session. - On construction the summary writer creates a new event file in `logdir`. + Typically, constructing a file writer creates a new event file in `logdir`. This event file will contain `Event` protocol buffers constructed when you call one of the following functions: `add_summary()`, `add_session_log()`, `add_event()`, or `add_graph()`. @@ -317,13 +324,16 @@ class FileWriter(SummaryToEventTransformer): writer = tf.summary.FileWriter(, sess.graph) ``` - The other arguments to the constructor control the asynchronous writes to - the event file: - - * `flush_secs`: How often, in seconds, to flush the added summaries - and events to disk. - * `max_queue`: Maximum number of summaries or events pending to be - written to disk before one of the 'add' calls block. + The `session` argument to the constructor makes the returned `FileWriter` a + a compatibility layer over new graph-based summaries (`tf.contrib.summary`). + Crucially, this means the underlying writer resource and events file will + be shared with any other `FileWriter` using the same `session` and `logdir`, + and with any `tf.contrib.summary.SummaryWriter` in this session using the + the same shared resource name (which by default scoped to the logdir). If + no such resource exists, one will be created using the remaining arguments + to this constructor, but if one already exists those arguments are ignored. + In either case, ops will be added to `session.graph` to control the + underlying file writer resource. See `tf.contrib.summary` for more details. Args: logdir: A string. Directory where event file will be written. @@ -334,6 +344,7 @@ class FileWriter(SummaryToEventTransformer): graph_def: DEPRECATED: Use the `graph` argument instead. filename_suffix: A string. Every event file's name is suffixed with `suffix`. + session: A `tf.Session` object. See details above. Raises: RuntimeError: If called with eager execution enabled. @@ -347,9 +358,12 @@ class FileWriter(SummaryToEventTransformer): raise RuntimeError( "tf.summary.FileWriter is not compatible with eager execution. " "Use tf.contrib.summary instead.") - - event_writer = EventFileWriter(logdir, max_queue, flush_secs, - filename_suffix) + if session is not None: + event_writer = EventFileWriterV2( + session, logdir, max_queue, flush_secs, filename_suffix) + else: + event_writer = EventFileWriter(logdir, max_queue, flush_secs, + filename_suffix) super(FileWriter, self).__init__(event_writer, graph, graph_def) def __enter__(self): diff --git a/tensorflow/python/summary/writer/writer_test.py b/tensorflow/python/summary/writer/writer_test.py index 88ade0aac3..dc990c2602 100644 --- a/tensorflow/python/summary/writer/writer_test.py +++ b/tensorflow/python/summary/writer/writer_test.py @@ -29,10 +29,12 @@ from tensorflow.core.protobuf import config_pb2 from tensorflow.core.protobuf import meta_graph_pb2 from tensorflow.core.util import event_pb2 from tensorflow.core.util.event_pb2 import SessionLog +from tensorflow.python.client import session from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import meta_graph from tensorflow.python.framework import ops +from tensorflow.python.ops import summary_ops_v2 from tensorflow.python.platform import gfile from tensorflow.python.platform import test from tensorflow.python.summary import plugin_asset @@ -42,7 +44,10 @@ from tensorflow.python.summary.writer import writer_cache from tensorflow.python.util import compat -class SummaryWriterTestCase(test.TestCase): +class FileWriterTestCase(test.TestCase): + + def _FileWriter(self, *args, **kwargs): + return writer.FileWriter(*args, **kwargs) def _TestDir(self, test_name): test_dir = os.path.join(self.get_temp_dir(), test_name) @@ -96,7 +101,7 @@ class SummaryWriterTestCase(test.TestCase): def testAddingSummaryGraphAndRunMetadata(self): test_dir = self._CleanTestDir("basics") - sw = writer.FileWriter(test_dir) + sw = self._FileWriter(test_dir) sw.add_session_log(event_pb2.SessionLog(status=SessionLog.START), 1) sw.add_summary( @@ -171,7 +176,7 @@ class SummaryWriterTestCase(test.TestCase): test_dir = self._CleanTestDir("basics_named_graph") with ops.Graph().as_default() as g: constant_op.constant([12], name="douze") - sw = writer.FileWriter(test_dir, graph=g) + sw = self._FileWriter(test_dir, graph=g) sw.close() self._assertEventsWithGraph(test_dir, g, True) @@ -179,7 +184,7 @@ class SummaryWriterTestCase(test.TestCase): test_dir = self._CleanTestDir("basics_positional_graph") with ops.Graph().as_default() as g: constant_op.constant([12], name="douze") - sw = writer.FileWriter(test_dir, g) + sw = self._FileWriter(test_dir, g) sw.close() self._assertEventsWithGraph(test_dir, g, True) @@ -188,7 +193,7 @@ class SummaryWriterTestCase(test.TestCase): with ops.Graph().as_default() as g: constant_op.constant([12], name="douze") gd = g.as_graph_def() - sw = writer.FileWriter(test_dir, graph_def=gd) + sw = self._FileWriter(test_dir, graph_def=gd) sw.close() self._assertEventsWithGraph(test_dir, g, False) @@ -197,7 +202,7 @@ class SummaryWriterTestCase(test.TestCase): with ops.Graph().as_default() as g: constant_op.constant([12], name="douze") gd = g.as_graph_def() - sw = writer.FileWriter(test_dir, gd) + sw = self._FileWriter(test_dir, gd) sw.close() self._assertEventsWithGraph(test_dir, g, False) @@ -207,18 +212,18 @@ class SummaryWriterTestCase(test.TestCase): with ops.Graph().as_default() as g: constant_op.constant([12], name="douze") gd = g.as_graph_def() - sw = writer.FileWriter(test_dir, graph=g, graph_def=gd) + sw = self._FileWriter(test_dir, graph=g, graph_def=gd) sw.close() def testNeitherGraphNorGraphDef(self): with self.assertRaises(TypeError): test_dir = self._CleanTestDir("basics_string_instead_of_graph") - sw = writer.FileWriter(test_dir, "string instead of graph object") + sw = self._FileWriter(test_dir, "string instead of graph object") sw.close() def testCloseAndReopen(self): test_dir = self._CleanTestDir("close_and_reopen") - sw = writer.FileWriter(test_dir) + sw = self._FileWriter(test_dir) sw.add_session_log(event_pb2.SessionLog(status=SessionLog.START), 1) sw.close() # Sleep at least one second to make sure we get a new event file name. @@ -261,7 +266,7 @@ class SummaryWriterTestCase(test.TestCase): def testNonBlockingClose(self): test_dir = self._CleanTestDir("non_blocking_close") - sw = writer.FileWriter(test_dir) + sw = self._FileWriter(test_dir) # Sleep 1.2 seconds to make sure event queue is empty. time.sleep(1.2) time_before_close = time.time() @@ -270,7 +275,7 @@ class SummaryWriterTestCase(test.TestCase): def testWithStatement(self): test_dir = self._CleanTestDir("with_statement") - with writer.FileWriter(test_dir) as sw: + with self._FileWriter(test_dir) as sw: sw.add_session_log(event_pb2.SessionLog(status=SessionLog.START), 1) event_paths = sorted(glob.glob(os.path.join(test_dir, "event*"))) self.assertEquals(1, len(event_paths)) @@ -280,7 +285,7 @@ class SummaryWriterTestCase(test.TestCase): # protocol buffers correctly. def testAddingSummariesFromSessionRunCalls(self): test_dir = self._CleanTestDir("global_step") - sw = writer.FileWriter(test_dir) + sw = self._FileWriter(test_dir) with self.test_session(): i = constant_op.constant(1, dtype=dtypes.int32, shape=[]) l = constant_op.constant(2, dtype=dtypes.int64, shape=[]) @@ -327,7 +332,7 @@ class SummaryWriterTestCase(test.TestCase): def testPluginMetadataStrippedFromSubsequentEvents(self): test_dir = self._CleanTestDir("basics") - sw = writer.FileWriter(test_dir) + sw = self._FileWriter(test_dir) sw.add_session_log(event_pb2.SessionLog(status=SessionLog.START), 1) @@ -386,7 +391,7 @@ class SummaryWriterTestCase(test.TestCase): def testFileWriterWithSuffix(self): test_dir = self._CleanTestDir("test_suffix") - sw = writer.FileWriter(test_dir, filename_suffix="_test_suffix") + sw = self._FileWriter(test_dir, filename_suffix="_test_suffix") for _ in range(10): sw.add_summary( summary_pb2.Summary(value=[ @@ -400,9 +405,178 @@ class SummaryWriterTestCase(test.TestCase): for filename in event_filenames: self.assertTrue(filename.endswith("_test_suffix")) + def testPluginAssetSerialized(self): + class ExamplePluginAsset(plugin_asset.PluginAsset): + plugin_name = "example" + + def assets(self): + return {"foo.txt": "foo!", "bar.txt": "bar!"} + + with ops.Graph().as_default() as g: + plugin_asset.get_plugin_asset(ExamplePluginAsset) + + logdir = self.get_temp_dir() + fw = self._FileWriter(logdir) + fw.add_graph(g) + plugin_dir = os.path.join(logdir, writer._PLUGINS_DIR, "example") + + with gfile.Open(os.path.join(plugin_dir, "foo.txt"), "r") as f: + content = f.read() + self.assertEqual(content, "foo!") + + with gfile.Open(os.path.join(plugin_dir, "bar.txt"), "r") as f: + content = f.read() + self.assertEqual(content, "bar!") -class SummaryWriterCacheTest(test.TestCase): - """SummaryWriterCache tests.""" + +class SessionBasedFileWriterTestCase(FileWriterTestCase): + """Tests for FileWriter behavior when passed a Session argument.""" + + def _FileWriter(self, *args, **kwargs): + if "session" not in kwargs: + # Pass in test_session() as the session. It will be cached during this + # test method invocation so that any other use of test_session() with no + # graph should result in re-using the same underlying Session. + with self.test_session() as sess: + kwargs["session"] = sess + return writer.FileWriter(*args, **kwargs) + return writer.FileWriter(*args, **kwargs) + + def _createTaggedSummary(self, tag): + summary = summary_pb2.Summary() + summary.value.add(tag=tag) + return summary + + def testSharing_withOtherSessionBasedFileWriters(self): + logdir = self.get_temp_dir() + with session.Session() as sess: + # Initial file writer + writer1 = writer.FileWriter(session=sess, logdir=logdir) + writer1.add_summary(self._createTaggedSummary("one"), 1) + writer1.flush() + + # File writer, should share file with writer1 + writer2 = writer.FileWriter(session=sess, logdir=logdir) + writer2.add_summary(self._createTaggedSummary("two"), 2) + writer2.flush() + + # File writer with different logdir (shouldn't be in this logdir at all) + writer3 = writer.FileWriter(session=sess, logdir=logdir + "-other") + writer3.add_summary(self._createTaggedSummary("three"), 3) + writer3.flush() + + # File writer in a different session (should be in separate file) + time.sleep(1.1) # Ensure filename has a different timestamp + with session.Session() as other_sess: + writer4 = writer.FileWriter(session=other_sess, logdir=logdir) + writer4.add_summary(self._createTaggedSummary("four"), 4) + writer4.flush() + + # One more file writer, should share file with writer1 + writer5 = writer.FileWriter(session=sess, logdir=logdir) + writer5.add_summary(self._createTaggedSummary("five"), 5) + writer5.flush() + + event_paths = iter(sorted(glob.glob(os.path.join(logdir, "event*")))) + + # First file should have tags "one", "two", and "five" + events = summary_iterator.summary_iterator(next(event_paths)) + self.assertEqual("brain.Event:2", next(events).file_version) + self.assertEqual("one", next(events).summary.value[0].tag) + self.assertEqual("two", next(events).summary.value[0].tag) + self.assertEqual("five", next(events).summary.value[0].tag) + self.assertRaises(StopIteration, lambda: next(events)) + + # Second file should have just "four" + events = summary_iterator.summary_iterator(next(event_paths)) + self.assertEqual("brain.Event:2", next(events).file_version) + self.assertEqual("four", next(events).summary.value[0].tag) + self.assertRaises(StopIteration, lambda: next(events)) + + # No more files + self.assertRaises(StopIteration, lambda: next(event_paths)) + + # Just check that the other logdir file exists to be sure we wrote it + self.assertTrue(glob.glob(os.path.join(logdir + "-other", "event*"))) + + def testSharing_withExplicitSummaryFileWriters(self): + logdir = self.get_temp_dir() + with session.Session() as sess: + # Initial file writer via FileWriter(session=?) + writer1 = writer.FileWriter(session=sess, logdir=logdir) + writer1.add_summary(self._createTaggedSummary("one"), 1) + writer1.flush() + + # Next one via create_file_writer(), should use same file + writer2 = summary_ops_v2.create_file_writer(logdir=logdir) + with summary_ops_v2.always_record_summaries(), writer2.as_default(): + summary2 = summary_ops_v2.scalar("two", 2.0, step=2) + sess.run(writer2.init()) + sess.run(summary2) + sess.run(writer2.flush()) + + # Next has different shared name, should be in separate file + time.sleep(1.1) # Ensure filename has a different timestamp + writer3 = summary_ops_v2.create_file_writer(logdir=logdir, name="other") + with summary_ops_v2.always_record_summaries(), writer3.as_default(): + summary3 = summary_ops_v2.scalar("three", 3.0, step=3) + sess.run(writer3.init()) + sess.run(summary3) + sess.run(writer3.flush()) + + # Next uses a second session, should be in separate file + time.sleep(1.1) # Ensure filename has a different timestamp + with session.Session() as other_sess: + writer4 = summary_ops_v2.create_file_writer(logdir=logdir) + with summary_ops_v2.always_record_summaries(), writer4.as_default(): + summary4 = summary_ops_v2.scalar("four", 4.0, step=4) + other_sess.run(writer4.init()) + other_sess.run(summary4) + other_sess.run(writer4.flush()) + + # Next via FileWriter(session=?) uses same second session, should be in + # same separate file. (This checks sharing in the other direction) + writer5 = writer.FileWriter(session=other_sess, logdir=logdir) + writer5.add_summary(self._createTaggedSummary("five"), 5) + writer5.flush() + + # One more via create_file_writer(), should use same file + writer6 = summary_ops_v2.create_file_writer(logdir=logdir) + with summary_ops_v2.always_record_summaries(), writer6.as_default(): + summary6 = summary_ops_v2.scalar("six", 6.0, step=6) + sess.run(writer6.init()) + sess.run(summary6) + sess.run(writer6.flush()) + + event_paths = iter(sorted(glob.glob(os.path.join(logdir, "event*")))) + + # First file should have tags "one", "two", and "six" + events = summary_iterator.summary_iterator(next(event_paths)) + self.assertEqual("brain.Event:2", next(events).file_version) + self.assertEqual("one", next(events).summary.value[0].tag) + self.assertEqual("two", next(events).summary.value[0].tag) + self.assertEqual("six", next(events).summary.value[0].tag) + self.assertRaises(StopIteration, lambda: next(events)) + + # Second file should have just "three" + events = summary_iterator.summary_iterator(next(event_paths)) + self.assertEqual("brain.Event:2", next(events).file_version) + self.assertEqual("three", next(events).summary.value[0].tag) + self.assertRaises(StopIteration, lambda: next(events)) + + # Third file should have "four" and "five" + events = summary_iterator.summary_iterator(next(event_paths)) + self.assertEqual("brain.Event:2", next(events).file_version) + self.assertEqual("four", next(events).summary.value[0].tag) + self.assertEqual("five", next(events).summary.value[0].tag) + self.assertRaises(StopIteration, lambda: next(events)) + + # No more files + self.assertRaises(StopIteration, lambda: next(event_paths)) + + +class FileWriterCacheTest(test.TestCase): + """FileWriterCache tests.""" def _test_dir(self, test_name): """Create an empty dir to use for tests. @@ -448,32 +622,5 @@ class SummaryWriterCacheTest(test.TestCase): self.assertFalse(sw1 == sw2) -class ExamplePluginAsset(plugin_asset.PluginAsset): - plugin_name = "example" - - def assets(self): - return {"foo.txt": "foo!", "bar.txt": "bar!"} - - -class PluginAssetsTest(test.TestCase): - - def testPluginAssetSerialized(self): - with ops.Graph().as_default() as g: - plugin_asset.get_plugin_asset(ExamplePluginAsset) - - logdir = self.get_temp_dir() - fw = writer.FileWriter(logdir) - fw.add_graph(g) - plugin_dir = os.path.join(logdir, writer._PLUGINS_DIR, "example") - - with gfile.Open(os.path.join(plugin_dir, "foo.txt"), "r") as f: - content = f.read() - self.assertEqual(content, "foo!") - - with gfile.Open(os.path.join(plugin_dir, "bar.txt"), "r") as f: - content = f.read() - self.assertEqual(content, "bar!") - - if __name__ == "__main__": test.main() diff --git a/tensorflow/tools/api/golden/tensorflow.summary.-file-writer.pbtxt b/tensorflow/tools/api/golden/tensorflow.summary.-file-writer.pbtxt index dcf747971b..6b65b0ace3 100644 --- a/tensorflow/tools/api/golden/tensorflow.summary.-file-writer.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.summary.-file-writer.pbtxt @@ -5,7 +5,7 @@ tf_class { is_instance: "" member_method { name: "__init__" - argspec: "args=[\'self\', \'logdir\', \'graph\', \'max_queue\', \'flush_secs\', \'graph_def\', \'filename_suffix\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'120\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'logdir\', \'graph\', \'max_queue\', \'flush_secs\', \'graph_def\', \'filename_suffix\', \'session\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'120\', \'None\', \'None\', \'None\'], " } member_method { name: "add_event" -- GitLab From 2fc718c21cb82b2905cfc0ade2c801ce56af62d1 Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Wed, 11 Apr 2018 02:16:25 -0700 Subject: [PATCH 154/791] [TF:XLA] Mark oom_test as optonly, it's really slow when compiled without optimization. PiperOrigin-RevId: 192420481 --- tensorflow/compiler/tests/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD index a7a8d2d1ff..47c6ab58c0 100644 --- a/tensorflow/compiler/tests/BUILD +++ b/tensorflow/compiler/tests/BUILD @@ -203,6 +203,7 @@ tf_xla_py_test( tags = [ # Allocates very large amounts of memory and does not work under TSAN. "notsan", + "optonly", # Times out frequently in fastbuild. ], deps = [ ":xla_test", -- GitLab From ef6637771b2582245bb15507a6796b3c3f1db6b5 Mon Sep 17 00:00:00 2001 From: ManHyuk Date: Wed, 11 Apr 2018 20:48:32 +0900 Subject: [PATCH 155/791] fix typo --- tensorflow/core/framework/collective.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/framework/collective.h b/tensorflow/core/framework/collective.h index 5810c7fa54..a82fb50d88 100644 --- a/tensorflow/core/framework/collective.h +++ b/tensorflow/core/framework/collective.h @@ -178,7 +178,7 @@ class StepSequenceInterface { virtual void RefreshStepIdSequenceAsync(int64 graph_key, const StatusCallback& done) = 0; - // Returns the the step_id that should be used for initiating a new execution + // Returns the step_id that should be used for initiating a new execution // on the specified graph. May return the same step_id multiple times if // RetireStepId or RefreshStepIdReservation is not called. virtual int64 NextStepId(int64 graph_key) = 0; -- GitLab From acd9725e72af749c60153cd4d7efdd679c935426 Mon Sep 17 00:00:00 2001 From: ManHyuk Date: Wed, 11 Apr 2018 20:49:46 +0900 Subject: [PATCH 156/791] fix typo --- tensorflow/contrib/lite/toco/model.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/lite/toco/model.h b/tensorflow/contrib/lite/toco/model.h index 56ef9fe2a8..8a936842d9 100644 --- a/tensorflow/contrib/lite/toco/model.h +++ b/tensorflow/contrib/lite/toco/model.h @@ -151,7 +151,7 @@ enum class AxesOrder { }; // The type of the scalars in an array. -// Note that that does not by itself tell whether the values in the array are +// Note that does not by itself tell whether the values in the array are // real (are literally interpreted as real numbers) or quantized (only acquire // a meaning as real numbers in conjunction with QuantizationParams). // -- GitLab From bbfff939e45013a7b5f8f6412981e7b50a4273d4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 07:47:26 -0700 Subject: [PATCH 157/791] Fixing propagation of minmax info on constant gather ops. PiperOrigin-RevId: 192448922 --- .../resolve_constant_concatenation.cc | 16 ++++++++++++---- .../resolve_constant_gather.cc | 10 ++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_concatenation.cc b/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_concatenation.cc index 064810b53e..d916ae0ddf 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_concatenation.cc +++ b/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_concatenation.cc @@ -18,6 +18,7 @@ limitations under the License. #include #include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" #include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h" #include "tensorflow/contrib/lite/toco/model.h" #include "tensorflow/contrib/lite/toco/tooling_util.h" @@ -105,7 +106,8 @@ void ConcatenateTensorBuffers(const std::vector& input_arrays, // already set (e.g. because of previous pass in TOCO), it doesn't change it and // returns. Otherwise it uses the input arrays min and max values to compute the // concatenated array min and max. -void SetMinMaxForConcatenedArray(const std::vector& input_arrays, +void SetMinMaxForConcatenedArray(GraphTransformation* transformation, + const std::vector& input_arrays, Array* concatenated_array) { CHECK(concatenated_array->data_type == ArrayDataType::kFloat); // If the minmax is already set, use it @@ -125,6 +127,9 @@ void SetMinMaxForConcatenedArray(const std::vector& input_arrays, MinMax& minmax = concatenated_array->GetOrCreateMinMax(); minmax.min = concat_min; minmax.max = concat_max; + + transformation->AddMessageF("Setting concatenated array min/max to %g,%g", + concat_min, concat_max); } } // namespace @@ -161,11 +166,14 @@ bool ResolveConstantConcatenation::Run(Model* model, std::size_t op_index) { input_arrays.push_back(&model->GetArray(input_name)); } + AddMessageF("Performing constant concat of %s into %s", + absl::StrJoin(concat_op->inputs, ", "), concatenated_array_name); + switch (concatenated_array.data_type) { case ArrayDataType::kFloat: ConcatenateTensorBuffers( input_arrays, concatenation_axis, &concatenated_array); - SetMinMaxForConcatenedArray(input_arrays, &concatenated_array); + SetMinMaxForConcatenedArray(this, input_arrays, &concatenated_array); break; case ArrayDataType::kUint8: ConcatenateTensorBuffers( @@ -189,13 +197,13 @@ bool ResolveConstantConcatenation::Run(Model* model, std::size_t op_index) { // Remove all the resolved arrays. for (const string& input_name : concat_op->inputs) { - // Check to prevent removal of shared tensors + // Check to prevent removal of shared tensors. if (CountOpsWithInput(*model, input_name) == 1) { model->EraseArray(input_name); } } - // Remove concatenate operator + // Remove concatenate operator. model->operators.erase(concat_it); return true; } diff --git a/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_gather.cc b/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_gather.cc index d999c2df94..debe298a5a 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_gather.cc +++ b/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_gather.cc @@ -98,6 +98,16 @@ bool ResolveConstantGather::Run(Model* model, std::size_t op_index) { CHECK(coords_array.data_type == ArrayDataType::kInt32) << "Only int32 indices are supported"; + // Copy min/max info if present. The ranges of the selected values may be + // a subset of the original range but we want to ensure the quantization + // params stay the same. + if (input_array.minmax) { + const auto& input_minmax = input_array.GetMinMax(); + auto& output_minmax = output_array.GetOrCreateMinMax(); + output_minmax.min = input_minmax.min; + output_minmax.max = input_minmax.max; + } + CHECK(!output_array.buffer); switch (output_array.data_type) { case ArrayDataType::kFloat: -- GitLab From 77548a7877028614e4c5e0b4c2a8d25660785c6f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 08:11:50 -0700 Subject: [PATCH 158/791] Remove unused former source of tensorflow.org/tutorials/image_retraining. The source of https://tensorflow.org/tutorials/image_retraining has moved from https://github.com/tensorflow/tensorflow/tree/master/tensorflow/docs_src/tutorials to https://github.com/tensorflow/hub/tree/master/docs/tutorials because of its use of TensorFlow Hub. This change replaces the now-defunct version with a pointer to the new location, in order to avoid dead code. PiperOrigin-RevId: 192451570 --- .../docs_src/tutorials/image_retraining.md | 404 +----------------- 1 file changed, 2 insertions(+), 402 deletions(-) diff --git a/tensorflow/docs_src/tutorials/image_retraining.md b/tensorflow/docs_src/tutorials/image_retraining.md index 93d7c86e42..27784eef9c 100644 --- a/tensorflow/docs_src/tutorials/image_retraining.md +++ b/tensorflow/docs_src/tutorials/image_retraining.md @@ -1,404 +1,4 @@ # How to Retrain Inception's Final Layer for New Categories -Modern object recognition models have millions of parameters and can take weeks -to fully train. Transfer learning is a technique that shortcuts a lot of this -work by taking a fully-trained model for a set of categories like ImageNet, and -retrains from the existing weights for new classes. In this example we'll be -retraining the final layer from scratch, while leaving all the others untouched. -For more information on the approach you can see -[this paper on Decaf](https://arxiv.org/pdf/1310.1531v1.pdf). - -Though it's not as good as a full training run, this is surprisingly effective -for many applications, and can be run in as little as thirty minutes on a -laptop, without requiring a GPU. This tutorial will show you how to run the -example script on your own images, and will explain some of the options you have -to help control the training process. - -Note: A version of this tutorial is also available -[as a codelab](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/#0). - -Before you start, you must @{$install$install tensorflow}. - -[TOC] - -## Training on Flowers - -![Daisies by Kelly Sikkema](https://www.tensorflow.org/images/daisies.jpg) - -[Image by Kelly Sikkema](https://www.flickr.com/photos/95072945@N05/9922116524/) - -Before you start any training, you'll need a set of images to teach the network -about the new classes you want to recognize. There's a later section that -explains how to prepare your own images, but to make it easy we've created an -archive of creative-commons licensed flower photos to use initially. To get the -set of flower photos, run these commands: - -```sh -cd ~ -curl -O http://download.tensorflow.org/example_images/flower_photos.tgz -tar xzf flower_photos.tgz -``` - -Once you have the images, you can clone the tensorflow repository using the -following command (these examples are not included in the installation): - -```sh -git clone https://github.com/tensorflow/tensorflow -``` - -Then checkout the version of the tensorflow repository matching your -installation and this tutorial as follows: - -``` sh -cd tensorflow -git checkout {version} -``` - -In the simplest cases the retrainer can then be run like this: - -```sh -python tensorflow/examples/image_retraining/retrain.py --image_dir ~/flower_photos -``` - -The script has many other options. You can get a full listing with: - -```sh -python tensorflow/examples/image_retraining/retrain.py -h -``` - -This script loads the pre-trained Inception v3 model, removes the old top layer, -and trains a new one on the flower photos you've downloaded. None of the flower -species were in the original ImageNet classes the full network was trained on. -The magic of transfer learning is that lower layers that have been trained to -distinguish between some objects can be reused for many recognition tasks -without any alteration. - -## Bottlenecks - -The script can take thirty minutes or more to complete, depending on the speed -of your machine. The first phase analyzes all the images on disk and calculates -the bottleneck values for each of them. 'Bottleneck' is an informal term we -often use for the layer just before the final output layer that actually does -the classification. This penultimate layer has been trained to output a set of -values that's good enough for the classifier to use to distinguish between all -the classes it's been asked to recognize. That means it has to be a meaningful -and compact summary of the images, since it has to contain enough information -for the classifier to make a good choice in a very small set of values. The -reason our final layer retraining can work on new classes is that it turns out -the kind of information needed to distinguish between all the 1,000 classes in -ImageNet is often also useful to distinguish between new kinds of objects. - -Because every image is reused multiple times during training and calculating -each bottleneck takes a significant amount of time, it speeds things up to -cache these bottleneck values on disk so they don't have to be repeatedly -recalculated. By default they're stored in the `/tmp/bottleneck` directory, and -if you rerun the script they'll be reused so you don't have to wait for this -part again. - -## Training - -Once the bottlenecks are complete, the actual training of the top layer of the -network begins. You'll see a series of step outputs, each one showing training -accuracy, validation accuracy, and the cross entropy. The training accuracy -shows what percent of the images used in the current training batch were -labeled with the correct class. The validation accuracy is the precision on a -randomly-selected group of images from a different set. The key difference is -that the training accuracy is based on images that the network has been able -to learn from so the network can overfit to the noise in the training data. A -true measure of the performance of the network is to measure its performance on -a data set not contained in the training data -- this is measured by the -validation accuracy. If the train accuracy is high but the validation accuracy -remains low, that means the network is overfitting and memorizing particular -features in the training images that aren't helpful more generally. Cross -entropy is a loss function which gives a glimpse into how well the learning -process is progressing. The training's objective is to make the loss as small as -possible, so you can tell if the learning is working by keeping an eye on -whether the loss keeps trending downwards, ignoring the short-term noise. - -By default this script will run 4,000 training steps. Each step chooses 100 -images at random from the training set, finds their bottlenecks from the cache, -and feeds them into the final layer to get predictions. Those predictions are -then compared against the actual labels to update the final layer's weights -through the back-propagation process. As the process continues you should see -the reported accuracy improve, and after all the steps are done, a final test -accuracy evaluation is run on a set of images kept separate from the training -and validation pictures. This test evaluation is the best estimate of how the -trained model will perform on the classification task. You should see an -accuracy value of between 90% and 95%, though the exact value will vary from run -to run since there's randomness in the training process. This number is based on -the percent of the images in the test set that are given the correct label -after the model is fully trained. - -## Visualizing the Retraining with TensorBoard - -The script includes TensorBoard summaries that make it easier to understand, debug, and optimize the retraining. For example, you can visualize the graph and statistics, such as how the weights or accuracy varied during training. - -To launch TensorBoard, run this command during or after retraining: - -```sh -tensorboard --logdir /tmp/retrain_logs -``` - -Once TensorBoard is running, navigate your web browser to `localhost:6006` to view the TensorBoard. - -The script will log TensorBoard summaries to `/tmp/retrain_logs` by default. You can change the directory with the `--summaries_dir` flag. - -The [TensorBoard's GitHub](https://github.com/tensorflow/tensorboard) has a lot more information on TensorBoard usage, including tips & tricks, and debugging information. - -## Using the Retrained Model - -The script will write out a version of the Inception v3 network with a final -layer retrained to your categories to /tmp/output_graph.pb, and a text file -containing the labels to /tmp/output_labels.txt. These are both in a format that -the @{$image_recognition$C++ and Python image classification examples} -can read in, so you can start using your new model immediately. Since you've -replaced the top layer, you will need to specify the new name in the script, for -example with the flag `--output_layer=final_result` if you're using label_image. - -Here's an example of how to run the label_image example with your -retrained graphs: - -```sh -python tensorflow/examples/label_image/label_image.py \ ---graph=/tmp/output_graph.pb --labels=/tmp/output_labels.txt \ ---input_layer=Mul \ ---output_layer=final_result \ ---input_mean=128 --input_std=128 \ ---image=$HOME/flower_photos/daisy/21652746_cc379e0eea_m.jpg -``` - -You should see a list of flower labels, in most cases with daisy on top -(though each retrained model may be slightly different). You can replace the -`--image` parameter with your own images to try those out. - -If you'd like to use the retrained model in your own Python program, then the -above -[`label_image` script](https://www.tensorflow.org/code/tensorflow/examples/label_image/label_image.py) -is a reasonable starting point. The `label_image` -directory also contains C++ code which you can use as a template to integrate -tensorflow with your own applications. - -If you find the default Inception v3 model is too large or slow for your -application, take a look at the [Other Model Architectures section](/tutorials/image_retraining#other_model_architectures) -below for options to speed up and slim down your network. - -## Training on Your Own Categories - -If you've managed to get the script working on the flower example images, you -can start looking at teaching it to recognize categories you care about instead. -In theory all you'll need to do is point it at a set of sub-folders, each named -after one of your categories and containing only images from that category. If -you do that and pass the root folder of the subdirectories as the argument to -`--image_dir`, the script should train just like it did for the flowers. - -Here's what the folder structure of the flowers archive looks like, to give you -and example of the kind of layout the script is looking for: - -![Folder Structure](https://www.tensorflow.org/images/folder_structure.png) - -In practice it may take some work to get the accuracy you want. I'll try to -guide you through some of the common problems you might encounter below. - -## Creating a Set of Training Images - -The first place to start is by looking at the images you've gathered, since the -most common issues we see with training come from the data that's being fed in. - -For training to work well, you should gather at least a hundred photos of each -kind of object you want to recognize. The more you can gather, the better the -accuracy of your trained model is likely to be. You also need to make sure that -the photos are a good representation of what your application will actually -encounter. For example, if you take all your photos indoors against a blank wall -and your users are trying to recognize objects outdoors, you probably won't see -good results when you deploy. - -Another pitfall to avoid is that the learning process will pick up on anything -that the labeled images have in common with each other, and if you're not -careful that might be something that's not useful. For example if you photograph -one kind of object in a blue room, and another in a green one, then the model -will end up basing its prediction on the background color, not the features of -the object you actually care about. To avoid this, try to take pictures in as -wide a variety of situations as you can, at different times, and with different -devices. If you want to know more about this problem, you can read about the -classic (and possibly apocryphal) -[tank recognition problem](https://www.jefftk.com/p/detecting-tanks). - -You may also want to think about the categories you use. It might be worth -splitting big categories that cover a lot of different physical forms into -smaller ones that are more visually distinct. For example instead of 'vehicle' -you might use 'car', 'motorbike', and 'truck'. It's also worth thinking about -whether you have a 'closed world' or an 'open world' problem. In a closed world, -the only things you'll ever be asked to categorize are the classes of object you -know about. This might apply to a plant recognition app where you know the user -is likely to be taking a picture of a flower, so all you have to do is decide -which species. By contrast a roaming robot might see all sorts of different -things through its camera as it wanders around the world. In that case you'd -want the classifier to report if it wasn't sure what it was seeing. This can be -hard to do well, but often if you collect a large number of typical 'background' -photos with no relevant objects in them, you can add them to an extra 'unknown' -class in your image folders. - -It's also worth checking to make sure that all of your images are labeled -correctly. Often user-generated tags are unreliable for our purposes, for -example using #daisy for pictures of a person named Daisy. If you go through -your images and weed out any mistakes it can do wonders for your overall -accuracy. - -## Training Steps - -If you're happy with your images, you can take a look at improving your results -by altering the details of the learning process. The simplest one to try is -`--how_many_training_steps`. This defaults to 4,000, but if you increase it to -8,000 it will train for twice as long. The rate of improvement in the accuracy -slows the longer you train for, and at some point will stop altogether, but you -can experiment to see when you hit that limit for your model. - -## Distortions - -A common way of improving the results of image training is by deforming, -cropping, or brightening the training inputs in random ways. This has the -advantage of expanding the effective size of the training data thanks to all the -possible variations of the same images, and tends to help the network learn to -cope with all the distortions that will occur in real-life uses of the -classifier. The biggest disadvantage of enabling these distortions in our script -is that the bottleneck caching is no longer useful, since input images are never -reused exactly. This means the training process takes a lot longer, so I -recommend trying this as a way of fine-tuning your model once you've got one -that you're reasonably happy with. - -You enable these distortions by passing `--random_crop`, `--random_scale` and -`--random_brightness` to the script. These are all percentage values that -control how much of each of the distortions is applied to each image. It's -reasonable to start with values of 5 or 10 for each of them and then experiment -to see which of them help with your application. `--flip_left_right` will -randomly mirror half of the images horizontally, which makes sense as long as -those inversions are likely to happen in your application. For example it -wouldn't be a good idea if you were trying to recognize letters, since flipping -them destroys their meaning. - -## Hyper-parameters - -There are several other parameters you can try adjusting to see if they help -your results. The `--learning_rate` controls the magnitude of the updates to the -final layer during training. Intuitively if this is smaller then the learning -will take longer, but it can end up helping the overall precision. That's not -always the case though, so you need to experiment carefully to see what works -for your case. The `--train_batch_size` controls how many images are examined -during one training step, and because the learning rate is applied per batch -you'll need to reduce it if you have larger batches to get the same overall -effect. - -## Training, Validation, and Testing Sets - -One of the things the script does under the hood when you point it at a folder -of images is divide them up into three different sets. The largest is usually -the training set, which are all the images fed into the network during training, -with the results used to update the model's weights. You might wonder why we -don't use all the images for training? A big potential problem when we're doing -machine learning is that our model may just be memorizing irrelevant details of -the training images to come up with the right answers. For example, you could -imagine a network remembering a pattern in the background of each photo it was -shown, and using that to match labels with objects. It could produce good -results on all the images it's seen before during training, but then fail on new -images because it's not learned general characteristics of the objects, just -memorized unimportant details of the training images. - -This problem is known as overfitting, and to avoid it we keep some of our data -out of the training process, so that the model can't memorize them. We then use -those images as a check to make sure that overfitting isn't occurring, since if -we see good accuracy on them it's a good sign the network isn't overfitting. The -usual split is to put 80% of the images into the main training set, keep 10% -aside to run as validation frequently during training, and then have a final 10% -that are used less often as a testing set to predict the real-world performance -of the classifier. These ratios can be controlled using the -`--testing_percentage` and `--validation_percentage` flags. In general -you should be able to leave these values at their defaults, since you won't -usually find any advantage to training to adjusting them. - -Note that the script uses the image filenames (rather than a completely random -function) to divide the images among the training, validation, and test sets. -This is done to ensure that images don't get moved between training and testing -sets on different runs, since that could be a problem if images that had been -used for training a model were subsequently used in a validation set. - -You might notice that the validation accuracy fluctuates among iterations. Much -of this fluctuation arises from the fact that a random subset of the validation -set is chosen for each validation accuracy measurement. The fluctuations can be -greatly reduced, at the cost of some increase in training time, by choosing -`--validation_batch_size=-1`, which uses the entire validation set for each -accuracy computation. - -Once training is complete, you may find it insightful to examine misclassified -images in the test set. This can be done by adding the flag -`--print_misclassified_test_images`. This may help you get a feeling for which -types of images were most confusing for the model, and which categories were -most difficult to distinguish. For instance, you might discover that some -subtype of a particular category, or some unusual photo angle, is particularly -difficult to identify, which may encourage you to add more training images of -that subtype. Oftentimes, examining misclassified images can also point to -errors in the input data set, such as mislabeled, low-quality, or ambiguous -images. However, one should generally avoid point-fixing individual errors in -the test set, since they are likely to merely reflect more general problems in -the (much larger) training set. - -## Other Model Architectures - -By default the script uses a pretrained version of the Inception v3 model -architecture. This is a good place to start because it provides high accuracy -results, but if you intend to deploy your model on mobile devices or other -resource-constrained environments you may want to trade off a little accuracy -for much smaller file sizes or faster speeds. To help with that, the -[retrain.py script](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/image_retraining/retrain.py) -supports different variations on the [Mobilenet architecture](https://research.googleblog.com/2017/06/mobilenets-open-source-models-for.html). - -These are a little less precise than Inception v3, but can result in far -smaller file sizes (a few megabytes) and can be many times faster -to run. To train with one of these models, pass in the `--architecture` flag, -for example: - -``` -python tensorflow/examples/image_retraining/retrain.py \ - --image_dir ~/flower_photos --architecture mobilenet_0.25_128 -``` - -This will create a 1.9MB model file in `/tmp/output_graph.pb`, with only 25% of -the number of neurons of the full Mobilenet, and trained to take 128x128 sized -input images. - -You can choose '1.0', '0.75', '0.50', or '0.25' to control the number of -neurons (activations of hidden layers); the number of weights (and hence to -some extent the file size and speed) shrinks like the square of that fraction. -You can choose '224', '192', '160', or '128' for the input image size, -with smaller sizes giving faster speeds. - -The speed and size advantages come at a loss to accuracy of course, but for many -purposes this isn't critical. They can also be somewhat offset with improved -training data. For example, training with distortions allows me to get above 80% -accuracy on the flower data set even with the 0.25/128 graph above. - -If you're going to be using the Mobilenet models in label_image or your own -programs, you'll need to feed in an image of the specified size converted to a -float range into the 'input' tensor. Typically 24-bit images are in the range -[0,255], and you must convert them to the [-1,1] float range expected by the -model with the formula `(image - 128.)/128.`. - -The default arguments for the `label_image` script are set for Inception V3. -To use it with a MobileNet, specify the above normalization parameters as -`input_mean` and `input_std` on the command line. You also must specify the -image size that your model expects, as follows: - -```sh -python tensorflow/examples/label_image/label_image.py \ ---graph=/tmp/output_graph.pb --labels=/tmp/output_labels.txt \ ---input_layer=input \ ---output_layer=final_result \ ---input_height=224 --input_width=224 \ ---input_mean=128 --input_std=128 \ ---image=$HOME/flower_photos/daisy/21652746_cc379e0eea_m.jpg -``` - -For more information on deploying the retrained model to a mobile device, see -the [codelab version](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/#0) -of this tutorial, especially [part 2](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets-2-tflite/#0), which describes -[TensorFlow Lite](/mobile/tflite/) and the additional optimizations it offers -(including quantization of model weights). +**NOTE: This tutorial has moved to** +https://github.com/tensorflow/hub/tree/master/docs/tutorials/image_retraining.md -- GitLab From 8e1b323be4b5d56d531b2d5ee7a1fc573a2a0b5f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 08:30:18 -0700 Subject: [PATCH 159/791] Temporarily remove prelu from generated_examples_zip_test PiperOrigin-RevId: 192453411 --- tensorflow/contrib/lite/testing/BUILD | 1 - .../contrib/lite/testing/generate_examples.py | 48 ------------------- 2 files changed, 49 deletions(-) diff --git a/tensorflow/contrib/lite/testing/BUILD b/tensorflow/contrib/lite/testing/BUILD index 198984e7e7..1ce89a25fd 100644 --- a/tensorflow/contrib/lite/testing/BUILD +++ b/tensorflow/contrib/lite/testing/BUILD @@ -42,7 +42,6 @@ gen_zipped_test_files( "minimum.zip", "mul.zip", "pad.zip", - "prelu.zip", "relu.zip", "relu1.zip", "relu6.zip", diff --git a/tensorflow/contrib/lite/testing/generate_examples.py b/tensorflow/contrib/lite/testing/generate_examples.py index 672158aa2f..0e6aceeb86 100644 --- a/tensorflow/contrib/lite/testing/generate_examples.py +++ b/tensorflow/contrib/lite/testing/generate_examples.py @@ -630,54 +630,6 @@ def make_relu6_tests(zip_path): make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs) -def make_prelu_tests(zip_path): - """Make a set of tests to do PReLU.""" - - test_parameters = [{ - # The canonical case for image processing is having a 4D `input` (NHWC) - # and `shared_axes`=[1, 2], so the alpha parameter is per channel. - "input_shape": [[1, 10, 10, 3], [3, 3, 3, 3]], - "shared_axes": [[1, 2], [1]], - }] - - def build_graph(parameters): - """Build the graph for the test case.""" - - input_tensor = tf.placeholder( - dtype=tf.float32, name="input", shape=parameters["input_shape"]) - prelu = tf.keras.layers.PReLU(shared_axes=parameters["shared_axes"]) - out = prelu(input_tensor) - return [input_tensor], [out] - - def build_inputs(parameters, sess, inputs, outputs): - """Build the inputs for the test case.""" - - input_shape = parameters["input_shape"] - input_values = create_tensor_data( - np.float32, input_shape, min_value=-10, max_value=10) - shared_axes = parameters["shared_axes"] - - alpha_shape = [] - for dim in range(1, len(input_shape)): - alpha_shape.append(1 if dim in shared_axes else input_shape[dim]) - - alpha_values = create_tensor_data(np.float32, alpha_shape) - - with tf.variable_scope("", reuse=True): - alpha = tf.get_variable("p_re_lu/alpha") - sess.run(alpha.assign(alpha_values)) - - return [input_values], sess.run( - outputs, feed_dict=dict(zip(inputs, [input_values]))) - - make_zip_of_tests( - zip_path, - test_parameters, - build_graph, - build_inputs, - use_frozen_graph=True) - - # This function tests various TensorFLow functions that generates Const op, # including `tf.ones`, `tf.zeros` and random functions. def make_constant_tests(zip_path): -- GitLab From 0073d1375add58b0493449c356af76aa33455f7d Mon Sep 17 00:00:00 2001 From: Michael Case Date: Wed, 11 Apr 2018 09:34:44 -0700 Subject: [PATCH 160/791] Fix Windows GPU TensorFlow Bazel builds. The configure.py script will error out on Windows GPU builds due to NCCL attempted to be configured (and is currently Linux only). PiperOrigin-RevId: 192461362 --- configure.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configure.py b/configure.py index 81d5ad77ee..8fb8979111 100644 --- a/configure.py +++ b/configure.py @@ -1516,7 +1516,8 @@ def main(): set_tf_cudnn_version(environ_cp) if is_linux(): set_tf_tensorrt_install_path(environ_cp) - set_tf_nccl_install_path(environ_cp) + set_tf_nccl_install_path(environ_cp) + set_tf_cuda_compute_capabilities(environ_cp) if 'LD_LIBRARY_PATH' in environ_cp and environ_cp.get( 'LD_LIBRARY_PATH') != '1': -- GitLab From adfbc272ded60a221444423b1fee58551c6445c7 Mon Sep 17 00:00:00 2001 From: Nupur Garg Date: Wed, 11 Apr 2018 09:34:51 -0700 Subject: [PATCH 161/791] Fixing dependencies. PiperOrigin-RevId: 192461382 --- tensorflow/contrib/lite/python/BUILD | 5 ++++ .../lite/python/convert_saved_model_test.py | 12 +++++----- tensorflow/contrib/saved_model/BUILD | 23 ++++++++++++++----- tensorflow/python/tools/BUILD | 14 ++++------- 4 files changed, 32 insertions(+), 22 deletions(-) diff --git a/tensorflow/contrib/lite/python/BUILD b/tensorflow/contrib/lite/python/BUILD index e735062a7f..6fafaf0727 100644 --- a/tensorflow/contrib/lite/python/BUILD +++ b/tensorflow/contrib/lite/python/BUILD @@ -106,8 +106,13 @@ py_test( deps = [ ":convert_saved_model", "//tensorflow/python:client_testlib", + "//tensorflow/python:layers", + "//tensorflow/python:nn", "//tensorflow/python:platform_test", "//tensorflow/python:session", + "//tensorflow/python/estimator:estimator_py", + "//tensorflow/python/keras", + "//tensorflow/python/ops/losses", "//tensorflow/python/saved_model", ], ) diff --git a/tensorflow/contrib/lite/python/convert_saved_model_test.py b/tensorflow/contrib/lite/python/convert_saved_model_test.py index d87fbeb91c..734e42d619 100644 --- a/tensorflow/contrib/lite/python/convert_saved_model_test.py +++ b/tensorflow/contrib/lite/python/convert_saved_model_test.py @@ -25,21 +25,21 @@ from __future__ import print_function import os from tensorflow.contrib.lite.python import convert_saved_model -from tensorflow.python import estimator from tensorflow.python import keras -from tensorflow.python import layers -from tensorflow.python import losses -from tensorflow.python import nn -from tensorflow.python import saved_model -from tensorflow.python import train from tensorflow.python.client import session +from tensorflow.python.estimator import estimator_lib as estimator from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import test_util +from tensorflow.python.layers import layers from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops import nn from tensorflow.python.ops import random_ops +from tensorflow.python.ops.losses import losses from tensorflow.python.platform import test +from tensorflow.python.saved_model import saved_model +from tensorflow.python.training import training as train class ConvertSavedModelTestBasicGraph(test_util.TensorFlowTestCase): diff --git a/tensorflow/contrib/saved_model/BUILD b/tensorflow/contrib/saved_model/BUILD index e431c464ef..26fd4e2023 100644 --- a/tensorflow/contrib/saved_model/BUILD +++ b/tensorflow/contrib/saved_model/BUILD @@ -48,16 +48,14 @@ py_library( ], ) -py_test( - name = "reader_test", - size = "small", - srcs = ["python/saved_model/reader_test.py"], +py_library( + name = "reader", + srcs = ["python/saved_model/reader.py"], srcs_version = "PY2AND3", tags = ["no_windows"], # TODO: needs investigation on Windows - visibility = ["//visibility:private"], + visibility = ["//visibility:public"], deps = [ ":saved_model_py", - "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:lib", "//tensorflow/python:variables", @@ -66,6 +64,19 @@ py_test( ], ) +py_test( + name = "reader_test", + size = "small", + srcs = ["python/saved_model/reader_test.py"], + srcs_version = "PY2AND3", + tags = ["no_windows"], # TODO: needs investigation on Windows + visibility = ["//visibility:private"], + deps = [ + ":reader", + "//tensorflow/python:client_testlib", + ], +) + py_test( name = "signature_def_utils_test", size = "small", diff --git a/tensorflow/python/tools/BUILD b/tensorflow/python/tools/BUILD index 6e39ce8c80..cc2884a4f6 100644 --- a/tensorflow/python/tools/BUILD +++ b/tensorflow/python/tools/BUILD @@ -28,7 +28,7 @@ py_library( name = "saved_model_utils", srcs = ["saved_model_utils.py"], srcs_version = "PY2AND3", - deps = ["//tensorflow:tensorflow_py"], + deps = ["//tensorflow/contrib/saved_model:reader"], ) py_library( @@ -38,11 +38,12 @@ py_library( deps = [ ":saved_model_utils", "//tensorflow/core:protos_all_py", - "//tensorflow/python", # TODO(b/34059704): remove when fixed "//tensorflow/python:client", "//tensorflow/python:framework", + "//tensorflow/python:parsing_ops", "//tensorflow/python:platform", "//tensorflow/python:training", + "//tensorflow/python/saved_model:loader", "@six_archive//:six", ], ) @@ -52,14 +53,7 @@ py_binary( srcs = ["freeze_graph.py"], srcs_version = "PY2AND3", deps = [ - ":saved_model_utils", - "//tensorflow/core:protos_all_py", - "//tensorflow/python", # TODO(b/34059704): remove when fixed - "//tensorflow/python:client", - "//tensorflow/python:framework", - "//tensorflow/python:platform", - "//tensorflow/python:training", - "@six_archive//:six", + ":freeze_graph_lib", ], ) -- GitLab From a9a3b98a76f1d4a8fb7a02e451fb71147a842f31 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 09:43:32 -0700 Subject: [PATCH 162/791] Import FunctionDef as GrapplerFunctionItem Explicitly track function input arg expansion into Placeholders, and keep metadata to map between FunctionDef and GraphDef connectivity formats. PiperOrigin-RevId: 192462592 --- tensorflow/core/grappler/grappler_item.h | 3 +- .../grappler/optimizers/function_optimizer.cc | 29 +- .../optimizers/function_optimizer_test.cc | 16 +- tensorflow/core/grappler/utils/BUILD | 2 + tensorflow/core/grappler/utils/functions.cc | 385 +++++++++++++----- tensorflow/core/grappler/utils/functions.h | 116 +++++- .../core/grappler/utils/functions_test.cc | 277 +++++++++---- 7 files changed, 627 insertions(+), 201 deletions(-) diff --git a/tensorflow/core/grappler/grappler_item.h b/tensorflow/core/grappler/grappler_item.h index 06bba544c3..45eed47b50 100644 --- a/tensorflow/core/grappler/grappler_item.h +++ b/tensorflow/core/grappler/grappler_item.h @@ -35,8 +35,9 @@ namespace grappler { // nodes, and potentially a set of nodes to feed. // TODO(volunteer_needed): turn this struct into a class. struct GrapplerItem { - GrapplerItem() {} + GrapplerItem() = default; GrapplerItem(const GrapplerItem& other, GraphDef&& graphDef); + virtual ~GrapplerItem() = default; string id; // A unique id for this item diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc index 343c89a9da..6d67ead355 100644 --- a/tensorflow/core/grappler/optimizers/function_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc @@ -38,11 +38,14 @@ class FunctionInliningContext { public: explicit FunctionInliningContext(const GrapplerItem& item, RewriterConfig::Toggle opt_level) - : library_(&item.graph.library()), - opt_level_(opt_level), - functions_(InliningCandidates(item)) {} + : opt_level_(opt_level), + functions_(InliningCandidates(item)), + function_library_(FunctionLibraryDefinition(OpRegistry::Global(), + item.graph.library())) {} - const FunctionDefLibrary& Library() const { return *library_; } + const FunctionLibraryDefinition& FunctionLibrary() const { + return function_library_; + } bool HasInlinedFunctions() const { return !functions_.empty(); } @@ -78,9 +81,9 @@ class FunctionInliningContext { return functions; } - const FunctionDefLibrary* library_; RewriterConfig::Toggle opt_level_; std::unordered_map functions_; + FunctionLibraryDefinition function_library_; TF_DISALLOW_COPY_AND_ASSIGN(FunctionInliningContext); }; @@ -150,11 +153,14 @@ Status InlineFunction(const NodeDef& func_node, const FunctionDef& func, const std::unordered_map func_attr( func_node.attr().begin(), func_node.attr().end()); - std::unique_ptr item = - GrapplerItemFromFunctionDef(func, func_attr, ctx.Library()); - if (!item) { + GrapplerFunctionItem item; + Status item_status = + MakeGrapplerFunctionItem(func, func_attr, ctx.FunctionLibrary(), &item); + + if (!item_status.ok()) { return errors::InvalidArgument("Failed to inline function ", func_node.op(), - " instantiated by ", func_node.name()); + " instantiated by ", func_node.name(), + ". Error: ", item_status.error_message()); } std::unordered_map input_nodes; @@ -168,7 +174,7 @@ Status InlineFunction(const NodeDef& func_node, const FunctionDef& func, TF_RETURN_IF_ERROR( HookInlinedFunctionInputs(func_node, func, func_attr, func_inputs)); - for (NodeDef& func_body_node : *item->graph.mutable_node()) { + for (NodeDef& func_body_node : *item.mutable_function_body().mutable_node()) { if (input_nodes.find(func_body_node.name()) != input_nodes.end()) { CHECK_EQ(0, func_body_node.input_size()); // Turn input placeholders into identity nodes @@ -217,8 +223,9 @@ Status InlineFunction(const NodeDef& func_node, const FunctionDef& func, // Hook inlined function outputs to IdentityN node NodeDef* func_outputs = optimized_graph->add_node(); + std::vector fetch = OutputTensors(item); TF_RETURN_IF_ERROR(HookInlinedFunctionOutputs(func_node, func, func_attr, - item->fetch, func_outputs)); + fetch, func_outputs)); return Status::OK(); } diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc index fe26a56fc2..099fe7caf2 100644 --- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc +++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc @@ -92,13 +92,13 @@ TEST_F(FunctionOptimizerTest, SimpleFunction) { EXPECT_EQ(device, node.device()); EXPECT_EQ(2, node.input_size()); EXPECT_EQ("y/x", node.input(0)); - EXPECT_EQ("y/scale:0", node.input(1)); + EXPECT_EQ("y/scale", node.input(1)); } else if (node.name() == "y") { count++; EXPECT_EQ("IdentityN", node.op()); EXPECT_EQ(device, node.device()); EXPECT_EQ(1, node.input_size()); - EXPECT_EQ("y/y:0", node.input(0)); + EXPECT_EQ("y/y", node.input(0)); } else if (node.name() == "z") { count++; EXPECT_EQ("Identity", node.op()); @@ -180,13 +180,13 @@ TEST_F(FunctionOptimizerTest, FixedTypeFunction) { EXPECT_EQ(device, node.device()); EXPECT_EQ(2, node.input_size()); EXPECT_EQ("y/x", node.input(0)); - EXPECT_EQ("y/two:0", node.input(1)); + EXPECT_EQ("y/two", node.input(1)); } else if (node.name() == "y") { count++; EXPECT_EQ("IdentityN", node.op()); EXPECT_EQ(device, node.device()); EXPECT_EQ(1, node.input_size()); - EXPECT_EQ("y/y:0", node.input(0)); + EXPECT_EQ("y/y", node.input(0)); } else if (node.name() == "z") { count++; EXPECT_EQ("Identity", node.op()); @@ -264,13 +264,13 @@ TEST_F(FunctionOptimizerTest, FunctionWithOutputMapping) { EXPECT_EQ("Exp", node.op()); EXPECT_EQ(device, node.device()); EXPECT_EQ(1, node.input_size()); - EXPECT_EQ("y/Linear_func:0", node.input(0)); + EXPECT_EQ("y/Linear_func", node.input(0)); } else if (node.name() == "y") { count++; EXPECT_EQ("IdentityN", node.op()); EXPECT_EQ(device, node.device()); EXPECT_EQ(1, node.input_size()); - EXPECT_EQ("y/Exp:0", node.input(0)); + EXPECT_EQ("y/Exp", node.input(0)); } else if (node.name() == "z") { count++; EXPECT_EQ("Identity", node.op()); @@ -453,12 +453,12 @@ TEST_F(FunctionOptimizerTest, InlineFunctionWithNestedFunctionCall) { EXPECT_EQ("IdentityN", node.op()); EXPECT_EQ(kDevice, node.device()); EXPECT_EQ(1, node.input_size()); - EXPECT_EQ("square/output/output:0", node.input(0)); + EXPECT_EQ("square/output/output", node.input(0)); } else if (node.name() == "square" && count++) { EXPECT_EQ("IdentityN", node.op()); EXPECT_EQ(kDevice, node.device()); EXPECT_EQ(1, node.input_size()); - EXPECT_EQ("square/output:0", node.input(0)); + EXPECT_EQ("square/output", node.input(0)); } else if (node.name() == "outputs" && count++) { EXPECT_EQ("Identity", node.op()); EXPECT_EQ(kDevice, node.device()); diff --git a/tensorflow/core/grappler/utils/BUILD b/tensorflow/core/grappler/utils/BUILD index 7419c26dff..05d9cbaa2b 100644 --- a/tensorflow/core/grappler/utils/BUILD +++ b/tensorflow/core/grappler/utils/BUILD @@ -161,6 +161,8 @@ cc_library( deps = [ "//tensorflow/core:framework", "//tensorflow/core:framework_internal", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", "//tensorflow/core:protos_all_cc", "//tensorflow/core/grappler:grappler_item", "//tensorflow/core/grappler:utils", diff --git a/tensorflow/core/grappler/utils/functions.cc b/tensorflow/core/grappler/utils/functions.cc index 4f286ce1c8..dd0d918e72 100644 --- a/tensorflow/core/grappler/utils/functions.cc +++ b/tensorflow/core/grappler/utils/functions.cc @@ -24,50 +24,285 @@ limitations under the License. #include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/types.pb.h" #include "tensorflow/core/grappler/utils.h" +#include "tensorflow/core/lib/strings/scanner.h" namespace tensorflow { namespace grappler { -std::unique_ptr GrapplerItemFromFunctionDef( - const FunctionDef& func, - const std::unordered_map& func_attr, - const FunctionDefLibrary& library) { - if (func.signature().name().empty()) { - LOG(ERROR) << "function name must be specified."; - return nullptr; +void GrapplerFunctionConnectivity::RegisterInputArgExpansion( + const InputArgExpansion& input_arg_expansion) { + input_arg_expansions_.insert( + {input_arg_expansion.input_name, input_arg_expansion}); +} + +void GrapplerFunctionConnectivity::RegisterFunctionBodyOutputs( + const string& node_name, const tensorflow::NameRangeMap& outputs) { + function_body_outputs_.insert({node_name, outputs}); +} + +Status GrapplerFunctionConnectivity::ExpandFunctionDefInput( + const string& func_def_input, std::vector* graph_def_inputs) const { + using ::tensorflow::strings::Scanner; + + // Parse input format: "node_name[:node_output][:position]" + string node_name; + string node_output; + int position = -1; + + StringPiece capture; + StringPiece remaining; + + // Parse "node_name" + if (Scanner(func_def_input) + .One(strings::Scanner::LETTER_DIGIT_DOT_UNDERSCORE) + .Any(strings::Scanner::LETTER_DIGIT_DASH_DOT_SLASH_UNDERSCORE) + .GetResult(&remaining, &capture)) { + node_name = string(capture.data(), capture.size()); } - std::unique_ptr new_item(new GrapplerItem()); - new_item->id = func.signature().name(); - - std::unordered_map port_map; - - // Add the function inputs as placeholder - for (const auto& inp : func.signature().input_arg()) { - NodeDef* ph = new_item->graph.add_node(); - ph->set_name(inp.name()); - ph->set_op("Placeholder"); - if (inp.type() != DT_INVALID) { - (*ph->mutable_attr())["T"].set_type(inp.type()); - } else { - auto it = func_attr.find(inp.type_attr()); - if (it == func_attr.end()) { - LOG(ERROR) << "Unknown type attribute " << inp.type_attr() - << " for function input " << inp.name(); - return nullptr; + + // Parse "node_output" if it exists + if (Scanner(remaining) + .OneLiteral(":") + .RestartCapture() + .One(strings::Scanner::LOWERLETTER) + .Any(strings::Scanner::LETTER_DIGIT_UNDERSCORE) + .GetResult(&remaining, &capture)) { + node_output = string(capture.data(), capture.size()); + } + + // Parse "position" if it exists + if (Scanner(remaining) + .OneLiteral(":") + .RestartCapture() + .Many(strings::Scanner::DIGIT) + .GetResult(nullptr, &capture)) { + CHECK(strings::safe_strto32(capture, &position)); + } + + // If "node_output" is not empty, it must be an output of a function body node + bool is_function_body_output = !node_output.empty(); + + // Function input argument: "node_name[:position]" + if (!is_function_body_output) { + auto input_arg = input_arg_expansions_.find(node_name); + if (input_arg != input_arg_expansions_.end()) { + const InputArgExpansion& input_arg_expansion = input_arg->second; + const auto& placeholders = input_arg_expansion.placeholders; + + if (position == -1) { + // If position is not defined use all placeholders + graph_def_inputs->reserve(placeholders.size()); + for (const string& placeholder : placeholders) { + graph_def_inputs->push_back(placeholder); + } } else { - (*ph->mutable_attr())["T"] = it->second; + if (position > input_arg_expansion.placeholders.size() - 1) { + return errors::InvalidArgument("Invalid input ", node_name, + "position: ", position, + " (out of range)"); + } + graph_def_inputs->push_back(input_arg_expansion.placeholders[position]); + } + + return Status::OK(); + } + } + + // Function body output: "node_name:node_output[:position]" + if (is_function_body_output) { + auto function_body_outputs = function_body_outputs_.find(node_name); + if (function_body_outputs != function_body_outputs_.end()) { + const tensorflow::NameRangeMap& outputs = function_body_outputs->second; + auto output = outputs.find(node_output); + if (output != outputs.end()) { + const auto& output_range = output->second; + + if (position == -1) { + // If position is not defined expand node output range + for (int i = output_range.first; i < output_range.second; ++i) { + i == 0 ? graph_def_inputs->push_back(node_name) + : graph_def_inputs->push_back( + strings::StrCat(node_name, ":", i)); + } + } else { + if (position > (output_range.second - output_range.first)) { + return errors::InvalidArgument( + "Invalid node ", node_name, " output ", node_output, + " position: ", position, " (out of range)"); + } + int pos = output_range.first + position; + pos == 0 ? graph_def_inputs->push_back(node_name) + : graph_def_inputs->push_back( + strings::StrCat(node_name, ":", pos)); + } + + return Status::OK(); } } - port_map[inp.name()] = inp.name(); } - // Add the function body to the graph. - FunctionLibraryDefinition func_def(OpRegistry::Global(), library); + return errors::InvalidArgument("Failed to expand a function def input: ", + func_def_input); +} + +Status GrapplerFunctionConnectivity::ExpandNodeInputs( + NodeDef* function_body_node) const { + std::vector expanded_inputs; + + for (const string& function_def_input : function_body_node->input()) { + if (!IsControlInput(function_def_input)) + TF_RETURN_IF_ERROR( + ExpandFunctionDefInput(function_def_input, &expanded_inputs)); + else + expanded_inputs.push_back(function_def_input); + } + + function_body_node->clear_input(); + for (const string& expanded_input : expanded_inputs) + function_body_node->add_input(expanded_input); + return Status::OK(); +} + +Status GrapplerFunctionItemBuilder::GetTypeAttr(const string& type_attr_name, + DataType* data_type) const { + auto it = func_attr_->find(type_attr_name); + if (it == func_attr_->end()) { + return errors::InvalidArgument("Type attribute ", type_attr_name, + " is not defined"); + } else if (it->second.type() == DT_INVALID) { + return errors::InvalidArgument("Type attribute ", type_attr_name, + " is not defined with a valid type"); + } else { + *data_type = it->second.type(); + } + return Status::OK(); +} + +Status GrapplerFunctionItemBuilder::GetArgType(const OpDef::ArgDef& arg, + DataType* data_type) const { + if (arg.type() != DT_INVALID) { + *data_type = arg.type(); + } else { + TF_RETURN_IF_ERROR(GetTypeAttr(arg.type_attr(), data_type)); + } + return Status::OK(); +} + +GrapplerFunctionItem::GrapplerFunctionItem( + const string& function_name, + const std::vector& input_arg_expansions, + const std::vector& output_arg_expansions, + GraphDef&& function_body) + : function_name_(function_name), + input_arg_expansions_(input_arg_expansions), + output_arg_expansions_(output_arg_expansions) { + graph.Swap(&function_body); +} + +const string& GrapplerFunctionItem::function_name() const { + return function_name_; +} + +const std::vector& GrapplerFunctionItem::inputs() const { + return input_arg_expansions_; +} + +const InputArgExpansion& GrapplerFunctionItem::input(int i) const { + return input_arg_expansions_[i]; +} + +const std::size_t GrapplerFunctionItem::input_size() const { + return input_arg_expansions_.size(); +} + +const std::vector& GrapplerFunctionItem::outputs() const { + return output_arg_expansions_; +} + +const OutputArgExpansion& GrapplerFunctionItem::output(int i) const { + return output_arg_expansions_[i]; +} + +const std::size_t GrapplerFunctionItem::output_size() const { + return output_arg_expansions_.size(); +} + +const GraphDef& GrapplerFunctionItem::function_body() const { return graph; } + +GraphDef& GrapplerFunctionItem::mutable_function_body() { return graph; } + +std::vector OutputTensors(const GrapplerFunctionItem& item) { + std::vector output_tensors; + for (const OutputArgExpansion& output : item.outputs()) { + for (const string& tensor : output.output_tensors) { + output_tensors.push_back(tensor); + } + } + return output_tensors; +} + +Status MakeGrapplerFunctionItem( + const FunctionDef& func, + const std::unordered_map& func_attr, + const FunctionLibraryDefinition& func_library, GrapplerFunctionItem* item) { + const OpDef& signature = func.signature(); + + if (signature.name().empty()) { + return errors::InvalidArgument("Function name must be specified"); + } + + // Helper methods to lookup function attributes + GrapplerFunctionItemBuilder builder(&func_attr); + + // Mapping from FunctionDef input format (name[:output][:position]) to + // GraphDef input format (name[:position]) + GrapplerFunctionConnectivity connectivity; + + std::vector inputs; + std::vector outputs; + GraphDef function_body; + + // TODO(ezhulenev): support functions with tensor sequence inputs/outputs + + // Make sure that there is no tensor sequences in outputs + for (const OpDef::ArgDef& output : signature.output_arg()) { + if (!output.type_list_attr().empty() || !output.number_attr().empty()) { + return errors::InvalidArgument( + "Outputs with sequence of tensors are not supported. Unsupported " + "output: ", + output.name()); + } + } + + // For each input argument create a placeholder in function body. + for (const OpDef::ArgDef& input : signature.input_arg()) { + if (!input.type_list_attr().empty() || !input.number_attr().empty()) { + return errors::InvalidArgument( + "Inputs with sequence of tensors are not supported. Unsupported " + "input: ", + input.name()); + } + + DataType input_data_type; + TF_RETURN_IF_ERROR(builder.GetArgType(input, &input_data_type)); + + NodeDef* placeholder = function_body.add_node(); + placeholder->set_name(input.name()); + placeholder->set_op("Placeholder"); + (*placeholder->mutable_attr())["T"].set_type(input_data_type); + + InputArgExpansion input_expansion{/*input_name=*/input.name(), + /*placeholders=*/{input.name()}}; + connectivity.RegisterInputArgExpansion(input_expansion); + inputs.push_back(input_expansion); + } + + // Add all function nodes to the function body + for (const NodeDef& func_def_node : func.node_def()) { + NodeDef* new_node = function_body.add_node(); + *new_node = func_def_node; - for (const NodeDef& node : func.node_def()) { - NodeDef* new_node = new_item->graph.add_node(); - *new_node = node; - // Replace the placeholder attribute values with the specified value. + // Replace the placeholder attribute values with the specified value for (auto& attr : *new_node->mutable_attr()) { const string& ph_name = attr.second.placeholder(); auto it = func_attr.find(ph_name); @@ -78,75 +313,39 @@ std::unique_ptr GrapplerItemFromFunctionDef( // Functions use a custom format to encode connectivity. Map these custom // strings to regular ones. + tensorflow::NameRangeMap outputs_range_map; const OpRegistrationData* registration; - Status status = func_def.LookUp(node.op(), ®istration); - if (!status.ok()) { - LOG(ERROR) << "Op " << node.op() << " not registered: " << status; - return nullptr; - } - - tensorflow::NameRangeMap inputs; - tensorflow::NameRangeMap outputs; - status = tensorflow::NameRangesForNode(node, registration->op_def, &inputs, - &outputs); - if (!status.ok()) { - LOG(ERROR) << "Op " << node.op() << " invalid: " << status; - return nullptr; - } - for (const auto& name_range : outputs) { - string port_prefix = - strings::StrCat(node.name(), ":", name_range.first, ":"); - int index_start = name_range.second.first; - int index_end = name_range.second.second; - for (int i = index_start; i < index_end; ++i) { - string port_id = strings::StrCat(port_prefix, i - index_start); - string port_name = strings::StrCat(node.name(), ":", i); - port_map[port_id] = port_name; - } - } + TF_RETURN_IF_ERROR(func_library.LookUp(func_def_node.op(), ®istration)); + TF_RETURN_IF_ERROR(tensorflow::NameRangesForNode( + func_def_node, registration->op_def, nullptr, &outputs_range_map)); + connectivity.RegisterFunctionBodyOutputs(func_def_node.name(), + outputs_range_map); } - for (auto& node : *new_item->graph.mutable_node()) { - // Rewrite the inputs to use the normal naming convention. - for (int i = 0; i < node.input_size(); ++i) { - const string& input = node.input(i); - if (IsControlInput(input)) { - // No need to remap control dependencies. - continue; - } else { - auto it = port_map.find(input); - if (it == port_map.end()) { - LOG(ERROR) << "Unknown input: " << input; - return nullptr; - } - node.set_input(i, it->second); - } - } + // Rewrite inputs to use GraphDef format + for (NodeDef& node : *function_body.mutable_node()) { + TF_RETURN_IF_ERROR(connectivity.ExpandNodeInputs(&node)); } - // Add the function outputs to the list of fetch nodes, taking into account - // the output mapping if any. - for (const auto& out : func.signature().output_arg()) { - auto it = func.ret().find(out.name()); - if (it != func.ret().end()) { - auto it2 = port_map.find(it->second); - if (it2 == port_map.end()) { - LOG(ERROR) << "Unknown output mapping: " << it->first << " to " - << it->second; - return nullptr; - } else { - new_item->fetch.emplace_back(it2->second); - } + // Add function outputs + for (const OpDef::ArgDef& out : signature.output_arg()) { + std::vector output_tensors; + auto ret = func.ret().find(out.name()); + if (ret != func.ret().end()) { + // Expand outputs using provided output mapping + TF_RETURN_IF_ERROR( + connectivity.ExpandFunctionDefInput(ret->second, &output_tensors)); } else { - new_item->fetch.emplace_back(out.name()); + // Otherwise output must be one of the function inputs + TF_RETURN_IF_ERROR( + connectivity.ExpandFunctionDefInput(out.name(), &output_tensors)); } - } - // Add the function inputs to the list of feeds. - for (const auto& inp : func.signature().input_arg()) { - new_item->feed.emplace_back(inp.name(), Tensor()); + outputs.push_back({out.name(), output_tensors}); } - return new_item; + *item = GrapplerFunctionItem(signature.name(), inputs, outputs, + std::move(function_body)); + return Status::OK(); } } // end namespace grappler diff --git a/tensorflow/core/grappler/utils/functions.h b/tensorflow/core/grappler/utils/functions.h index 8f9b7d848a..60ea8857c0 100644 --- a/tensorflow/core/grappler/utils/functions.h +++ b/tensorflow/core/grappler/utils/functions.h @@ -19,19 +19,125 @@ limitations under the License. #include #include #include "tensorflow/core/framework/attr_value.pb.h" +#include "tensorflow/core/framework/function.h" #include "tensorflow/core/framework/function.pb.h" +#include "tensorflow/core/framework/node_def_util.h" +#include "tensorflow/core/framework/op_def.pb.h" #include "tensorflow/core/grappler/grappler_item.h" namespace tensorflow { - namespace grappler { -// Factory method for creating a GrapplerItem from a FunctionDef. -// Returns nullptr if the given function def cannot be converted. -std::unique_ptr GrapplerItemFromFunctionDef( +// Depending on the function instantiation attributes, input argument to the +// function might be a single tensor, list of tensors of the same type, or a +// list of tensors of different types. +// +// InputArgExpansion keeps track of the placeholders that were added to the +// function body in place of function inputs. +struct InputArgExpansion { + string input_name; // name of the function input argument + std::vector placeholders; // names of placeholder nodes in the + // function body +}; + +// Depending on the function instantiation attributes, output argument is mapped +// to one or more outputs of one of the function body nodes. +// +// OutputArgExpansion keeps mapping from a function output arg to the output +// tensors of a function body nodes, that compute function outputs. +struct OutputArgExpansion { + string output_name; // name of the function output argument + std::vector output_tensors; // names of output tensors from the + // function body graph nodes +}; + +// FunctionDef uses different connectivity encoding for the function body nodes, +// then a GraphDef (see function.proto for details). Input name in FunctionDef +// can potentially represent a sequence of tensors (instead just one tensor in +// GraphDef), we need to expand it when converting from FunctionDef to GraphDef, +// and fold it back when doing backward conversion. +class GrapplerFunctionConnectivity { + public: + void RegisterInputArgExpansion(const InputArgExpansion& input_arg_expansion); + void RegisterFunctionBodyOutputs(const string& node_name, + const tensorflow::NameRangeMap& outputs); + + // Expand input encoded in FunctionDef format (name[:output][:position]) into + // multiple inputs in GraphDef format (name[:position]). + Status ExpandFunctionDefInput(const string& func_def_input, + std::vector* graph_def_inputs) const; + + // Update Node inputs from FunctionDef to GraphDef format + Status ExpandNodeInputs(NodeDef* function_body_node) const; + + // TODO(ezhulenev): fold GraphDef inputs back to FunctionDef format + // Status FoldGraphDefInputs(const std::vector graph_def_inputs, + // std::vector* function_def_inputs) const; + + private: + std::unordered_map input_arg_expansions_; + std::unordered_map function_body_outputs_; +}; + +// Helper methods to build GrapplerFunctionItem from a function def and function +// attributes. +class GrapplerFunctionItemBuilder { + public: + using FunctionAttr = std::unordered_map; + + explicit GrapplerFunctionItemBuilder(const FunctionAttr* func_attr) + : func_attr_(func_attr) {} + + // Get DataType from attributes by name. Return error if attribute is missing, + // or it doesn't define a valid data type. + Status GetTypeAttr(const string& type_attr_name, DataType* data_type) const; + + // Get argument data type. If data type is not explicitly defined, uses + // provided attribute name to look it up in function attributes. + Status GetArgType(const OpDef::ArgDef& arg, DataType* data_type) const; + + private: + const FunctionAttr* func_attr_; // do not own +}; + +// A special case of GrapplerItem, constructed from a TensorFlow Function. +class GrapplerFunctionItem : public GrapplerItem { + public: + GrapplerFunctionItem() {} + GrapplerFunctionItem( + const string& function_name, + const std::vector& input_arg_expansions, + const std::vector& output_arg_expansions, + GraphDef&& function_body); + + const string& function_name() const; + + const std::vector& inputs() const; + const InputArgExpansion& input(int i) const; + const std::size_t input_size() const; + + const std::vector& outputs() const; + const OutputArgExpansion& output(int i) const; + const std::size_t output_size() const; + + const GraphDef& function_body() const; + GraphDef& mutable_function_body(); + + private: + string function_name_; + std::vector input_arg_expansions_; + std::vector output_arg_expansions_; +}; + +// Return all output tensors referenced by item output args. +std::vector OutputTensors(const GrapplerFunctionItem& item); + +// Make a GrapplerFunctionItem from the function definition and attributes. +// Return error if the given function def cannot be converted. +Status MakeGrapplerFunctionItem( const FunctionDef& func, const std::unordered_map& func_attr, - const FunctionDefLibrary& library); + const FunctionLibraryDefinition& func_library, GrapplerFunctionItem* item); } // end namespace grappler } // end namespace tensorflow diff --git a/tensorflow/core/grappler/utils/functions_test.cc b/tensorflow/core/grappler/utils/functions_test.cc index 6a7d766b1c..1eb3298e89 100644 --- a/tensorflow/core/grappler/utils/functions_test.cc +++ b/tensorflow/core/grappler/utils/functions_test.cc @@ -19,6 +19,7 @@ limitations under the License. #include "tensorflow/core/framework/node_def.pb.h" #include "tensorflow/core/framework/node_def_util.h" #include "tensorflow/core/framework/tensor_testutil.h" +#include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/platform/test.h" #include "tensorflow/core/protobuf/meta_graph.pb.h" @@ -28,6 +29,88 @@ namespace { class FunctionsTest : public ::testing::Test {}; +TEST_F(FunctionsTest, GrapplerFunctionConnectivity_ExpandFunctionDefInput) { + GrapplerFunctionConnectivity connectivity; + + connectivity.RegisterInputArgExpansion({"inputA", {"inputA"}}); + connectivity.RegisterInputArgExpansion({"inputB", {"inputB_0", "inputB_1"}}); + + connectivity.RegisterFunctionBodyOutputs("Add", {{"z", {0, 1}}}); + connectivity.RegisterFunctionBodyOutputs("Func", + {{"o1", {0, 2}}, {"o2", {2, 4}}}); + + std::vector inputs; + TF_EXPECT_OK(connectivity.ExpandFunctionDefInput("inputA", &inputs)); + ASSERT_EQ(1, inputs.size()); + EXPECT_EQ("inputA", inputs[0]); + + inputs.clear(); + TF_EXPECT_OK(connectivity.ExpandFunctionDefInput("inputB", &inputs)); + ASSERT_EQ(2, inputs.size()); + EXPECT_EQ("inputB_0", inputs[0]); + EXPECT_EQ("inputB_1", inputs[1]); + + inputs.clear(); + TF_EXPECT_OK(connectivity.ExpandFunctionDefInput("inputB:1", &inputs)); + ASSERT_EQ(1, inputs.size()); + EXPECT_EQ("inputB_1", inputs[0]); + + inputs.clear(); + TF_EXPECT_OK(connectivity.ExpandFunctionDefInput("Add:z", &inputs)); + ASSERT_EQ(1, inputs.size()); + EXPECT_EQ("Add", inputs[0]); + + inputs.clear(); + TF_EXPECT_OK(connectivity.ExpandFunctionDefInput("Func:o1", &inputs)); + ASSERT_EQ(2, inputs.size()); + EXPECT_EQ("Func", inputs[0]); + EXPECT_EQ("Func:1", inputs[1]); + + inputs.clear(); + TF_EXPECT_OK(connectivity.ExpandFunctionDefInput("Func:o2", &inputs)); + ASSERT_EQ(2, inputs.size()); + EXPECT_EQ("Func:2", inputs[0]); + EXPECT_EQ("Func:3", inputs[1]); + + inputs.clear(); + TF_EXPECT_OK(connectivity.ExpandFunctionDefInput("Func:o1:0", &inputs)); + ASSERT_EQ(1, inputs.size()); + EXPECT_EQ("Func", inputs[0]); + + inputs.clear(); + TF_EXPECT_OK(connectivity.ExpandFunctionDefInput("Func:o1:1", &inputs)); + ASSERT_EQ(1, inputs.size()); + EXPECT_EQ("Func:1", inputs[0]); + + inputs.clear(); + TF_EXPECT_OK(connectivity.ExpandFunctionDefInput("Func:o2:0", &inputs)); + ASSERT_EQ(1, inputs.size()); + EXPECT_EQ("Func:2", inputs[0]); + + inputs.clear(); + TF_EXPECT_OK(connectivity.ExpandFunctionDefInput("Func:o2:1", &inputs)); + ASSERT_EQ(1, inputs.size()); + EXPECT_EQ("Func:3", inputs[0]); +} + +TEST_F(FunctionsTest, GrapplerFunctionConnectivity_ExpandNodeInputs) { + GrapplerFunctionConnectivity connectivity; + + connectivity.RegisterInputArgExpansion({"inputA", {"inputA"}}); + connectivity.RegisterInputArgExpansion({"inputB", {"inputB_0", "inputB_1"}}); + + NodeDef node; + node.add_input("inputA:0"); + node.add_input("inputB"); + + TF_EXPECT_OK(connectivity.ExpandNodeInputs(&node)); + + EXPECT_EQ(3, node.input_size()); + EXPECT_EQ("inputA", node.input(0)); + EXPECT_EQ("inputB_0", node.input(1)); + EXPECT_EQ("inputB_1", node.input(2)); +} + TEST_F(FunctionsTest, FromSimpleFunctionDef) { const Tensor kTwo = test::AsScalar(2); FunctionDef func = FunctionDefHelper::Define( @@ -48,37 +131,45 @@ TEST_F(FunctionsTest, FromSimpleFunctionDef) { std::unordered_map func_attr; func_attr["T"].set_type(DT_FLOAT); - FunctionDefLibrary library; - std::unique_ptr item = - GrapplerItemFromFunctionDef(func, func_attr, library); - CHECK(item); - EXPECT_EQ("XTimesTwo", item->id); - EXPECT_EQ(4, item->graph.node_size()); - EXPECT_EQ(std::vector({"y:0"}), item->fetch); - EXPECT_EQ(1, item->feed.size()); - EXPECT_EQ("x", item->feed[0].first); - - for (const NodeDef &node : item->graph.node()) { - if (node.name() == "x") { + FunctionLibraryDefinition library(OpRegistry::Global(), FunctionDefLibrary()); + + GrapplerFunctionItem item; + TF_EXPECT_OK(MakeGrapplerFunctionItem(func, func_attr, library, &item)); + + EXPECT_EQ("XTimesTwo", item.function_name()); + EXPECT_EQ(4, item.function_body().node_size()); + + EXPECT_EQ(1, item.input_size()); + EXPECT_EQ("x", item.input(0).input_name); + EXPECT_EQ(std::vector{"x"}, item.input(0).placeholders); + + EXPECT_EQ(1, item.output_size()); + EXPECT_EQ("y", item.output(0).output_name); + EXPECT_EQ("y", item.output(0).output_tensors[0]); + + int count = 0; + for (const NodeDef &node : item.function_body().node()) { + if (node.name() == "x" && count++) { EXPECT_EQ("Placeholder", node.op()); EXPECT_EQ(DT_FLOAT, node.attr().at("T").type()); EXPECT_EQ(0, node.input_size()); - } else if (node.name() == "two") { + } else if (node.name() == "two" && count++) { EXPECT_EQ("Const", node.op()); EXPECT_EQ(0, node.input_size()); - } else if (node.name() == "scale") { + } else if (node.name() == "scale" && count++) { EXPECT_EQ("Cast", node.op()); EXPECT_EQ(DT_FLOAT, node.attr().at("DstT").type()); EXPECT_EQ(1, node.input_size()); - EXPECT_EQ("two:0", node.input(0)); - } else if (node.name() == "y") { + EXPECT_EQ("two", node.input(0)); + } else if (node.name() == "y" && count++) { EXPECT_EQ("Mul", node.op()); EXPECT_EQ(DT_FLOAT, node.attr().at("T").type()); EXPECT_EQ(2, node.input_size()); EXPECT_EQ("x", node.input(0)); - EXPECT_EQ("scale:0", node.input(1)); + EXPECT_EQ("scale", node.input(1)); } } + EXPECT_EQ(4, count); } TEST_F(FunctionsTest, FromFunctionDefWithMultiOutputNodes) { @@ -115,45 +206,53 @@ TEST_F(FunctionsTest, FromFunctionDefWithMultiOutputNodes) { std::unordered_map func_attr; func_attr["T"].set_type(DT_FLOAT); - FunctionDefLibrary library; - std::unique_ptr item = - GrapplerItemFromFunctionDef(func, func_attr, library); - CHECK(item); - EXPECT_EQ("SubGrad", item->id); - EXPECT_EQ(12, item->graph.node_size()); - EXPECT_EQ(std::vector({"dx:0", "dy:0"}), item->fetch); - EXPECT_EQ(3, item->feed.size()); - EXPECT_EQ("x", item->feed[0].first); - EXPECT_EQ("y", item->feed[1].first); - EXPECT_EQ("dz", item->feed[2].first); - - for (const NodeDef &node : item->graph.node()) { + FunctionLibraryDefinition library(OpRegistry::Global(), FunctionDefLibrary()); + + GrapplerFunctionItem item; + TF_EXPECT_OK(MakeGrapplerFunctionItem(func, func_attr, library, &item)); + + EXPECT_EQ("SubGrad", item.function_name()); + EXPECT_EQ(12, item.function_body().node_size()); + + ASSERT_EQ(3, item.input_size()); + EXPECT_EQ("x", item.input(0).input_name); + EXPECT_EQ("y", item.input(1).input_name); + EXPECT_EQ("dz", item.input(2).input_name); + + ASSERT_EQ(2, item.output_size()); + EXPECT_EQ("dx", item.output(0).output_tensors[0]); + EXPECT_EQ("dy", item.output(1).output_tensors[0]); + + int count = 0; + for (const NodeDef &node : item.function_body().node()) { if (node.name() == "x" || node.name() == "y" || node.name() == "dz") { + count++; EXPECT_EQ("Placeholder", node.op()); EXPECT_EQ(DT_FLOAT, node.attr().at("T").type()); EXPECT_EQ(0, node.input_size()); - } else if (node.name() == "rx") { + } else if (node.name() == "rx" && count++) { EXPECT_EQ("BroadcastGradientArgs", node.op()); EXPECT_EQ(2, node.input_size()); - EXPECT_EQ("sx:0", node.input(0)); - EXPECT_EQ("sy:0", node.input(1)); - } else if (node.name() == "sum_gx") { + EXPECT_EQ("sx", node.input(0)); + EXPECT_EQ("sy", node.input(1)); + } else if (node.name() == "sum_gx" && count++) { EXPECT_EQ("Sum", node.op()); EXPECT_EQ(2, node.input_size()); - EXPECT_EQ("gx:0", node.input(0)); - EXPECT_EQ("rx:0", node.input(1)); - } else if (node.name() == "sum_gy") { + EXPECT_EQ("gx", node.input(0)); + EXPECT_EQ("rx", node.input(1)); + } else if (node.name() == "sum_gy" && count++) { EXPECT_EQ("Sum", node.op()); EXPECT_EQ(2, node.input_size()); - EXPECT_EQ("gy:0", node.input(0)); + EXPECT_EQ("gy", node.input(0)); EXPECT_EQ("rx:1", node.input(1)); } } + EXPECT_EQ(6, count); } TEST_F(FunctionsTest, FromFunctionDefWithNestedFuncs) { - FunctionDefLibrary library; - *library.add_function() = FunctionDefHelper::Define( + FunctionLibraryDefinition library(OpRegistry::Global(), FunctionDefLibrary()); + TF_ASSERT_OK(library.AddFunctionDef(FunctionDefHelper::Define( // Name "Swap", // Args @@ -164,7 +263,7 @@ TEST_F(FunctionsTest, FromFunctionDefWithNestedFuncs) { {"T: {float, double}"}, // Nodes {{{"o0"}, "Identity", {"i1"}, {{"T", "$T"}}}, - {{"o1"}, "Identity", {"i0"}, {{"T", "$T"}}}}); + {{"o1"}, "Identity", {"i0"}, {{"T", "$T"}}}}))); FunctionDef func = FunctionDefHelper::Create( // Name @@ -189,43 +288,47 @@ TEST_F(FunctionsTest, FromFunctionDefWithNestedFuncs) { std::unordered_map func_attr; func_attr["T"].set_type(DT_FLOAT); - std::unique_ptr item = - GrapplerItemFromFunctionDef(func, func_attr, library); - for (const NodeDef &node : item->graph.node()) { + GrapplerFunctionItem item; + TF_EXPECT_OK(MakeGrapplerFunctionItem(func, func_attr, library, &item)); + + int count = 0; + for (const NodeDef &node : item.function_body().node()) { if (node.name() == "x" || node.name() == "y") { + count++; EXPECT_EQ("Placeholder", node.op()); EXPECT_EQ(DT_FLOAT, node.attr().at("T").type()); EXPECT_EQ(0, node.input_size()); - } else if (node.name() == "a0") { + } else if (node.name() == "a0" && count++) { EXPECT_EQ("Swap", node.op()); EXPECT_EQ(3, node.input_size()); EXPECT_EQ("x", node.input(0)); EXPECT_EQ("y", node.input(1)); EXPECT_EQ("^x2", node.input(2)); - } else if (node.name() == "a1") { + } else if (node.name() == "a1" && count++) { EXPECT_EQ("Swap", node.op()); EXPECT_EQ(2, node.input_size()); - EXPECT_EQ("a0:0", node.input(0)); + EXPECT_EQ("a0", node.input(0)); EXPECT_EQ("a0:1", node.input(1)); - } else if (node.name() == "x2") { + } else if (node.name() == "x2" && count++) { EXPECT_EQ("Mul", node.op()); EXPECT_EQ(2, node.input_size()); EXPECT_EQ("x", node.input(0)); EXPECT_EQ("x", node.input(1)); - } else if (node.name() == "y2") { + } else if (node.name() == "y2" && count++) { EXPECT_EQ("Mul", node.op()); EXPECT_EQ(3, node.input_size()); EXPECT_EQ("y", node.input(0)); EXPECT_EQ("y", node.input(1)); EXPECT_EQ("^a1", node.input(2)); - } else if (node.name() == "o") { + } else if (node.name() == "o" && count++) { EXPECT_EQ("Add", node.op()); EXPECT_EQ(2, node.input_size()); - EXPECT_EQ("x2:0", node.input(0)); - EXPECT_EQ("y2:0", node.input(1)); + EXPECT_EQ("x2", node.input(0)); + EXPECT_EQ("y2", node.input(1)); } } + EXPECT_EQ(7, count); } TEST_F(FunctionsTest, FromFunctionDefWithOutputMappings) { @@ -245,28 +348,31 @@ TEST_F(FunctionsTest, FromFunctionDefWithOutputMappings) { {{"out", "Exp:y:0"}}); std::unordered_map func_attr; - FunctionDefLibrary library; - std::unique_ptr item = - GrapplerItemFromFunctionDef(func, func_attr, library); + FunctionLibraryDefinition library(OpRegistry::Global(), FunctionDefLibrary()); + + GrapplerFunctionItem item; + TF_EXPECT_OK(MakeGrapplerFunctionItem(func, func_attr, library, &item)); - EXPECT_EQ(1, item->fetch.size()); - EXPECT_EQ("Exp:0", item->fetch[0]); + EXPECT_EQ(1, item.output_size()); + EXPECT_EQ("Exp", item.output(0).output_tensors[0]); - for (const NodeDef &node : item->graph.node()) { - if (node.name() == "in") { + int count = 0; + for (const NodeDef &node : item.function_body().node()) { + if (node.name() == "in" && count++) { EXPECT_EQ("Placeholder", node.op()); EXPECT_EQ(DT_FLOAT, node.attr().at("T").type()); EXPECT_EQ(0, node.input_size()); - } else if (node.name() == "Linear_func") { + } else if (node.name() == "Linear_func" && count++) { EXPECT_EQ("Identity", node.op()); EXPECT_EQ(1, node.input_size()); EXPECT_EQ("in", node.input(0)); - } else if (node.name() == "Exp") { + } else if (node.name() == "Exp" && count++) { EXPECT_EQ("Exp", node.op()); EXPECT_EQ(1, node.input_size()); - EXPECT_EQ("Linear_func:0", node.input(0)); + EXPECT_EQ("Linear_func", node.input(0)); } } + EXPECT_EQ(3, count); } TEST_F(FunctionsTest, FromFunctionDefWithInputForwarding) { @@ -285,20 +391,25 @@ TEST_F(FunctionsTest, FromFunctionDefWithInputForwarding) { {{"out0", "in0"}}); std::unordered_map func_attr; - FunctionDefLibrary library; - std::unique_ptr item = - GrapplerItemFromFunctionDef(func, func_attr, library); + FunctionLibraryDefinition library(OpRegistry::Global(), FunctionDefLibrary()); - EXPECT_EQ(3, item->fetch.size()); - EXPECT_EQ("in0", item->fetch[0]); - EXPECT_EQ("arg2", item->fetch[1]); - EXPECT_EQ("arg3", item->fetch[2]); + GrapplerFunctionItem item; + TF_EXPECT_OK(MakeGrapplerFunctionItem(func, func_attr, library, &item)); - EXPECT_EQ(5, item->graph.node_size()); - for (const NodeDef &node : item->graph.node()) { + EXPECT_EQ("ForwardInputs", item.function_name()); + EXPECT_EQ(5, item.function_body().node_size()); + + EXPECT_EQ(3, item.output_size()); + EXPECT_EQ("in0", item.output(0).output_tensors[0]); + EXPECT_EQ("arg2", item.output(1).output_tensors[0]); + EXPECT_EQ("arg3", item.output(2).output_tensors[0]); + + int count = 0; + for (const NodeDef &node : item.function_body().node()) { EXPECT_TRUE(node.name() == "in0" || node.name() == "in1" || node.name() == "arg2" || node.name() == "arg3" || node.name() == "arg4"); + count++; EXPECT_EQ("Placeholder", node.op()); if (node.name() == "arg3") { EXPECT_EQ(DT_INT32, node.attr().at("T").type()); @@ -306,6 +417,7 @@ TEST_F(FunctionsTest, FromFunctionDefWithInputForwarding) { EXPECT_EQ(DT_FLOAT, node.attr().at("T").type()); } } + EXPECT_EQ(5, count); } TEST_F(FunctionsTest, FromFunctionDefWithoutInput) { @@ -325,24 +437,23 @@ TEST_F(FunctionsTest, FromFunctionDefWithoutInput) { std::unordered_map func_attr; func_attr["T"].set_type(DT_FLOAT); - FunctionDefLibrary library; - std::unique_ptr item = - GrapplerItemFromFunctionDef(func, func_attr, library); + FunctionLibraryDefinition library(OpRegistry::Global(), FunctionDefLibrary()); - EXPECT_EQ(0, item->feed.size()); - EXPECT_EQ(1, item->fetch.size()); - EXPECT_EQ("o:0", item->fetch[0]); + GrapplerFunctionItem item; + TF_EXPECT_OK(MakeGrapplerFunctionItem(func, func_attr, library, &item)); - EXPECT_EQ(2, item->graph.node_size()); - const NodeDef &two = item->graph.node(0); + EXPECT_EQ(0, item.input_size()); + EXPECT_EQ(1, item.output_size()); + EXPECT_EQ("o", item.output(0).output_tensors[0]); + + EXPECT_EQ(2, item.function_body().node_size()); + const NodeDef &two = item.function_body().node(0); EXPECT_EQ("two", two.name()); EXPECT_EQ(0, two.input_size()); - const NodeDef &cast = item->graph.node(1); + const NodeDef &cast = item.function_body().node(1); EXPECT_EQ("o", cast.name()); EXPECT_EQ(1, cast.input_size()); - EXPECT_EQ("two:0", cast.input(0)); - - std::cout << item->graph.DebugString() << std::endl; + EXPECT_EQ("two", cast.input(0)); } } // namespace -- GitLab From 08a12ca6016c34d9476d2e93bd0f2dc9ae60abc5 Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Wed, 11 Apr 2018 09:50:40 -0700 Subject: [PATCH 163/791] Add a clear error message for when a doc does not have a title. PiperOrigin-RevId: 192463583 --- tensorflow/tools/docs/generate_lib.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tensorflow/tools/docs/generate_lib.py b/tensorflow/tools/docs/generate_lib.py index 34dd419f15..9cc261d7dd 100644 --- a/tensorflow/tools/docs/generate_lib.py +++ b/tensorflow/tools/docs/generate_lib.py @@ -308,6 +308,10 @@ def build_doc_index(src_dir): continue title_parser = _GetMarkdownTitle() title_parser.process(os.path.join(dirpath, base_name)) + if title_parser.title is None: + msg = ('`{}` has no markdown title (# title)'.format( + os.path.join(dirpath, base_name))) + raise ValueError(msg) key_parts = os.path.join(suffix, base_name[:-3]).split('/') if key_parts[-1] == 'index': key_parts = key_parts[:-1] -- GitLab From 8f753859dd50a4c8d25b99a7b57c61e0e5c20578 Mon Sep 17 00:00:00 2001 From: Jacques Pienaar Date: Wed, 11 Apr 2018 09:53:21 -0700 Subject: [PATCH 164/791] Add gradient in cond test to match CallGradInLoop. PiperOrigin-RevId: 192463997 --- .../kernel_tests/control_flow_ops_py_test.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py index 75f8644f69..e27eb00818 100644 --- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py +++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py @@ -664,6 +664,23 @@ class ControlFlowTest(test.TestCase): self.assertAllEqual(42.0, grad.eval(feed_dict={c: 1})) self.assertAllEqual(3.0, grad.eval(feed_dict={c: 3})) + def testCondGrad_3(self): + with self.test_session(): + c = array_ops.placeholder(dtypes.int32, shape=[]) + ox = constant_op.constant(10.0) + pred = math_ops.less(c, 2) + + def fn1(x): + m = x * x + return gradients_impl.gradients(m, [ox])[0] + + fn2 = lambda: math_ops.multiply(ox, 3.0) + y = math_ops.multiply(7.0, ox) + r = control_flow_ops.cond(pred, lambda: fn1(y), fn2) + + self.assertAllEqual(980.0, r.eval(feed_dict={c: 1})) + self.assertAllEqual(30.0, r.eval(feed_dict={c: 3})) + def testNestedCond_Simple(self): with self.test_session(): x = constant_op.constant(0., name="X") -- GitLab From ae9542a8582d2e95229265d324f1b83a6e1d4a37 Mon Sep 17 00:00:00 2001 From: Billy Lamberta Date: Wed, 11 Apr 2018 10:57:30 -0700 Subject: [PATCH 165/791] Docs: Clarify using_tpu.md --- tensorflow/docs_src/programmers_guide/using_tpu.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/docs_src/programmers_guide/using_tpu.md b/tensorflow/docs_src/programmers_guide/using_tpu.md index cb0d86fc4c..5e3e49d434 100644 --- a/tensorflow/docs_src/programmers_guide/using_tpu.md +++ b/tensorflow/docs_src/programmers_guide/using_tpu.md @@ -280,8 +280,8 @@ Where `params['batch-size']` will contain the batch size. ### Static shapes and batch size The input pipeline generated by your `input_fn` is run on CPU. So it is mostly -free strict static shape requirements imposed by the XLA/TPU environment. The -one requirement is that the batches of data fed from your input pipeline to +free from the strict static shape requirements imposed by the XLA/TPU environment. +The one requirement is that the batches of data fed from your input pipeline to the TPU have a static shape, as determined by the standard TensorFlow shape inference algorithm. Intermediate tensors are free to have a dynamic shapes. If shape inference has failed, but the shape is known it is possible to -- GitLab From 5757d091a5c915b5ca99da7bc44feebdb374c569 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 11:02:48 -0700 Subject: [PATCH 166/791] Use tf.train.get_or_create_global_step() instead of deprecated variables.get_or_create_global_step(). PiperOrigin-RevId: 192476077 --- tensorflow/contrib/training/python/training/evaluation.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tensorflow/contrib/training/python/training/evaluation.py b/tensorflow/contrib/training/python/training/evaluation.py index 4bb53e8678..f7fd66d33f 100644 --- a/tensorflow/contrib/training/python/training/evaluation.py +++ b/tensorflow/contrib/training/python/training/evaluation.py @@ -138,7 +138,6 @@ from __future__ import print_function import time -from tensorflow.contrib.framework.python.ops import variables from tensorflow.python.ops import state_ops from tensorflow.python.platform import tf_logging as logging from tensorflow.python.summary import summary @@ -298,7 +297,7 @@ class SummaryAtEndHook(session_run_hook.SessionRunHook): def begin(self): if self._replace_summary_op: self._summary_op = summary.merge_all() - self._global_step = variables.get_or_create_global_step() + self._global_step = training_util.get_or_create_global_step() def after_create_session(self, session, coord): if self._summary_writer is None and self._log_dir: -- GitLab From 48b2bdc72541139bff7bf9a044eafee8234fe41f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 11:21:48 -0700 Subject: [PATCH 167/791] Fix uninitialized value. PiperOrigin-RevId: 192479630 --- tensorflow/compiler/xla/service/hlo_instruction.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h index a6cb19f331..9a9de07883 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.h +++ b/tensorflow/compiler/xla/service/hlo_instruction.h @@ -1446,7 +1446,7 @@ class HloInstruction { string channel_name_; // Estimate of the duration of a host computation in nanoseconds. - int64 cost_estimate_ns_; + int64 cost_estimate_ns_ = 0; // Computations called by this instruction. std::vector called_computations_; -- GitLab From 2ea5c1e867f029c3cda9ac099542858cd737d8e3 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 11:26:25 -0700 Subject: [PATCH 168/791] Disable prelu tests for real now. PiperOrigin-RevId: 192480452 --- tensorflow/contrib/lite/testing/generated_examples_zip_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc index 7426ab56af..84ae1d58fe 100644 --- a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc +++ b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc @@ -267,7 +267,7 @@ INSTANTIATE_TESTS(mul) INSTANTIATE_TESTS(pad) INSTANTIATE_TESTS(relu) INSTANTIATE_TESTS(relu1) -INSTANTIATE_TESTS(prelu) +// INSTANTIATE_TESTS(prelu) INSTANTIATE_TESTS(relu6) INSTANTIATE_TESTS(reshape) INSTANTIATE_TESTS(resize_bilinear) -- GitLab From 8b17a17ed5d92fb52922c1c4726180db0c220f8e Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 11:33:38 -0700 Subject: [PATCH 169/791] Script to create custom_ops inside a TensorFlow graphdef. PiperOrigin-RevId: 192481690 --- tensorflow/contrib/lite/python/BUILD | 13 ++ .../contrib/lite/python/create_custom_op.py | 111 ++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 tensorflow/contrib/lite/python/create_custom_op.py diff --git a/tensorflow/contrib/lite/python/BUILD b/tensorflow/contrib/lite/python/BUILD index 6fafaf0727..926896d609 100644 --- a/tensorflow/contrib/lite/python/BUILD +++ b/tensorflow/contrib/lite/python/BUILD @@ -97,6 +97,19 @@ py_binary( ], ) +py_binary( + name = "create_custom_op", + srcs = ["create_custom_op.py"], + srcs_version = "PY2AND3", + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/contrib/framework:framework_py", + "//tensorflow/core:protos_all_py", + "//tensorflow/python:platform", + "@absl_py//absl/flags", + ], +) + py_test( name = "convert_saved_model_test", srcs = ["convert_saved_model_test.py"], diff --git a/tensorflow/contrib/lite/python/create_custom_op.py b/tensorflow/contrib/lite/python/create_custom_op.py new file mode 100644 index 0000000000..830f95358c --- /dev/null +++ b/tensorflow/contrib/lite/python/create_custom_op.py @@ -0,0 +1,111 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +r"""Replaces a subgraph of a TensorFlow GraphDef with a single node. + +In conjunction with TOCO's --allow_custom_op this script allows selected +portions of a TensorFlow GraphDef to be executed by custom code. + +Example: + +bazel run tensorflow/contrib/lite/python:create_custom_op -- \ + --input_graph=/tmp/input.pb \ + --output_graph=/tmp/output.pb \ + --inputs=concat,concat_1 \ + --outputs=detection_classes \ + --op_definition='op:"PostProcessing" attr{key:"num" value:{i:10}}' + +The above will identify a subgraph starting at nodes 'concat' and 'concat_1', +and ending at 'detection_classes'. All nodes in between will be removed and +replaced by a new op called 'PostProcessing'. + +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import uuid as _uuid +from absl import app +from absl import flags +from google.protobuf import text_format +from tensorflow.contrib.framework.python.framework.graph_util import fuse_op +from tensorflow.core.framework import graph_pb2 +from tensorflow.core.framework import node_def_pb2 +from tensorflow.core.framework import types_pb2 +from tensorflow.python.platform import gfile + +FLAGS = flags.FLAGS + +flags.DEFINE_string("input_graph", "", "Binary graphdef to load.") +flags.DEFINE_string("output_graph", "", "Resulting binary graphdef.") + +flags.DEFINE_string("inputs", "", + "Comma-separated list of inputs to the subgraph.") +flags.DEFINE_string("outputs", "", + "Comma-separated list of outputs of the subgraph.") +flags.DEFINE_string("op_definition", "", + "A text NodeDef defining the contents of the custom op.") + + +def _read_graph_def(filename): + if not gfile.Exists(filename): + raise ValueError("Input graph file '" + filename + "' does not exist!") + + graph_def = graph_pb2.GraphDef() + with gfile.FastGFile(filename, "rb") as f: + graph_def.ParseFromString(f.read()) + return graph_def + + +def _write_graph_def(graph_def, filename): + if not filename: + raise ValueError("Output graph file not specified") + + with gfile.Open(filename, "wb") as f: + f.write(graph_def.SerializeToString()) + + +def _collapse_subgraph(graph_def, inputs, outputs, op_definition): + """Substitute a custom op for the subgraph delimited by inputs and outputs.""" + name = _uuid.uuid1().hex + # We need a default type, but it can be changed using 'op_definition'. + default_type = types_pb2.DT_FLOAT + new_graph = fuse_op( + graph_def=graph_def, + input_nodes=inputs, + output_nodes=outputs, + output_dtypes=[default_type for _ in outputs], + output_quantized=False, + op_name=name, + op_type="CustomTfLiteOp") + node_def = node_def_pb2.NodeDef() + text_format.Parse(op_definition, node_def) + for node in new_graph.node: + if node.name == name: + node.MergeFrom(node_def) + return new_graph + + +def main(argv): + del argv # unused + graph = _read_graph_def(filename=flags.FLAGS.input_graph) + graph = _collapse_subgraph( + graph_def=graph, + inputs=flags.FLAGS.inputs.split(","), + outputs=flags.FLAGS.outputs.split(","), + op_definition=flags.FLAGS.op_definition) + _write_graph_def(graph_def=graph, filename=flags.FLAGS.output_graph) + + +if __name__ == "__main__": + app.run(main) -- GitLab From abc26c182ce2e1f010c53ca4f384759587740578 Mon Sep 17 00:00:00 2001 From: Adria Puigdomenech Date: Wed, 11 Apr 2018 11:36:56 -0700 Subject: [PATCH 170/791] Update docs for softmax_cross_entropy_with_logits. PiperOrigin-RevId: 192482242 --- tensorflow/python/ops/nn_ops.py | 38 ++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py index 07ca32953f..bb454b3c3a 100644 --- a/tensorflow/python/ops/nn_ops.py +++ b/tensorflow/python/ops/nn_ops.py @@ -1803,8 +1803,11 @@ def softmax_cross_entropy_with_logits_v2( on `logits` internally for efficiency. Do not call this op with the output of `softmax`, as it will produce incorrect results. - `logits` and `labels` must have the same shape, e.g. - `[batch_size, num_classes]` and the same dtype (either `float16`, `float32`, + A common use case is to have logits and labels of shape + `[batch_size, num_classes]`, but higher dimensions are supported, with + the `dim` argument specifying the class dimension. + + `logits` and `labels` must have the same dtype (either `float16`, `float32`, or `float64`). Backpropagation will happen into both `logits` and `labels`. To disallow @@ -1816,14 +1819,17 @@ def softmax_cross_entropy_with_logits_v2( Args: _sentinel: Used to prevent positional parameters. Internal, do not use. - labels: Each row `labels[i]` must be a valid probability distribution. + labels: Each vector along the class dimension should hold a valid + probability distribution e.g. for the case in which labels are of shape + `[batch_size, num_classes]`, each row of `labels[i]` must be a valid + probability distribution. logits: Unscaled log probabilities. dim: The class dimension. Defaulted to -1 which is the last dimension. name: A name for the operation (optional). Returns: - A 1-D `Tensor` of length `batch_size` of the same type as `logits` with the - softmax cross entropy loss. + A `Tensor` of the same shape as `labels` and of the same type as `logits` + with the softmax cross entropy loss. """ _ensure_xent_args("softmax_cross_entropy_with_logits", _sentinel, labels, logits) @@ -1926,9 +1932,9 @@ def softmax_cross_entropy_with_logits( on `logits` internally for efficiency. Do not call this op with the output of `softmax`, as it will produce incorrect results. - `logits` and `labels` must have the same shape, e.g. - `[batch_size, num_classes]` and the same dtype (either `float16`, `float32`, - or `float64`). + A common use case is to have logits and labels of shape + `[batch_size, num_classes]`, but higher dimensions are supported, with + the `dim` argument specifying the class dimension. Backpropagation will happen only into `logits`. To calculate a cross entropy loss that allows backpropagation into both `logits` and `labels`, see @@ -1939,14 +1945,17 @@ def softmax_cross_entropy_with_logits( Args: _sentinel: Used to prevent positional parameters. Internal, do not use. - labels: Each row `labels[i]` must be a valid probability distribution. + labels: Each vector along the class dimension should hold a valid + probability distribution e.g. for the case in which labels are of shape + `[batch_size, num_classes]`, each row of `labels[i]` must be a valid + probability distribution. logits: Unscaled log probabilities. dim: The class dimension. Defaulted to -1 which is the last dimension. name: A name for the operation (optional). Returns: - A 1-D `Tensor` of length `batch_size` of the same type as `logits` with the - softmax cross entropy loss. + A `Tensor` of the same shape as `labels` and of the same type as `logits` + with the softmax cross entropy loss. """ _ensure_xent_args("softmax_cross_entropy_with_logits", _sentinel, labels, logits) @@ -1983,8 +1992,11 @@ def sparse_softmax_cross_entropy_with_logits( on `logits` internally for efficiency. Do not call this op with the output of `softmax`, as it will produce incorrect results. - A common use case is to have logits of shape `[batch_size, num_classes]` and - labels of shape `[batch_size]`. But higher dimensions are supported. + A common use case is to have logits and labels of shape + `[batch_size, num_classes]`, but higher dimensions are supported, in which + case the `dim`-th dimension is assumed to be of size `num_classes`. + `logits` and `labels` must have the same dtype (either `float16`, `float32`, + or `float64`). **Note that to avoid confusion, it is required to pass only named arguments to this function.** -- GitLab From 5eccb5afe6f8ecda6a0aa9ecdd2d4a6636996509 Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Wed, 11 Apr 2018 11:52:52 -0700 Subject: [PATCH 171/791] Increase size of tensorflow/contrib/data/python/kernel_tests:batch_dataset_op_test to "medium". PiperOrigin-RevId: 192484895 --- tensorflow/contrib/data/python/kernel_tests/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD index c8699e0d5a..5d6dbdcbdf 100644 --- a/tensorflow/contrib/data/python/kernel_tests/BUILD +++ b/tensorflow/contrib/data/python/kernel_tests/BUILD @@ -8,7 +8,7 @@ load("//tensorflow:tensorflow.bzl", "py_test", "tf_py_test") py_test( name = "batch_dataset_op_test", - size = "small", + size = "medium", srcs = ["batch_dataset_op_test.py"], srcs_version = "PY2AND3", tags = ["no_pip"], -- GitLab From 5a2129e863d7983a34a86865c6fb3f1d382ef4a5 Mon Sep 17 00:00:00 2001 From: Billy Lamberta Date: Wed, 11 Apr 2018 12:05:39 -0700 Subject: [PATCH 172/791] Tidy up doc for rebuild project --- tensorflow/docs_src/mobile/android_build.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/docs_src/mobile/android_build.md b/tensorflow/docs_src/mobile/android_build.md index 0cd0a98be4..c35530061d 100644 --- a/tensorflow/docs_src/mobile/android_build.md +++ b/tensorflow/docs_src/mobile/android_build.md @@ -51,8 +51,8 @@ If you haven't already, do the following two things: // set to 'bazel', 'cmake', 'makefile', 'none' def nativeBuildSystem = 'none' -4. Running "Build -> Rebuild Project" from Android Studio menu and click the - Run button (the green arrow) or use **Run -> Run 'android'** from the top menu. +4. Click the *Run* button (the green arrow) or select *Run > Run 'android'* from the + top menu. You may need to rebuild the project using *Build > Rebuild Project*. If it asks you to use Instant Run, click **Proceed Without Instant Run**. -- GitLab From cc1525125c497772f25ee4851c7b832048cd5bd8 Mon Sep 17 00:00:00 2001 From: Austin Anderson Date: Wed, 11 Apr 2018 12:32:08 -0700 Subject: [PATCH 173/791] Internal TF Lite test changes PiperOrigin-RevId: 192491201 --- tensorflow/contrib/lite/kernels/BUILD | 205 ++++++-------------------- 1 file changed, 41 insertions(+), 164 deletions(-) diff --git a/tensorflow/contrib/lite/kernels/BUILD b/tensorflow/contrib/lite/kernels/BUILD index f07eca0ba9..914893cd90 100644 --- a/tensorflow/contrib/lite/kernels/BUILD +++ b/tensorflow/contrib/lite/kernels/BUILD @@ -12,10 +12,7 @@ tf_cc_test( name = "optional_tensor_test", size = "small", srcs = ["optional_tensor_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -108,10 +105,7 @@ tf_cc_test( name = "kernel_util_test", size = "small", srcs = ["kernel_util_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":kernel_util", "//tensorflow/contrib/lite/testing:util", @@ -243,10 +237,7 @@ tf_cc_test( name = "activations_test", size = "small", srcs = ["activations_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -259,10 +250,7 @@ tf_cc_test( name = "add_test", size = "small", srcs = ["add_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -291,10 +279,7 @@ tf_cc_test( name = "div_test", size = "small", srcs = ["div_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -307,10 +292,7 @@ tf_cc_test( name = "sub_test", size = "small", srcs = ["sub_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -323,10 +305,7 @@ tf_cc_test( name = "transpose_test", size = "small", srcs = ["transpose_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -341,10 +320,7 @@ tf_cc_test( name = "space_to_batch_nd_test", size = "small", srcs = ["space_to_batch_nd_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -357,10 +333,7 @@ tf_cc_test( name = "batch_to_space_nd_test", size = "small", srcs = ["batch_to_space_nd_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -385,10 +358,7 @@ tf_cc_test( name = "concatenation_test", size = "small", srcs = ["concatenation_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -401,10 +371,7 @@ tf_cc_test( name = "conv_test", size = "small", srcs = ["conv_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -418,10 +385,7 @@ tf_cc_test( name = "depthwise_conv_test", size = "small", srcs = ["depthwise_conv_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -447,10 +411,7 @@ tf_cc_test( name = "basic_rnn_test", size = "small", srcs = ["basic_rnn_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -463,10 +424,7 @@ tf_cc_test( name = "bidirectional_sequence_lstm_test", size = "small", srcs = ["bidirectional_sequence_lstm_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -479,10 +437,7 @@ tf_cc_test( name = "unidirectional_sequence_lstm_test", size = "small", srcs = ["unidirectional_sequence_lstm_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -510,10 +465,7 @@ tf_cc_test( name = "unidirectional_sequence_rnn_test", size = "small", srcs = ["unidirectional_sequence_rnn_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -526,10 +478,7 @@ tf_cc_test( name = "l2norm_test", size = "small", srcs = ["l2norm_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -542,10 +491,7 @@ tf_cc_test( name = "exp_test", size = "small", srcs = ["exp_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -570,10 +516,7 @@ tf_cc_test( name = "mean_test", size = "small", srcs = ["mean_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -586,10 +529,7 @@ tf_cc_test( name = "mul_test", size = "small", srcs = ["mul_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -602,10 +542,7 @@ tf_cc_test( name = "pad_test", size = "small", srcs = ["pad_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -618,10 +555,7 @@ tf_cc_test( name = "reshape_test", size = "small", srcs = ["reshape_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -634,10 +568,7 @@ tf_cc_test( name = "gather_test", size = "small", srcs = ["gather_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:builtin_op_data", @@ -651,10 +582,7 @@ tf_cc_test( name = "topk_v2_test", size = "small", srcs = ["topk_v2_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:builtin_op_data", @@ -668,10 +596,7 @@ tf_cc_test( name = "resize_bilinear_test", size = "small", srcs = ["resize_bilinear_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -684,10 +609,7 @@ tf_cc_test( name = "svdf_test", size = "small", srcs = ["svdf_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -700,10 +622,7 @@ tf_cc_test( name = "embedding_lookup_test", size = "small", srcs = ["embedding_lookup_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -716,10 +635,7 @@ tf_cc_test( name = "embedding_lookup_sparse_test", size = "small", srcs = ["embedding_lookup_sparse_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -732,10 +648,7 @@ tf_cc_test( name = "fully_connected_test", size = "small", srcs = ["fully_connected_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -749,10 +662,7 @@ tf_cc_test( name = "local_response_norm_test", size = "small", srcs = ["local_response_norm_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -765,10 +675,7 @@ tf_cc_test( name = "pooling_test", size = "small", srcs = ["pooling_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -781,10 +688,7 @@ tf_cc_test( name = "softmax_test", size = "small", srcs = ["softmax_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -798,10 +702,7 @@ tf_cc_test( name = "log_softmax_test", size = "small", srcs = ["log_softmax_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -815,10 +716,7 @@ tf_cc_test( name = "lsh_projection_test", size = "small", srcs = ["lsh_projection_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -831,10 +729,7 @@ tf_cc_test( name = "hashtable_lookup_test", size = "small", srcs = ["hashtable_lookup_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -848,10 +743,7 @@ tf_cc_test( name = "lstm_test", size = "small", srcs = ["lstm_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -864,10 +756,7 @@ tf_cc_test( name = "skip_gram_test", size = "small", srcs = ["skip_gram_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -881,10 +770,7 @@ tf_cc_test( name = "space_to_depth_test", size = "small", srcs = ["space_to_depth_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -897,10 +783,7 @@ tf_cc_test( name = "split_test", size = "small", srcs = ["split_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -913,10 +796,7 @@ tf_cc_test( name = "squeeze_test", size = "small", srcs = ["squeeze_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -929,10 +809,7 @@ tf_cc_test( name = "strided_slice_test", size = "small", srcs = ["strided_slice_test.cc"], - tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", - ], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", -- GitLab From d983832d8fe01ab85b761fa1effd2d3b7a8ee794 Mon Sep 17 00:00:00 2001 From: Rohan Jain Date: Wed, 11 Apr 2018 12:33:04 -0700 Subject: [PATCH 174/791] Adding hp5y back. PiperOrigin-RevId: 192491335 --- .../python/learn/learn_io/data_feeder_test.py | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/tensorflow/contrib/learn/python/learn/learn_io/data_feeder_test.py b/tensorflow/contrib/learn/python/learn/learn_io/data_feeder_test.py index 82848be7df..1f439965da 100644 --- a/tensorflow/contrib/learn/python/learn/learn_io/data_feeder_test.py +++ b/tensorflow/contrib/learn/python/learn/learn_io/data_feeder_test.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import os.path import numpy as np import six from six.moves import xrange # pylint: disable=redefined-builtin @@ -26,6 +27,7 @@ from six.moves import xrange # pylint: disable=redefined-builtin from tensorflow.contrib.learn.python.learn.learn_io import * from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.lib.io import file_io from tensorflow.python.platform import test # pylint: enable=wildcard-import @@ -35,6 +37,13 @@ class DataFeederTest(test.TestCase): # pylint: disable=undefined-variable """Tests for `DataFeeder`.""" + def setUp(self): + self._base_dir = os.path.join(self.get_temp_dir(), 'base_dir') + file_io.create_dir(self._base_dir) + + def tearDown(self): + file_io.delete_recursively(self._base_dir) + def _wrap_dict(self, data, prepend=''): return {prepend + '1': data, prepend + '2': data} @@ -45,14 +54,14 @@ class DataFeederTest(test.TestCase): def _assert_dtype(self, expected_np_dtype, expected_tf_dtype, input_data): feeder = data_feeder.DataFeeder(input_data, None, n_classes=0, batch_size=1) if isinstance(input_data, dict): - for k, v in list(feeder.input_dtype.items()): + for v in list(feeder.input_dtype.values()): self.assertEqual(expected_np_dtype, v) else: self.assertEqual(expected_np_dtype, feeder.input_dtype) with ops.Graph().as_default() as g, self.test_session(g): inp, _ = feeder.input_builder() if isinstance(inp, dict): - for k, v in list(inp.items()): + for v in list(inp.values()): self.assertEqual(expected_tf_dtype, v.dtype) else: self.assertEqual(expected_tf_dtype, inp.dtype) @@ -301,7 +310,10 @@ class DataFeederTest(test.TestCase): [0.60000002, 0.2]]) self.assertAllClose(feed_dict[out.name], [[0., 0., 1.], [0., 1., 0.]]) - def test_hdf5_data_feeder(self): + # TODO(rohanj): Fix this test by fixing data_feeder. Currently, h5py doesn't + # support permutation based indexing lookups (More documentation at + # http://docs.h5py.org/en/latest/high/dataset.html#fancy-indexing) + def DISABLED_test_hdf5_data_feeder(self): def func(df): inp, out = df.input_builder() @@ -314,11 +326,12 @@ class DataFeederTest(test.TestCase): import h5py # pylint: disable=g-import-not-at-top x = np.matrix([[1, 2], [3, 4]]) y = np.array([1, 2]) - h5f = h5py.File('test_hdf5.h5', 'w') + file_path = os.path.join(self._base_dir, 'test_hdf5.h5') + h5f = h5py.File(file_path, 'w') h5f.create_dataset('x', data=x) h5f.create_dataset('y', data=y) h5f.close() - h5f = h5py.File('test_hdf5.h5', 'r') + h5f = h5py.File(file_path, 'r') x = h5f['x'] y = h5f['y'] func(data_feeder.DataFeeder(x, y, n_classes=0, batch_size=3)) -- GitLab From c9df9896422a5509b55f92f66c1310bb48249afb Mon Sep 17 00:00:00 2001 From: Rajendra arora Date: Thu, 12 Apr 2018 01:19:31 +0530 Subject: [PATCH 175/791] Updating tensorboard link in Readme.md (#18161) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 29418dc2e9..e1a50c87e2 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ data flow graphs. The graph nodes represent mathematical operations, while the graph edges represent the multidimensional data arrays (tensors) that flow between them. This flexible architecture enables you to deploy computation to one or more CPUs or GPUs in a desktop, server, or mobile device without rewriting -code. TensorFlow also includes TensorBoard, a data visualization toolkit. +code. TensorFlow also includes [TensorBoard](https://www.tensorflow.org/programmers_guide/summaries_and_tensorboard), a data visualization toolkit. TensorFlow was originally developed by researchers and engineers working on the Google Brain team within Google's Machine Intelligence Research -- GitLab From c5d59c6a3cd8c15ee2f93608e412a1e9335d3465 Mon Sep 17 00:00:00 2001 From: Anna R Date: Wed, 11 Apr 2018 13:22:53 -0700 Subject: [PATCH 176/791] Internal change. PiperOrigin-RevId: 192498471 --- tensorflow/stream_executor/cuda/cudnn_version_test.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/stream_executor/cuda/cudnn_version_test.cc b/tensorflow/stream_executor/cuda/cudnn_version_test.cc index 230adafeb1..42b3dc8cc6 100644 --- a/tensorflow/stream_executor/cuda/cudnn_version_test.cc +++ b/tensorflow/stream_executor/cuda/cudnn_version_test.cc @@ -15,7 +15,6 @@ limitations under the License. #include "tensorflow/stream_executor/cuda/cudnn_version.h" -#include "testing/base/public/gunit.h" #include "tensorflow/core/platform/test.h" namespace perftools { -- GitLab From 371d5132a5558ef06a0951f3197bde63565a1805 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 13:29:12 -0700 Subject: [PATCH 177/791] DepthwiseConv Optimizations PiperOrigin-RevId: 192499401 --- .../internal/optimized/depthwiseconv_uint8.h | 18 +- .../depthwiseconv_uint8_3x3_filter.h | 5015 +++++++++++++++-- 2 files changed, 4434 insertions(+), 599 deletions(-) diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8.h b/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8.h index 0f78e0f728..dd6932ffe7 100644 --- a/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8.h +++ b/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8.h @@ -1696,15 +1696,15 @@ inline void DepthwiseConv(const uint8* input_data, const Dims<4>& input_dims, #ifdef __aarch64__ // Call kernel optimized for depthwise convolutions using 3x3 filters if // parameters are supported. - if (Fast3by3FilterKernelSupported(input_dims, filter_dims, stride_width, - stride_height, pad_width, pad_height, - depth_multiplier, output_dims)) { - DepthwiseConv3by3FilterDepth16( - input_data, input_dims, input_offset, filter_data, filter_dims, - filter_offset, bias_data, bias_dims, stride_width, stride_height, - pad_width, pad_height, depth_multiplier, output_offset, - output_multiplier, output_shift, output_activation_min, - output_activation_max, output_data, output_dims); + if (Fast3x3FilterKernelSupported(input_dims, filter_dims, stride_width, + stride_height, pad_width, pad_height, + depth_multiplier, output_dims)) { + DepthwiseConv3x3Filter(input_data, input_dims, input_offset, filter_data, + filter_dims, filter_offset, bias_data, bias_dims, + stride_width, stride_height, pad_width, pad_height, + depth_multiplier, output_offset, output_multiplier, + output_shift, output_activation_min, + output_activation_max, output_data, output_dims); return; } #endif diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h b/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h index a349892076..cdcb166b2f 100644 --- a/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h +++ b/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h @@ -1,4 +1,4 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -40,412 +40,4380 @@ inline void preload_l1_keep(const uint8* ptr) { // NEON intrinsics vector data types. // See: https://bugs.llvm.org/show_bug.cgi?id=34945 -struct Int32x16 { - int32x4_t v0, v1, v2, v3; +struct Int32x8 { + int32x4_t low, high; }; -struct Int16x16 { - int16x8_t low, high; +struct Filter3x3x8 { + int16x8_t f0, f1, f2, f3, f4, f5, f6, f7, f8; }; -struct Int16x16x3 { - Int16x16 v0, v1, v2; +// Loads 3x3 filter of depth 8 and adds filter offsets. +inline Filter3x3x8 Load3x3Filter(const uint8* filter_ptr, int32 filter_offset, + int output_depth) { + Filter3x3x8 filter; + + uint8x8_t temp_u8_0, temp_u8_1, temp_u8_2, temp_u8_3, temp_u8_4, temp_u8_5, + temp_u8_6, temp_u8_7, temp_u8_8; + int16x8_t filter_offset_vec = vdupq_n_s16(filter_offset); + + temp_u8_0 = vld1_u8(filter_ptr + 0 * output_depth); + temp_u8_1 = vld1_u8(filter_ptr + 1 * output_depth); + temp_u8_2 = vld1_u8(filter_ptr + 2 * output_depth); + temp_u8_3 = vld1_u8(filter_ptr + 3 * output_depth); + temp_u8_4 = vld1_u8(filter_ptr + 4 * output_depth); + temp_u8_5 = vld1_u8(filter_ptr + 5 * output_depth); + temp_u8_6 = vld1_u8(filter_ptr + 6 * output_depth); + temp_u8_7 = vld1_u8(filter_ptr + 7 * output_depth); + temp_u8_8 = vld1_u8(filter_ptr + 8 * output_depth); + + filter.f0 = vreinterpretq_s16_u16(vmovl_u8(temp_u8_0)); + filter.f1 = vreinterpretq_s16_u16(vmovl_u8(temp_u8_1)); + filter.f2 = vreinterpretq_s16_u16(vmovl_u8(temp_u8_2)); + filter.f3 = vreinterpretq_s16_u16(vmovl_u8(temp_u8_3)); + filter.f4 = vreinterpretq_s16_u16(vmovl_u8(temp_u8_4)); + filter.f5 = vreinterpretq_s16_u16(vmovl_u8(temp_u8_5)); + filter.f6 = vreinterpretq_s16_u16(vmovl_u8(temp_u8_6)); + filter.f7 = vreinterpretq_s16_u16(vmovl_u8(temp_u8_7)); + filter.f8 = vreinterpretq_s16_u16(vmovl_u8(temp_u8_8)); + + filter.f0 = vaddq_s16(filter.f0, filter_offset_vec); + filter.f1 = vaddq_s16(filter.f1, filter_offset_vec); + filter.f2 = vaddq_s16(filter.f2, filter_offset_vec); + filter.f3 = vaddq_s16(filter.f3, filter_offset_vec); + filter.f4 = vaddq_s16(filter.f4, filter_offset_vec); + filter.f5 = vaddq_s16(filter.f5, filter_offset_vec); + filter.f6 = vaddq_s16(filter.f6, filter_offset_vec); + filter.f7 = vaddq_s16(filter.f7, filter_offset_vec); + filter.f8 = vaddq_s16(filter.f8, filter_offset_vec); + + return filter; +} + +// Applies activation, offset and downquantize on a set of accumulator +// registers that correspond to a 2x2 output of depth 8. +// Stores results to output. +inline void DownquantizeAndStore2x2Output( + Int32x8 acc_0, Int32x8 acc_1, Int32x8 acc_2, Int32x8 acc_3, + int32 output_offset, int32 output_multiplier, int output_shift, + int32 output_activation_min, int32 output_activation_max, uint8* output_ptr, + int output_depth, int output_width) { + using gemmlowp::RoundingDivideByPOT; + const int32x4_t output_offset_vec = vdupq_n_s32(output_offset); + const int32x4_t output_activation_min_vec = + vdupq_n_s32(output_activation_min); + const int32x4_t output_activation_max_vec = + vdupq_n_s32(output_activation_max); + + // Fixed-point multiplication. + acc_0.low = vqrdmulhq_n_s32(acc_0.low, output_multiplier); + acc_0.high = vqrdmulhq_n_s32(acc_0.high, output_multiplier); + acc_1.low = vqrdmulhq_n_s32(acc_1.low, output_multiplier); + acc_1.high = vqrdmulhq_n_s32(acc_1.high, output_multiplier); + acc_2.low = vqrdmulhq_n_s32(acc_2.low, output_multiplier); + acc_2.high = vqrdmulhq_n_s32(acc_2.high, output_multiplier); + acc_3.low = vqrdmulhq_n_s32(acc_3.low, output_multiplier); + acc_3.high = vqrdmulhq_n_s32(acc_3.high, output_multiplier); + + acc_0.low = RoundingDivideByPOT(acc_0.low, output_shift); + acc_0.high = RoundingDivideByPOT(acc_0.high, output_shift); + acc_1.low = RoundingDivideByPOT(acc_1.low, output_shift); + acc_1.high = RoundingDivideByPOT(acc_1.high, output_shift); + acc_2.low = RoundingDivideByPOT(acc_2.low, output_shift); + acc_2.high = RoundingDivideByPOT(acc_2.high, output_shift); + acc_3.low = RoundingDivideByPOT(acc_3.low, output_shift); + acc_3.high = RoundingDivideByPOT(acc_3.high, output_shift); + + // Add the output offset. + acc_0.low = vaddq_s32(acc_0.low, output_offset_vec); + acc_0.high = vaddq_s32(acc_0.high, output_offset_vec); + acc_1.low = vaddq_s32(acc_1.low, output_offset_vec); + acc_1.high = vaddq_s32(acc_1.high, output_offset_vec); + acc_2.low = vaddq_s32(acc_2.low, output_offset_vec); + acc_2.high = vaddq_s32(acc_2.high, output_offset_vec); + acc_3.low = vaddq_s32(acc_3.low, output_offset_vec); + acc_3.high = vaddq_s32(acc_3.high, output_offset_vec); + + // Apply the activation function. + acc_0.low = vmaxq_s32(acc_0.low, output_activation_min_vec); + acc_0.high = vmaxq_s32(acc_0.high, output_activation_min_vec); + acc_1.low = vmaxq_s32(acc_1.low, output_activation_min_vec); + acc_1.high = vmaxq_s32(acc_1.high, output_activation_min_vec); + acc_2.low = vmaxq_s32(acc_2.low, output_activation_min_vec); + acc_2.high = vmaxq_s32(acc_2.high, output_activation_min_vec); + acc_3.low = vmaxq_s32(acc_3.low, output_activation_min_vec); + acc_3.high = vmaxq_s32(acc_3.high, output_activation_min_vec); + + acc_0.low = vminq_s32(acc_0.low, output_activation_max_vec); + acc_0.high = vminq_s32(acc_0.high, output_activation_max_vec); + acc_1.low = vminq_s32(acc_1.low, output_activation_max_vec); + acc_1.high = vminq_s32(acc_1.high, output_activation_max_vec); + acc_2.low = vminq_s32(acc_2.low, output_activation_max_vec); + acc_2.high = vminq_s32(acc_2.high, output_activation_max_vec); + acc_3.low = vminq_s32(acc_3.low, output_activation_max_vec); + acc_3.high = vminq_s32(acc_3.high, output_activation_max_vec); + + // Saturating cast to uint8 and store to destination. + int16x4_t acc_0_low_s16 = vqmovn_s32(acc_0.low); + int16x4_t acc_0_high_s16 = vqmovn_s32(acc_0.high); + int16x4_t acc_1_low_s16 = vqmovn_s32(acc_1.low); + int16x4_t acc_1_high_s16 = vqmovn_s32(acc_1.high); + int16x4_t acc_2_low_s16 = vqmovn_s32(acc_2.low); + int16x4_t acc_2_high_s16 = vqmovn_s32(acc_2.high); + int16x4_t acc_3_low_s16 = vqmovn_s32(acc_3.low); + int16x4_t acc_3_high_s16 = vqmovn_s32(acc_3.high); + + int16x8_t res_0_s16 = vcombine_s16(acc_0_low_s16, acc_0_high_s16); + int16x8_t res_1_s16 = vcombine_s16(acc_1_low_s16, acc_1_high_s16); + int16x8_t res_2_s16 = vcombine_s16(acc_2_low_s16, acc_2_high_s16); + int16x8_t res_3_s16 = vcombine_s16(acc_3_low_s16, acc_3_high_s16); + + uint8x8_t res_0_u8 = vqmovun_s16(res_0_s16); + uint8x8_t res_1_u8 = vqmovun_s16(res_1_s16); + uint8x8_t res_2_u8 = vqmovun_s16(res_2_s16); + uint8x8_t res_3_u8 = vqmovun_s16(res_3_s16); + + vst1_u8(output_ptr, res_0_u8); + vst1_u8(output_ptr + output_depth, res_1_u8); + vst1_u8(output_ptr + output_depth * output_width, res_2_u8); + vst1_u8(output_ptr + output_depth * output_width + output_depth, res_3_u8); +} + +inline void DownquantizeAndStore(Int32x8 acc, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, + uint8* output_ptr) { + using gemmlowp::RoundingDivideByPOT; + const int32x4_t output_offset_vec = vdupq_n_s32(output_offset); + const int32x4_t output_activation_min_vec = + vdupq_n_s32(output_activation_min); + const int32x4_t output_activation_max_vec = + vdupq_n_s32(output_activation_max); + + acc.low = vqrdmulhq_n_s32(acc.low, output_multiplier); + acc.high = vqrdmulhq_n_s32(acc.high, output_multiplier); + + acc.low = RoundingDivideByPOT(acc.low, output_shift); + acc.high = RoundingDivideByPOT(acc.high, output_shift); + + acc.low = vaddq_s32(acc.low, output_offset_vec); + acc.high = vaddq_s32(acc.high, output_offset_vec); + + acc.low = vmaxq_s32(acc.low, output_activation_min_vec); + acc.high = vmaxq_s32(acc.high, output_activation_min_vec); + + acc.low = vminq_s32(acc.low, output_activation_max_vec); + acc.high = vminq_s32(acc.high, output_activation_max_vec); + + int16x4_t acc_low_s16 = vqmovn_s32(acc.low); + int16x4_t acc_high_s16 = vqmovn_s32(acc.high); + + int16x8_t res_s16 = vcombine_s16(acc_low_s16, acc_high_s16); + uint8x8_t res_u8 = vqmovun_s16(res_s16); + vst1_u8(output_ptr, res_u8); +} + +inline void DownquantizeAndStore2Output( + Int32x8 acc_0, Int32x8 acc_1, int32 output_offset, int32 output_multiplier, + int output_shift, int32 output_activation_min, int32 output_activation_max, + uint8* output_ptr, int output_ptr_offset) { + { + using gemmlowp::RoundingDivideByPOT; + const int32x4_t output_offset_vec = vdupq_n_s32(output_offset); + const int32x4_t output_activation_min_vec = + vdupq_n_s32(output_activation_min); + const int32x4_t output_activation_max_vec = + vdupq_n_s32(output_activation_max); + + // Fixed-point multiplication. + acc_0.low = vqrdmulhq_n_s32(acc_0.low, output_multiplier); + acc_0.high = vqrdmulhq_n_s32(acc_0.high, output_multiplier); + acc_1.low = vqrdmulhq_n_s32(acc_1.low, output_multiplier); + acc_1.high = vqrdmulhq_n_s32(acc_1.high, output_multiplier); + + acc_0.low = RoundingDivideByPOT(acc_0.low, output_shift); + acc_0.high = RoundingDivideByPOT(acc_0.high, output_shift); + acc_1.low = RoundingDivideByPOT(acc_1.low, output_shift); + acc_1.high = RoundingDivideByPOT(acc_1.high, output_shift); + + // Add the output offset. + acc_0.low = vaddq_s32(acc_0.low, output_offset_vec); + acc_0.high = vaddq_s32(acc_0.high, output_offset_vec); + acc_1.low = vaddq_s32(acc_1.low, output_offset_vec); + acc_1.high = vaddq_s32(acc_1.high, output_offset_vec); + + // Apply the activation function. + acc_0.low = vmaxq_s32(acc_0.low, output_activation_min_vec); + acc_0.high = vmaxq_s32(acc_0.high, output_activation_min_vec); + acc_1.low = vmaxq_s32(acc_1.low, output_activation_min_vec); + acc_1.high = vmaxq_s32(acc_1.high, output_activation_min_vec); + + acc_0.low = vminq_s32(acc_0.low, output_activation_max_vec); + acc_0.high = vminq_s32(acc_0.high, output_activation_max_vec); + acc_1.low = vminq_s32(acc_1.low, output_activation_max_vec); + acc_1.high = vminq_s32(acc_1.high, output_activation_max_vec); + } + + // Saturating cast to uint8 and store to destination. + int16x8_t res_0_s16; + { + int16x4_t acc_0_low_s16 = vqmovn_s32(acc_0.low); + int16x4_t acc_0_high_s16 = vqmovn_s32(acc_0.high); + res_0_s16 = vcombine_s16(acc_0_low_s16, acc_0_high_s16); + } + + int16x8_t res_1_s16; + { + int16x4_t acc_1_low_s16 = vqmovn_s32(acc_1.low); + int16x4_t acc_1_high_s16 = vqmovn_s32(acc_1.high); + res_1_s16 = vcombine_s16(acc_1_low_s16, acc_1_high_s16); + } + + uint8x8_t res_0_u8 = vqmovun_s16(res_0_s16); + uint8x8_t res_1_u8 = vqmovun_s16(res_1_s16); + vst1_u8(output_ptr, res_0_u8); + vst1_u8(output_ptr + output_ptr_offset, res_1_u8); +} + +// Performs multiply accumulate on 3 inputs of depth 8. +inline Int32x8 MultiplyAccumulateRow(Int32x8 accum, int16x8_t f0, int16x8_t f1, + int16x8_t f2, int16x8_t i0, int16x8_t i1, + int16x8_t i2) { + accum.low = vmlal_s16(accum.low, vget_low_s16(f0), vget_low_s16(i0)); + accum.high = vmlal_s16(accum.high, vget_high_s16(f0), vget_high_s16(i0)); + accum.low = vmlal_s16(accum.low, vget_low_s16(f1), vget_low_s16(i1)); + accum.high = vmlal_s16(accum.high, vget_high_s16(f1), vget_high_s16(i1)); + accum.low = vmlal_s16(accum.low, vget_low_s16(f2), vget_low_s16(i2)); + accum.high = vmlal_s16(accum.high, vget_high_s16(f2), vget_high_s16(i2)); + return accum; +} + +// Performs multiply accumulate on 3 inputs of depth 8. +inline Int32x8 MultiplyAccumulate3x3Filter(const Filter3x3x8& f, int16x8_t i0, + int16x8_t i1, int16x8_t i2, + int16x8_t i3, int16x8_t i4, + int16x8_t i5, int16x8_t i6, + int16x8_t i7, int16x8_t i8, + Int32x8 accum) { + accum.low = vmlal_s16(accum.low, vget_low_s16(f.f0), vget_low_s16(i0)); + accum.high = vmlal_s16(accum.high, vget_high_s16(f.f0), vget_high_s16(i0)); + accum.low = vmlal_s16(accum.low, vget_low_s16(f.f1), vget_low_s16(i1)); + accum.high = vmlal_s16(accum.high, vget_high_s16(f.f1), vget_high_s16(i1)); + accum.low = vmlal_s16(accum.low, vget_low_s16(f.f2), vget_low_s16(i2)); + accum.high = vmlal_s16(accum.high, vget_high_s16(f.f2), vget_high_s16(i2)); + accum.low = vmlal_s16(accum.low, vget_low_s16(f.f3), vget_low_s16(i3)); + accum.high = vmlal_s16(accum.high, vget_high_s16(f.f3), vget_high_s16(i3)); + accum.low = vmlal_s16(accum.low, vget_low_s16(f.f4), vget_low_s16(i4)); + accum.high = vmlal_s16(accum.high, vget_high_s16(f.f4), vget_high_s16(i4)); + accum.low = vmlal_s16(accum.low, vget_low_s16(f.f5), vget_low_s16(i5)); + accum.high = vmlal_s16(accum.high, vget_high_s16(f.f5), vget_high_s16(i5)); + accum.low = vmlal_s16(accum.low, vget_low_s16(f.f6), vget_low_s16(i6)); + accum.high = vmlal_s16(accum.high, vget_high_s16(f.f6), vget_high_s16(i6)); + accum.low = vmlal_s16(accum.low, vget_low_s16(f.f7), vget_low_s16(i7)); + accum.high = vmlal_s16(accum.high, vget_high_s16(f.f7), vget_high_s16(i7)); + accum.low = vmlal_s16(accum.low, vget_low_s16(f.f8), vget_low_s16(i8)); + accum.high = vmlal_s16(accum.high, vget_high_s16(f.f8), vget_high_s16(i8)); + return accum; +} + +inline void DotProductAndStore(const Filter3x3x8& filter, int16x8_t i0, + int16x8_t i1, int16x8_t i2, int16x8_t i3, + int16x8_t i4, int16x8_t i5, int16x8_t i6, + int16x8_t i7, int16x8_t i8, + const int32* bias_ptr, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_ptr) { + Int32x8 acc; + acc.low = vld1q_s32(bias_ptr); + acc.high = vld1q_s32(bias_ptr + 4); + + acc = MultiplyAccumulate3x3Filter(filter, i0, i1, i2, i3, i4, i5, i6, i7, i8, + acc); + + DownquantizeAndStore(acc, output_offset, output_multiplier, output_shift, + output_activation_min, output_activation_max, + output_ptr); +} + +// Performs multiply-accumulate on a 3x4 input for 2 horizontal outputs. +inline void DotProductAndStore2xStride1( + const Filter3x3x8& filter, int16x8_t i0, int16x8_t i1, int16x8_t i2, + int16x8_t i3, int16x8_t i4, int16x8_t i5, int16x8_t i6, int16x8_t i7, + int16x8_t i8, int16x8_t i9, int16x8_t i10, int16x8_t i11, + const int32* bias_ptr, int32 output_offset, int32 output_multiplier, + int output_shift, int32 output_activation_min, int32 output_activation_max, + uint8* output_ptr, int output_ptr_offset) { + Int32x8 acc_0, acc_1; + acc_0.low = vld1q_s32(bias_ptr); + acc_1.low = vld1q_s32(bias_ptr); + acc_0.high = vld1q_s32(bias_ptr + 4); + acc_1.high = vld1q_s32(bias_ptr + 4); + + acc_0 = MultiplyAccumulate3x3Filter(filter, i0, i1, i2, i4, i5, i6, i8, i9, + i10, acc_0); + acc_1 = MultiplyAccumulate3x3Filter(filter, i1, i2, i3, i5, i6, i7, i9, i10, + i11, acc_1); + DownquantizeAndStore2Output(acc_0, acc_1, output_offset, output_multiplier, + output_shift, output_activation_min, + output_activation_max, output_ptr, + output_ptr_offset); +} + +// Performs multiply-accumulate on a 4x3 input for 2 vertical outputs. +inline void DotProductAndStore2yStride1( + const Filter3x3x8& filter, int16x8_t i0, int16x8_t i1, int16x8_t i2, + int16x8_t i3, int16x8_t i4, int16x8_t i5, int16x8_t i6, int16x8_t i7, + int16x8_t i8, int16x8_t i9, int16x8_t i10, int16x8_t i11, + const int32* bias_ptr, int32 output_offset, int32 output_multiplier, + int output_shift, int32 output_activation_min, int32 output_activation_max, + uint8* output_ptr, int output_ptr_offset) { + Int32x8 acc_0, acc_1; + acc_0.low = vld1q_s32(bias_ptr); + acc_1.low = vld1q_s32(bias_ptr); + acc_0.high = vld1q_s32(bias_ptr + 4); + acc_1.high = vld1q_s32(bias_ptr + 4); + + acc_0 = MultiplyAccumulate3x3Filter(filter, i0, i1, i2, i3, i4, i5, i6, i7, + i8, acc_0); + acc_1 = MultiplyAccumulate3x3Filter(filter, i3, i4, i5, i6, i7, i8, i9, i10, + i11, acc_1); + DownquantizeAndStore2Output(acc_0, acc_1, output_offset, output_multiplier, + output_shift, output_activation_min, + output_activation_max, output_ptr, + output_ptr_offset); +} + +// A kernel that is optimized on the number of output cells in the x and y +// direction, and the stride. Assumes 3x3 filters of 16 depth. +template +struct ConvKernel3x3FilterDepth8 {}; + +template <> +struct ConvKernel3x3FilterDepth8<8, 8, 1> { + static inline void Run(const uint8* input_ptr, int input_depth, + int32 input_offset, int input_row_size, + const uint8* filter_ptr, int32 filter_offset, + const int32* bias_ptr, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_ptr, + int output_depth, int output_width) { + Filter3x3x8 filter = Load3x3Filter(filter_ptr, filter_offset, output_depth); + + const int16x8_t input_offset_vec = vdupq_n_s16(input_offset); + const int output_row_size = output_depth * output_width; + + // To process 8x8 outputs using a 3x3 filter, we require 10x10 inputs. + // Load inputs for the first 2 filters on the top left, then slide to + // the right, down, left, down, right, etc. in a snake-like path. This + // minimizes the total number of loads. + // + // INPUT OUTPUT + // |\----------------\ |\------------\ + // | \ \ | \ \ + // | \----------------\ | \------------\ + // | | 0 ... 9 | | | 0 ... 7 | + // | | 10 ... 19 | ---> | | 8 ... 15 | + // | | 20 ... 29 | \ | .. ... .. | + // \ | .. ... .. | \| 56 ... 63 | + // \| 90 ... 109 | |------------| + // |----------------| + // + // The first set of loads corresponds to: + // + // INPUT OUTPUT + // |\----------------- |\----------- + // | \ | \ + // | \----------------- | \---------- + // | | 0 1 2 3 ... | | 0 1 ... + // | | 10 11 12 13 ... ---> | | .. ... + // | | 20 21 22 23 ... | .. ... + // | | .. ... ... + // + // The next set of loads correspond to a sliding window to the right. + // It loads inputs 4, 5, 14, 15, 23, 24 and keeps 2, 3, 12, 13, and 22: + // + // INPUT OUTPUT + // |\------------------- |\------------- + // | \ | \ + // | \------------------- | \------------ + // | | .. 2 3 4 5 ... | | .. 2 3 ... + // | | .. 12 13 14 15 ... ---> | | .. ... + // | | .. 21 22 23 24 ... | .. ... + // | | .. ... ... + // + // And so on... + + int16x8_t input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, input_9, input_10, input_11; + + // Load inputs for 1x2 outputs starting from the top left. Referring to the + // indexes in the diagram above, this corresponds to outputs (0) and (1). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3; + + const uint8* ptr = input_ptr; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_10 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_11 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + input_10 = vaddq_s16(input_10, input_offset_vec); + input_11 = vaddq_s16(input_11, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, input_9, input_10, input_11, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr, output_depth); + + // Slide to the right for outputs x = [2, 3], y = 0. Referring to the + // indexes in the diagram above, this corresponds to outputs (2) and (3). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 4 * input_depth; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_2, input_3, input_0, input_1, input_6, input_7, input_4, + input_5, input_10, input_11, input_8, input_9, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr + 2 * output_depth, output_depth); + + // Slide to the right again for outputs x = [4, 5], y = 0. Referring to the + // indexes in the diagram above, this corresponds to outputs (4) and (5). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 6 * input_depth; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_10 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_11 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + input_10 = vaddq_s16(input_10, input_offset_vec); + input_11 = vaddq_s16(input_11, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, input_9, input_10, input_11, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr + 4 * output_depth, output_depth); + + // Slide to the right one last time for outputs x = [6, 7], y = 0. + // Referring to the indexes in the diagram above, this corresponds to + // outputs (6) and (7). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 8 * input_depth; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_2, input_3, input_0, input_1, input_6, input_7, input_4, + input_5, input_10, input_11, input_8, input_9, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr + 6 * output_depth, output_depth); + + // Slide to down for outputs x = [6, 7], y = 1. Referring to the indexes in + // the diagram above, this corresponds to outputs (14) and (15). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3; + + const uint8* ptr = input_ptr + 6 * input_depth + 3 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_6, input_7, input_4, input_5, input_10, input_11, input_8, + input_9, input_2, input_3, input_0, input_1, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr + 6 * output_depth + output_row_size, + output_depth); + + // Slide left for outputs x = [4, 5], y = 1. Referring to the indexes in + // the diagram above, this corresponds to outputs (12) and (13). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 4 * input_depth + input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_4, input_5, input_6, input_7, input_8, input_9, input_10, + input_11, input_0, input_1, input_2, input_3, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr + 4 * output_depth + output_row_size, + output_depth); + + // Slide left again for outputs x = [2, 3], y = 1. Referring to the indexes + // in the diagram above, this corresponds to outputs (10) and (11). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 2 * input_depth + input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_10 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_11 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + input_10 = vaddq_s16(input_10, input_offset_vec); + input_11 = vaddq_s16(input_11, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_6, input_7, input_4, input_5, input_10, input_11, input_8, + input_9, input_2, input_3, input_0, input_1, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr + 2 * output_depth + output_row_size, + output_depth); + + // Slide left one more time for outputs x = [0, 1], y = 1. Referring to the + // indexes in the diagram above, this corresponds to outputs (8) and (9). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_4, input_5, input_6, input_7, input_8, input_9, input_10, + input_11, input_0, input_1, input_2, input_3, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr + output_row_size, output_depth); + + // Slide down for outputs x = [0, 1], y = 2. Referring to the + // indexes in the diagram above, this corresponds to outputs (16) and (17). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3; + + const uint8* ptr = input_ptr + 4 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_8, input_9, input_10, input_11, input_0, input_1, input_2, + input_3, input_4, input_5, input_6, input_7, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr + 2 * output_row_size, output_depth); + + // Slide right for outputs x = [2, 3], y = 2. Referring to the + // indexes in the diagram above, this corresponds to outputs (18) and (19). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 4 * input_depth + 2 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_10, input_11, input_8, input_9, input_2, input_3, input_0, + input_1, input_6, input_7, input_4, input_5, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, + output_ptr + 2 * output_depth + 2 * output_row_size, output_depth); + + // Slide right for outputs x = [4, 5], y = 2. Referring to the + // indexes in the diagram above, this corresponds to outputs (20) and (21). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 6 * input_depth + 2 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_10 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_11 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_10 = vaddq_s16(input_10, input_offset_vec); + input_11 = vaddq_s16(input_11, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_8, input_9, input_10, input_11, input_0, input_1, input_2, + input_3, input_4, input_5, input_6, input_7, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, + output_ptr + 4 * output_depth + 2 * output_row_size, output_depth); + + // Slide right one more time for outputs x = [6, 7], y = 2. Referring to the + // indexes in the diagram above, this corresponds to outputs (22) and (23). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 8 * input_depth + 2 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_10, input_11, input_8, input_9, input_2, input_3, input_0, + input_1, input_6, input_7, input_4, input_5, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, + output_ptr + 6 * output_depth + 2 * output_row_size, output_depth); + + // Slide down for outputs x = [6, 7], y = 3. Referring to the indexes in + // the diagram above, this corresponds to outputs (30) and (31). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3; + + const uint8* ptr = input_ptr + 6 * input_depth + 5 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_10 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_11 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_10 = vaddq_s16(input_10, input_offset_vec); + input_11 = vaddq_s16(input_11, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_2, input_3, input_0, input_1, input_6, input_7, input_4, + input_5, input_10, input_11, input_8, input_9, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, + output_ptr + 6 * output_depth + 3 * output_row_size, output_depth); + + // Slide left for outputs x = [4, 5], y = 3. Referring to the indexes in + // the diagram above, this corresponds to outputs (28) and (29). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 4 * input_depth + 3 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, input_9, input_10, input_11, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, + output_ptr + 4 * output_depth + 3 * output_row_size, output_depth); + + // Slide left for outputs x = [2, 3], y = 3. Referring to the indexes in + // the diagram above, this corresponds to outputs (26) and (27). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 2 * input_depth + 3 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_10 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_11 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + input_10 = vaddq_s16(input_10, input_offset_vec); + input_11 = vaddq_s16(input_11, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_2, input_3, input_0, input_1, input_6, input_7, input_4, + input_5, input_10, input_11, input_8, input_9, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, + output_ptr + 2 * output_depth + 3 * output_row_size, output_depth); + + // Slide left one more time for outputs x = [0, 1], y = 3. Referring to the + // indexes in the diagram above, this corresponds to outputs (24) and (25). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 3 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, input_9, input_10, input_11, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr + 3 * output_row_size, output_depth); + + // Slide down for outputs x = [0, 1], y = 4. Referring to the indexes in + // the diagram above, this corresponds to outputs (32) and (33). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3; + + const uint8* ptr = input_ptr + 6 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_4, input_5, input_6, input_7, input_8, input_9, input_10, + input_11, input_0, input_1, input_2, input_3, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr + 4 * output_row_size, output_depth); + + // Slide right for outputs x = [2, 3], y = 4. Referring to the indexes in + // the diagram above, this corresponds to outputs (34) and (35). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 4 * input_depth + 4 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_6, input_7, input_4, input_5, input_10, input_11, input_8, + input_9, input_2, input_3, input_0, input_1, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, + output_ptr + 2 * output_depth + 4 * output_row_size, output_depth); + + // Slide right for outputs x = [4, 5], y = 4. Referring to the indexes in + // the diagram above, this corresponds to outputs (36) and (37). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 6 * input_depth + 4 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_10 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_11 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + input_10 = vaddq_s16(input_10, input_offset_vec); + input_11 = vaddq_s16(input_11, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_4, input_5, input_6, input_7, input_8, input_9, input_10, + input_11, input_0, input_1, input_2, input_3, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, + output_ptr + 4 * output_depth + 4 * output_row_size, output_depth); + + // Slide right one more time for outputs x = [6, 7], y = 4. Referring to the + // indexes in the diagram above, this corresponds to outputs (38) and (39). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 8 * input_depth + 4 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_6, input_7, input_4, input_5, input_10, input_11, input_8, + input_9, input_2, input_3, input_0, input_1, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, + output_ptr + 6 * output_depth + 4 * output_row_size, output_depth); + + // Slide down for outputs x = [6, 7], y = 5. Referring to the indexes in + // the diagram above, this corresponds to outputs (46) and (47). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3; + + const uint8* ptr = input_ptr + 6 * input_depth + 7 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_10, input_11, input_8, input_9, input_2, input_3, input_0, + input_1, input_6, input_7, input_4, input_5, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, + output_ptr + 6 * output_depth + 5 * output_row_size, output_depth); + + // Slide left for outputs x = [4, 5], y = 5. Referring to the indexes in + // the diagram above, this corresponds to outputs (44) and (45). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 4 * input_depth + 5 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_8, input_9, input_10, input_11, input_0, input_1, input_2, + input_3, input_4, input_5, input_6, input_7, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, + output_ptr + 4 * output_depth + 5 * output_row_size, output_depth); + + // Slide left for outputs x = [2, 3], y = 5. Referring to the indexes in + // the diagram above, this corresponds to outputs (42) and (43). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 2 * input_depth + 5 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_10 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_11 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_10 = vaddq_s16(input_10, input_offset_vec); + input_11 = vaddq_s16(input_11, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_10, input_11, input_8, input_9, input_2, input_3, input_0, + input_1, input_6, input_7, input_4, input_5, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, + output_ptr + 2 * output_depth + 5 * output_row_size, output_depth); + + // Slide left one more time for outputs x = [0, 1], y = 5. Referring to the + // indexes in the diagram above, this corresponds to outputs (40) and (41). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 5 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_8, input_9, input_10, input_11, input_0, input_1, input_2, + input_3, input_4, input_5, input_6, input_7, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr + 5 * output_row_size, output_depth); + + // Slide down for outputs x = [0, 1], y = 6. Referring to the indexes in + // the diagram above, this corresponds to outputs (48) and (49). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3; + + const uint8* ptr = input_ptr + 8 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_10 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_11 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + input_10 = vaddq_s16(input_10, input_offset_vec); + input_11 = vaddq_s16(input_11, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, input_9, input_10, input_11, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr + 6 * output_row_size, output_depth); + + // Slide right for outputs x = [2, 3], y = 6. Referring to the indexes in + // the diagram above, this corresponds to outputs (50) and (51). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 4 * input_depth + 6 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_2, input_3, input_0, input_1, input_6, input_7, input_4, + input_5, input_10, input_11, input_8, input_9, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, + output_ptr + 2 * output_depth + 6 * output_row_size, output_depth); + + // Slide right for outputs x = [4, 5], y = 6. Referring to the indexes in + // the diagram above, this corresponds to outputs (52) and (53). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 6 * input_depth + 6 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_10 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_11 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + input_10 = vaddq_s16(input_10, input_offset_vec); + input_11 = vaddq_s16(input_11, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, input_9, input_10, input_11, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, + output_ptr + 4 * output_depth + 6 * output_row_size, output_depth); + + // Slide right one more time for outputs x = [6, 7], y = 6. Referring to the + // indexes in the diagram above, this corresponds to outputs (54) and (55). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 8 * input_depth + 6 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_2, input_3, input_0, input_1, input_6, input_7, input_4, + input_5, input_10, input_11, input_8, input_9, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, + output_ptr + 6 * output_depth + 6 * output_row_size, output_depth); + + // Slide down for outputs x = [6, 7], y = 7. Referring to the indexes in the + // diagram above, this corresponds to outputs (62) and (63). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3; + + const uint8* ptr = input_ptr + 6 * input_depth + 9 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_6, input_7, input_4, input_5, input_10, input_11, input_8, + input_9, input_2, input_3, input_0, input_1, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, + output_ptr + 6 * output_depth + 7 * output_row_size, output_depth); + + // Slide left for outputs x = [4, 5], y = 7. Referring to the indexes in the + // diagram above, this corresponds to outputs (60) and (61). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 4 * input_depth + 7 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_4, input_5, input_6, input_7, input_8, input_9, input_10, + input_11, input_0, input_1, input_2, input_3, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, + output_ptr + 4 * output_depth + 7 * output_row_size, output_depth); + + // Slide left for outputs x = [2, 3], y = 7. Referring to the indexes in the + // diagram above, this corresponds to outputs (58) and (59). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 2 * input_depth + 7 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_10 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_11 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + input_10 = vaddq_s16(input_10, input_offset_vec); + input_11 = vaddq_s16(input_11, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_6, input_7, input_4, input_5, input_10, input_11, input_8, + input_9, input_2, input_3, input_0, input_1, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, + output_ptr + 2 * output_depth + 7 * output_row_size, output_depth); + + // Slide left one more time for outputs x = [0, 1], y = 7. Referring to the + // indexes in the diagram above, this corresponds to outputs (56) and (57). + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 7 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_4, input_5, input_6, input_7, input_8, input_9, input_10, + input_11, input_0, input_1, input_2, input_3, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr + 7 * output_row_size, output_depth); + } +}; + +template <> +struct ConvKernel3x3FilterDepth8<4, 4, 1> { + static inline void Run(const uint8* input_ptr, int input_depth, + int32 input_offset, int input_row_size, + const uint8* filter_ptr, int32 filter_offset, + const int32* bias_ptr, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_ptr, + int output_depth, int output_width) { + Filter3x3x8 filter = Load3x3Filter(filter_ptr, filter_offset, output_depth); + + const int16x8_t input_offset_vec = vdupq_n_s16(input_offset); + const int output_row_size = output_depth * output_width; + + // To process 4x4 outputs using a 3x3 filter, we require 6x6 inputs. + // Load inputs for the first 2 filters on the top left, then slide to + // the right, down, left, down, right, etc. in a snake-like path. This + // minimizes the total number of loads. + int16x8_t input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, input_9, input_10, input_11; + + // Load inputs for 1x2 outputs starting from the top left. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3; + + const uint8* ptr = input_ptr; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_10 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_11 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + input_10 = vaddq_s16(input_10, input_offset_vec); + input_11 = vaddq_s16(input_11, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, input_9, input_10, input_11, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr, output_depth); + + // Now load 1x2 inputs on the top right. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 4 * input_depth; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_2, input_3, input_0, input_1, input_6, input_7, input_4, + input_5, input_10, input_11, input_8, input_9, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr + 2 * output_depth, output_depth); + + // Now load next inputs when sliding window down. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3; + + const uint8* ptr = input_ptr + 2 * input_depth + 3 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_6, input_7, input_4, input_5, input_10, input_11, input_8, + input_9, input_2, input_3, input_0, input_1, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr + 2 * output_depth + output_row_size, + output_depth); + + // Now load next inputs when sliding window left. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_4, input_5, input_6, input_7, input_8, input_9, input_10, + input_11, input_0, input_1, input_2, input_3, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr + output_row_size, output_depth); + + // Now load next inputs when sliding window down. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3; + + const uint8* ptr = input_ptr + 4 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_8, input_9, input_10, input_11, input_0, input_1, input_2, + input_3, input_4, input_5, input_6, input_7, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr + 2 * output_row_size, output_depth); + + // Now load next inputs when sliding window right. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 4 * input_depth + 2 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_10, input_11, input_8, input_9, input_2, input_3, input_0, + input_1, input_6, input_7, input_4, input_5, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, + output_ptr + 2 * output_depth + 2 * output_row_size, output_depth); + + // Now load next inputs when sliding window down. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3; + + const uint8* ptr = input_ptr + 2 * input_depth + 5 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_10 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_11 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_10 = vaddq_s16(input_10, input_offset_vec); + input_11 = vaddq_s16(input_11, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_2, input_3, input_0, input_1, input_6, input_7, input_4, + input_5, input_10, input_11, input_8, input_9, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, + output_ptr + 2 * output_depth + 3 * output_row_size, output_depth); + + // Now load next inputs when sliding window left. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 3 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, input_9, input_10, input_11, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr + 3 * output_row_size, output_depth); + } +}; + +template <> +struct ConvKernel3x3FilterDepth8<4, 2, 1> { + static inline void Run(const uint8* input_ptr, int input_depth, + int32 input_offset, int input_row_size, + const uint8* filter_ptr, int32 filter_offset, + const int32* bias_ptr, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_ptr, + int output_depth, int output_width) { + Filter3x3x8 filter = Load3x3Filter(filter_ptr, filter_offset, output_depth); + + const int16x8_t input_offset_vec = vdupq_n_s16(input_offset); + const int output_row_size = output_depth * output_width; + + int16x8_t input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, input_9, input_10, input_11; + + // Load inputs for 1x2 outputs starting from the top. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3; + + const uint8* ptr = input_ptr; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_10 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_11 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + input_10 = vaddq_s16(input_10, input_offset_vec); + input_11 = vaddq_s16(input_11, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, input_9, input_10, input_11, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr, output_depth); + + output_ptr += output_row_size; + + // Now load next inputs one row down. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3; + + const uint8* ptr = input_ptr + 3 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_4, input_5, input_6, input_7, input_8, input_9, input_10, + input_11, input_0, input_1, input_2, input_3, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr, output_depth); + + output_ptr += output_row_size; + + // Now load next row. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3; + + const uint8* ptr = input_ptr + 4 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_8, input_9, input_10, input_11, input_0, input_1, input_2, + input_3, input_4, input_5, input_6, input_7, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr, output_depth); + + output_ptr += output_row_size; + + // Now load last row. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3; + + const uint8* ptr = input_ptr + 5 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_10 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_11 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + input_10 = vaddq_s16(input_10, input_offset_vec); + input_11 = vaddq_s16(input_11, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, input_9, input_10, input_11, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr, output_depth); + } +}; + +template <> +struct ConvKernel3x3FilterDepth8<4, 1, 1> { + static inline void Run(const uint8* input_ptr, int input_depth, + int32 input_offset, int input_row_size, + const uint8* filter_ptr, int32 filter_offset, + const int32* bias_ptr, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_ptr, + int output_depth, int output_width) { + Filter3x3x8 filter = Load3x3Filter(filter_ptr, filter_offset, output_depth); + + const int16x8_t input_offset_vec = vdupq_n_s16(input_offset); + const int output_row_size = output_depth * output_width; + + int16x8_t input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, input_9, input_10, input_11; + + // Load inputs for 2x1 outputs starting from the top. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + ptr += input_row_size; + temp_3 = vld1_u8(ptr); + temp_4 = vld1_u8(ptr + input_depth); + temp_5 = vld1_u8(ptr + 2 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + ptr += input_row_size; + temp_3 = vld1_u8(ptr); + temp_4 = vld1_u8(ptr + input_depth); + temp_5 = vld1_u8(ptr + 2 * input_depth); + + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_10 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_11 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + input_10 = vaddq_s16(input_10, input_offset_vec); + input_11 = vaddq_s16(input_11, input_offset_vec); + } + + DotProductAndStore2yStride1( + filter, input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, input_9, input_10, input_11, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr, output_row_size); + + // Load inputs for bottom 2 rows. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 4 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + ptr += input_row_size; + temp_3 = vld1_u8(ptr); + temp_4 = vld1_u8(ptr + input_depth); + temp_5 = vld1_u8(ptr + 2 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + } + + DotProductAndStore2yStride1( + filter, input_6, input_7, input_8, input_9, input_10, input_11, input_0, + input_1, input_2, input_3, input_4, input_5, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr + 2 * output_row_size, + output_row_size); + } +}; + +template <> +struct ConvKernel3x3FilterDepth8<2, 2, 1> { + static inline void Run(const uint8* input_ptr, int input_depth, + int32 input_offset, int input_row_size, + const uint8* filter_ptr, int32 filter_offset, + const int32* bias_ptr, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_ptr, + int output_depth, int output_width) { + Filter3x3x8 filter = Load3x3Filter(filter_ptr, filter_offset, output_depth); + + Int32x8 acc_0, acc_1, acc_2, acc_3; + + acc_0.low = vld1q_s32(bias_ptr); + acc_1.low = vld1q_s32(bias_ptr); + acc_2.low = vld1q_s32(bias_ptr); + acc_3.low = vld1q_s32(bias_ptr); + + bias_ptr += 4; + acc_0.high = vld1q_s32(bias_ptr); + acc_1.high = vld1q_s32(bias_ptr); + acc_2.high = vld1q_s32(bias_ptr); + acc_3.high = vld1q_s32(bias_ptr); + + const int16x8_t input_offset_vec = vdupq_n_s16(input_offset); + + // Add scope for input registers to help the compiler know that it is + // not needed. + { + // To process 2x2 outputs using a 3x3 filter, we require 4x4 inputs. + // Load inputs for the top two filters first. + int16x8_t input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, input_9, input_10, input_11; + + const uint8* ptr = input_ptr; + + // Load top 3 rows. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3; + + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_10 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_11 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + input_10 = vaddq_s16(input_10, input_offset_vec); + input_11 = vaddq_s16(input_11, input_offset_vec); + } + + // Multiply-accum for top-left output. + acc_0 = MultiplyAccumulate3x3Filter(filter, input_0, input_1, input_2, + input_4, input_5, input_6, input_8, + input_9, input_10, acc_0); + + // Multiply-accum for top-right output. + acc_1 = MultiplyAccumulate3x3Filter(filter, input_1, input_2, input_3, + input_5, input_6, input_7, input_9, + input_10, input_11, acc_1); + + // Now load the bottom row. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3; + + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + } + + // Multiply-accum for bottom-left output. + acc_2 = MultiplyAccumulate3x3Filter(filter, input_4, input_5, input_6, + input_8, input_9, input_10, input_0, + input_1, input_2, acc_2); + + // Multiply-accum for bottom-right output. + acc_3 = MultiplyAccumulate3x3Filter(filter, input_5, input_6, input_7, + input_9, input_10, input_11, input_1, + input_2, input_3, acc_3); + } + + DownquantizeAndStore2x2Output(acc_0, acc_1, acc_2, acc_3, output_offset, + output_multiplier, output_shift, + output_activation_min, output_activation_max, + output_ptr, output_depth, output_width); + } +}; + +template <> +struct ConvKernel3x3FilterDepth8<2, 4, 1> { + static inline void Run(const uint8* input_ptr, int input_depth, + int32 input_offset, int input_row_size, + const uint8* filter_ptr, int32 filter_offset, + const int32* bias_ptr, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_ptr, + int output_depth, int output_width) { + Filter3x3x8 filter = Load3x3Filter(filter_ptr, filter_offset, output_depth); + + const int16x8_t input_offset_vec = vdupq_n_s16(input_offset); + const int output_row_size = output_depth * output_width; + + int16x8_t input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, input_9, input_10, input_11; + + // Load inputs for 1x2 outputs starting from the top left. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3; + + const uint8* ptr = input_ptr; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_10 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_11 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + input_10 = vaddq_s16(input_10, input_offset_vec); + input_11 = vaddq_s16(input_11, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, input_9, input_10, input_11, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr, output_depth); + + // Now load 1x2 inputs on the top right. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + 4 * input_depth; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_2, input_3, input_0, input_1, input_6, input_7, input_4, + input_5, input_10, input_11, input_8, input_9, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr + 2 * output_depth, output_depth); + + // Now load next inputs when sliding window down. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3; + + const uint8* ptr = input_ptr + 2 * input_depth + 3 * input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_6, input_7, input_4, input_5, input_10, input_11, input_8, + input_9, input_2, input_3, input_0, input_1, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr + 2 * output_depth + output_row_size, + output_depth); + + // Now load next inputs when sliding window left. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_4, input_5, input_6, input_7, input_8, input_9, input_10, + input_11, input_0, input_1, input_2, input_3, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr + output_row_size, output_depth); + } +}; + +template <> +struct ConvKernel3x3FilterDepth8<1, 4, 1> { + static inline void Run(const uint8* input_ptr, int input_depth, + int32 input_offset, int input_row_size, + const uint8* filter_ptr, int32 filter_offset, + const int32* bias_ptr, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_ptr, + int output_depth, int output_width) { + Filter3x3x8 filter = Load3x3Filter(filter_ptr, filter_offset, output_depth); + + const int16x8_t input_offset_vec = vdupq_n_s16(input_offset); + + int16x8_t input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, input_9, input_10, input_11; + + // Load inputs for 1x2 outputs starting from the left. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3; + + const uint8* ptr = input_ptr; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_10 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_11 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + input_10 = vaddq_s16(input_10, input_offset_vec); + input_11 = vaddq_s16(input_11, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, input_9, input_10, input_11, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr, output_depth); + + // Now load 1x2 inputs on the right. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr + input_depth * 4; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_2 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + } + + DotProductAndStore2xStride1( + filter, input_2, input_3, input_0, input_1, input_6, input_7, input_4, + input_5, input_10, input_11, input_8, input_9, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr + 2 * output_depth, output_depth); + } +}; + +template <> +struct ConvKernel3x3FilterDepth8<2, 1, 1> { + static inline void Run(const uint8* input_ptr, int input_depth, + int32 input_offset, int input_row_size, + const uint8* filter_ptr, int32 filter_offset, + const int32* bias_ptr, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_ptr, + int output_depth, int output_width) { + Filter3x3x8 filter = Load3x3Filter(filter_ptr, filter_offset, output_depth); + + // To process 2x1 outputs using a 3x3 filter, we require 4x3 inputs. + // Load all inputs at the beginning. + int16x8_t input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, input_9, input_10, input_11; + + // Load inputs for 1x2 outputs starting from the top left. + { + const int16x8_t input_offset_vec = vdupq_n_s16(input_offset); + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5; + + const uint8* ptr = input_ptr; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + ptr += input_row_size; + temp_3 = vld1_u8(ptr); + temp_4 = vld1_u8(ptr + input_depth); + temp_5 = vld1_u8(ptr + 2 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + ptr += input_row_size; + temp_3 = vld1_u8(ptr); + temp_4 = vld1_u8(ptr + input_depth); + temp_5 = vld1_u8(ptr + 2 * input_depth); + + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_10 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_11 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + input_10 = vaddq_s16(input_10, input_offset_vec); + input_11 = vaddq_s16(input_11, input_offset_vec); + } + + DotProductAndStore2yStride1( + filter, input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, input_9, input_10, input_11, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr, output_depth * output_width); + } +}; + +template <> +struct ConvKernel3x3FilterDepth8<4, 2, 2> { + static inline void Run(const uint8* input_ptr, int input_depth, + int32 input_offset, int input_row_size, + const uint8* filter_ptr, int32 filter_offset, + const int32* bias_ptr, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_ptr, + int output_depth, int output_width) { + const int output_row_size = output_depth * output_width; + + Filter3x3x8 filter = Load3x3Filter(filter_ptr, filter_offset, output_depth); + + Int32x8 acc_0, acc_1; + acc_0.low = vld1q_s32(bias_ptr); + acc_1.low = vld1q_s32(bias_ptr); + acc_0.high = vld1q_s32(bias_ptr + 4); + acc_1.high = vld1q_s32(bias_ptr + 4); + + const int16x8_t input_offset_vec = vdupq_n_s16(input_offset); + + int16x8_t input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, input_9; + + const uint8* ptr = input_ptr; + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4; + + // Load first 2 rows. + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + temp_4 = vld1_u8(ptr + 4 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + temp_4 = vld1_u8(ptr + 4 * input_depth); + + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + + input_5 = vaddq_s16(input_5, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + + acc_0 = MultiplyAccumulateRow(acc_0, filter.f0, filter.f1, filter.f2, + input_0, input_1, input_2); + + acc_1 = MultiplyAccumulateRow(acc_1, filter.f0, filter.f1, filter.f2, + input_2, input_3, input_4); + + acc_0 = MultiplyAccumulateRow(acc_0, filter.f3, filter.f4, filter.f5, + input_5, input_6, input_7); + + acc_1 = MultiplyAccumulateRow(acc_1, filter.f3, filter.f4, filter.f5, + input_7, input_8, input_9); + + // Load next 2 rows. + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + temp_4 = vld1_u8(ptr + 4 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + temp_4 = vld1_u8(ptr + 4 * input_depth); + + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + + input_5 = vaddq_s16(input_5, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + + acc_0 = MultiplyAccumulateRow(acc_0, filter.f6, filter.f7, filter.f8, + input_0, input_1, input_2); + + acc_1 = MultiplyAccumulateRow(acc_1, filter.f6, filter.f7, filter.f8, + input_2, input_3, input_4); + + DownquantizeAndStore2Output( + acc_0, acc_1, output_offset, output_multiplier, output_shift, + output_activation_min, output_activation_max, output_ptr, output_depth); + + output_ptr += output_row_size; + + // Moving onto the next row of outputs. + acc_0.low = vld1q_s32(bias_ptr); + acc_1.low = vld1q_s32(bias_ptr); + acc_0.high = vld1q_s32(bias_ptr + 4); + acc_1.high = vld1q_s32(bias_ptr + 4); + + acc_0 = MultiplyAccumulateRow(acc_0, filter.f0, filter.f1, filter.f2, + input_0, input_1, input_2); + + acc_1 = MultiplyAccumulateRow(acc_1, filter.f0, filter.f1, filter.f2, + input_2, input_3, input_4); + + acc_0 = MultiplyAccumulateRow(acc_0, filter.f3, filter.f4, filter.f5, + input_5, input_6, input_7); + + acc_1 = MultiplyAccumulateRow(acc_1, filter.f3, filter.f4, filter.f5, + input_7, input_8, input_9); + + // Load next 2 rows. + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + temp_4 = vld1_u8(ptr + 4 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + temp_4 = vld1_u8(ptr + 4 * input_depth); + + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + + input_5 = vaddq_s16(input_5, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + + acc_0 = MultiplyAccumulateRow(acc_0, filter.f6, filter.f7, filter.f8, + input_0, input_1, input_2); + + acc_1 = MultiplyAccumulateRow(acc_1, filter.f6, filter.f7, filter.f8, + input_2, input_3, input_4); + + DownquantizeAndStore2Output( + acc_0, acc_1, output_offset, output_multiplier, output_shift, + output_activation_min, output_activation_max, output_ptr, output_depth); + + output_ptr += output_row_size; + + // Moving onto the next row of outputs. + acc_0.low = vld1q_s32(bias_ptr); + acc_1.low = vld1q_s32(bias_ptr); + acc_0.high = vld1q_s32(bias_ptr + 4); + acc_1.high = vld1q_s32(bias_ptr + 4); + + acc_0 = MultiplyAccumulateRow(acc_0, filter.f0, filter.f1, filter.f2, + input_0, input_1, input_2); + + acc_1 = MultiplyAccumulateRow(acc_1, filter.f0, filter.f1, filter.f2, + input_2, input_3, input_4); + + acc_0 = MultiplyAccumulateRow(acc_0, filter.f3, filter.f4, filter.f5, + input_5, input_6, input_7); + + acc_1 = MultiplyAccumulateRow(acc_1, filter.f3, filter.f4, filter.f5, + input_7, input_8, input_9); + + // Load next 2 rows. + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + temp_4 = vld1_u8(ptr + 4 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + temp_4 = vld1_u8(ptr + 4 * input_depth); + + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + + input_5 = vaddq_s16(input_5, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + + acc_0 = MultiplyAccumulateRow(acc_0, filter.f6, filter.f7, filter.f8, + input_0, input_1, input_2); + + acc_1 = MultiplyAccumulateRow(acc_1, filter.f6, filter.f7, filter.f8, + input_2, input_3, input_4); + + DownquantizeAndStore2Output( + acc_0, acc_1, output_offset, output_multiplier, output_shift, + output_activation_min, output_activation_max, output_ptr, output_depth); + + output_ptr += output_row_size; + + // Moving onto the next row of outputs. + acc_0.low = vld1q_s32(bias_ptr); + acc_1.low = vld1q_s32(bias_ptr); + acc_0.high = vld1q_s32(bias_ptr + 4); + acc_1.high = vld1q_s32(bias_ptr + 4); + + acc_0 = MultiplyAccumulateRow(acc_0, filter.f0, filter.f1, filter.f2, + input_0, input_1, input_2); + + acc_1 = MultiplyAccumulateRow(acc_1, filter.f0, filter.f1, filter.f2, + input_2, input_3, input_4); + + acc_0 = MultiplyAccumulateRow(acc_0, filter.f3, filter.f4, filter.f5, + input_5, input_6, input_7); + + acc_1 = MultiplyAccumulateRow(acc_1, filter.f3, filter.f4, filter.f5, + input_7, input_8, input_9); + + // Load last row. + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + temp_4 = vld1_u8(ptr + 4 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + + acc_0 = MultiplyAccumulateRow(acc_0, filter.f6, filter.f7, filter.f8, + input_0, input_1, input_2); + + acc_1 = MultiplyAccumulateRow(acc_1, filter.f6, filter.f7, filter.f8, + input_2, input_3, input_4); + + DownquantizeAndStore2Output( + acc_0, acc_1, output_offset, output_multiplier, output_shift, + output_activation_min, output_activation_max, output_ptr, output_depth); + } +}; + +template <> +struct ConvKernel3x3FilterDepth8<4, 4, 2> { + static inline void Run(const uint8* input_ptr, int input_depth, + int32 input_offset, int input_row_size, + const uint8* filter_ptr, int32 filter_offset, + const int32* bias_ptr, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_ptr, + int output_depth, int output_width) { + // Reuse 4x2 kernel twice. + ConvKernel3x3FilterDepth8<4, 2, 2>::Run( + input_ptr, input_depth, input_offset, input_row_size, filter_ptr, + filter_offset, bias_ptr, output_offset, output_multiplier, output_shift, + output_activation_min, output_activation_max, output_ptr, output_depth, + output_width); + + ConvKernel3x3FilterDepth8<4, 2, 2>::Run( + input_ptr + 4 * input_depth, input_depth, input_offset, input_row_size, + filter_ptr, filter_offset, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, + output_ptr + 2 * output_depth, output_depth, output_width); + } +}; + +template <> +struct ConvKernel3x3FilterDepth8<4, 1, 2> { + static inline void Run(const uint8* input_ptr, int input_depth, + int32 input_offset, int input_row_size, + const uint8* filter_ptr, int32 filter_offset, + const int32* bias_ptr, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_ptr, + int output_depth, int output_width) { + const int output_row_size = output_depth * output_width; + + Filter3x3x8 filter = Load3x3Filter(filter_ptr, filter_offset, output_depth); + + const int16x8_t input_offset_vec = vdupq_n_s16(input_offset); + int16x8_t input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8; + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5, temp_6, temp_7, + temp_8; + + const uint8* ptr = input_ptr; + + // Load all inputs for top output. + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + ptr += input_row_size; + temp_3 = vld1_u8(ptr); + temp_4 = vld1_u8(ptr + input_depth); + temp_5 = vld1_u8(ptr + 2 * input_depth); + ptr += input_row_size; + temp_6 = vld1_u8(ptr); + temp_7 = vld1_u8(ptr + input_depth); + temp_8 = vld1_u8(ptr + 2 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_6)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_7)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_8)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + + DotProductAndStore( + filter, input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, output_ptr); + + // Second output. + output_ptr += output_row_size; + + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + ptr += input_row_size; + temp_3 = vld1_u8(ptr); + temp_4 = vld1_u8(ptr + input_depth); + temp_5 = vld1_u8(ptr + 2 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + + DotProductAndStore( + filter, input_6, input_7, input_8, input_0, input_1, input_2, input_3, + input_4, input_5, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, output_ptr); + + // Third output. + output_ptr += output_row_size; + + ptr += input_row_size; + temp_6 = vld1_u8(ptr); + temp_7 = vld1_u8(ptr + input_depth); + temp_8 = vld1_u8(ptr + 2 * input_depth); + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_6)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_7)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_8)); + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + + DotProductAndStore( + filter, input_3, input_4, input_5, input_6, input_7, input_8, input_0, + input_1, input_2, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, output_ptr); + + // Fourth output. + output_ptr += output_row_size; + + ptr += input_row_size; + temp_3 = vld1_u8(ptr); + temp_4 = vld1_u8(ptr + input_depth); + temp_5 = vld1_u8(ptr + 2 * input_depth); + ptr += input_row_size; + temp_6 = vld1_u8(ptr); + temp_7 = vld1_u8(ptr + input_depth); + temp_8 = vld1_u8(ptr + 2 * input_depth); + + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_6)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_7)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_8)); + + input_3 = vaddq_s16(input_3, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + + DotProductAndStore( + filter, input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, output_ptr); + } +}; + +template <> +struct ConvKernel3x3FilterDepth8<2, 2, 2> { + static inline void Run(const uint8* input_ptr, int input_depth, + int32 input_offset, int input_row_size, + const uint8* filter_ptr, int32 filter_offset, + const int32* bias_ptr, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_ptr, + int output_depth, int output_width) { + Filter3x3x8 filter = Load3x3Filter(filter_ptr, filter_offset, output_depth); + + Int32x8 acc_0, acc_1, acc_2, acc_3; + acc_0.low = vld1q_s32(bias_ptr); + acc_1.low = vld1q_s32(bias_ptr); + acc_2.low = vld1q_s32(bias_ptr); + acc_3.low = vld1q_s32(bias_ptr); + + bias_ptr += 4; + acc_0.high = vld1q_s32(bias_ptr); + acc_1.high = vld1q_s32(bias_ptr); + acc_2.high = vld1q_s32(bias_ptr); + acc_3.high = vld1q_s32(bias_ptr); + + const int16x8_t input_offset_vec = vdupq_n_s16(input_offset); + + // Add scope for input registers to help the compiler know that it is + // not needed. + { + // To process 2x2 outputs using a 3x3 filter at stride 2, we require + // 5x5 inputs. We load the first 5x2 inputs at a time. + int16x8_t input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, input_9; + + const uint8* ptr = input_ptr; + + // Load inputs. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4; + + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + temp_4 = vld1_u8(ptr + 4 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + temp_4 = vld1_u8(ptr + 4 * input_depth); + + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + + input_5 = vaddq_s16(input_5, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + } + + acc_0 = MultiplyAccumulateRow(acc_0, filter.f0, filter.f1, filter.f2, + input_0, input_1, input_2); + + acc_1 = MultiplyAccumulateRow(acc_1, filter.f0, filter.f1, filter.f2, + input_2, input_3, input_4); + + acc_0 = MultiplyAccumulateRow(acc_0, filter.f3, filter.f4, filter.f5, + input_5, input_6, input_7); + + acc_1 = MultiplyAccumulateRow(acc_1, filter.f3, filter.f4, filter.f5, + input_7, input_8, input_9); + + // Load next inputs. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4; + + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + temp_4 = vld1_u8(ptr + 4 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + temp_4 = vld1_u8(ptr + 4 * input_depth); + + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_9 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + + input_5 = vaddq_s16(input_5, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_9 = vaddq_s16(input_9, input_offset_vec); + } + + acc_0 = MultiplyAccumulateRow(acc_0, filter.f6, filter.f7, filter.f8, + input_0, input_1, input_2); + + acc_1 = MultiplyAccumulateRow(acc_1, filter.f6, filter.f7, filter.f8, + input_2, input_3, input_4); + + // Moving onto the two bottom outputs. + acc_2 = MultiplyAccumulateRow(acc_2, filter.f0, filter.f1, filter.f2, + input_0, input_1, input_2); + + acc_3 = MultiplyAccumulateRow(acc_3, filter.f0, filter.f1, filter.f2, + input_2, input_3, input_4); + + acc_2 = MultiplyAccumulateRow(acc_2, filter.f3, filter.f4, filter.f5, + input_5, input_6, input_7); + + acc_3 = MultiplyAccumulateRow(acc_3, filter.f3, filter.f4, filter.f5, + input_7, input_8, input_9); + + // Load last input row. + { + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4; + + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + temp_3 = vld1_u8(ptr + 3 * input_depth); + temp_4 = vld1_u8(ptr + 4 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + } + + acc_2 = MultiplyAccumulateRow(acc_2, filter.f6, filter.f7, filter.f8, + input_0, input_1, input_2); + + acc_3 = MultiplyAccumulateRow(acc_3, filter.f6, filter.f7, filter.f8, + input_2, input_3, input_4); + } + + DownquantizeAndStore2x2Output(acc_0, acc_1, acc_2, acc_3, output_offset, + output_multiplier, output_shift, + output_activation_min, output_activation_max, + output_ptr, output_depth, output_width); + } }; -struct Filter3x3x16 { - Int16x16x3 r0, r1, r2; +template <> +struct ConvKernel3x3FilterDepth8<2, 4, 2> { + static inline void Run(const uint8* input_ptr, int input_depth, + int32 input_offset, int input_row_size, + const uint8* filter_ptr, int32 filter_offset, + const int32* bias_ptr, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_ptr, + int output_depth, int output_width) { + // Reuse 2x2 kernel twice. + ConvKernel3x3FilterDepth8<2, 2, 2>::Run( + input_ptr, input_depth, input_offset, input_row_size, filter_ptr, + filter_offset, bias_ptr, output_offset, output_multiplier, output_shift, + output_activation_min, output_activation_max, output_ptr, output_depth, + output_width); + + ConvKernel3x3FilterDepth8<2, 2, 2>::Run( + input_ptr + 4 * input_depth, input_depth, input_offset, input_row_size, + filter_ptr, filter_offset, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, + output_ptr + 2 * output_depth, output_depth, output_width); + } }; -// Loads 3x3 filter of depth 16 and adds filter offsets. -inline Filter3x3x16 LoadFilterDepth16(const uint8* filter_ptr, - int32 filter_offset, int output_depth) { - Filter3x3x16 filter; +template <> +struct ConvKernel3x3FilterDepth8<2, 1, 2> { + static inline void Run(const uint8* input_ptr, int input_depth, + int32 input_offset, int input_row_size, + const uint8* filter_ptr, int32 filter_offset, + const int32* bias_ptr, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_ptr, + int output_depth, int output_width) { + const int output_row_size = output_depth * output_width; - uint8x8_t temp_u8_0, temp_u8_1, temp_u8_2, temp_u8_3, temp_u8_4, temp_u8_5, - temp_u8_6, temp_u8_7, temp_u8_8, temp_u8_9, temp_u8_10, temp_u8_11, - temp_u8_12, temp_u8_13, temp_u8_14, temp_u8_15, temp_u8_16, temp_u8_17; - int16x8_t filter_offset_vec = vdupq_n_s16(filter_offset); + Filter3x3x8 filter = Load3x3Filter(filter_ptr, filter_offset, output_depth); - temp_u8_0 = vld1_u8(filter_ptr + 0 * output_depth); - temp_u8_1 = vld1_u8(filter_ptr + 0 * output_depth + 8); - temp_u8_2 = vld1_u8(filter_ptr + 1 * output_depth); - temp_u8_3 = vld1_u8(filter_ptr + 1 * output_depth + 8); - temp_u8_4 = vld1_u8(filter_ptr + 2 * output_depth); - temp_u8_5 = vld1_u8(filter_ptr + 2 * output_depth + 8); - - temp_u8_6 = vld1_u8(filter_ptr + 3 * output_depth); - temp_u8_7 = vld1_u8(filter_ptr + 3 * output_depth + 8); - temp_u8_8 = vld1_u8(filter_ptr + 4 * output_depth); - temp_u8_9 = vld1_u8(filter_ptr + 4 * output_depth + 8); - temp_u8_10 = vld1_u8(filter_ptr + 5 * output_depth); - temp_u8_11 = vld1_u8(filter_ptr + 5 * output_depth + 8); - - temp_u8_12 = vld1_u8(filter_ptr + 6 * output_depth); - temp_u8_13 = vld1_u8(filter_ptr + 6 * output_depth + 8); - temp_u8_14 = vld1_u8(filter_ptr + 7 * output_depth); - temp_u8_15 = vld1_u8(filter_ptr + 7 * output_depth + 8); - temp_u8_16 = vld1_u8(filter_ptr + 8 * output_depth); - temp_u8_17 = vld1_u8(filter_ptr + 8 * output_depth + 8); - - filter.r0.v0.low = vreinterpretq_s16_u16(vmovl_u8(temp_u8_0)); - filter.r0.v0.high = vreinterpretq_s16_u16(vmovl_u8(temp_u8_1)); - filter.r0.v1.low = vreinterpretq_s16_u16(vmovl_u8(temp_u8_2)); - filter.r0.v1.high = vreinterpretq_s16_u16(vmovl_u8(temp_u8_3)); - filter.r0.v2.low = vreinterpretq_s16_u16(vmovl_u8(temp_u8_4)); - filter.r0.v2.high = vreinterpretq_s16_u16(vmovl_u8(temp_u8_5)); - - filter.r1.v0.low = vreinterpretq_s16_u16(vmovl_u8(temp_u8_6)); - filter.r1.v0.high = vreinterpretq_s16_u16(vmovl_u8(temp_u8_7)); - filter.r1.v1.low = vreinterpretq_s16_u16(vmovl_u8(temp_u8_8)); - filter.r1.v1.high = vreinterpretq_s16_u16(vmovl_u8(temp_u8_9)); - filter.r1.v2.low = vreinterpretq_s16_u16(vmovl_u8(temp_u8_10)); - filter.r1.v2.high = vreinterpretq_s16_u16(vmovl_u8(temp_u8_11)); - - filter.r2.v0.low = vreinterpretq_s16_u16(vmovl_u8(temp_u8_12)); - filter.r2.v0.high = vreinterpretq_s16_u16(vmovl_u8(temp_u8_13)); - filter.r2.v1.low = vreinterpretq_s16_u16(vmovl_u8(temp_u8_14)); - filter.r2.v1.high = vreinterpretq_s16_u16(vmovl_u8(temp_u8_15)); - filter.r2.v2.low = vreinterpretq_s16_u16(vmovl_u8(temp_u8_16)); - filter.r2.v2.high = vreinterpretq_s16_u16(vmovl_u8(temp_u8_17)); - - filter.r0.v0.low = vaddq_s16(filter.r0.v0.low, filter_offset_vec); - filter.r0.v0.high = vaddq_s16(filter.r0.v0.high, filter_offset_vec); - filter.r0.v1.low = vaddq_s16(filter.r0.v1.low, filter_offset_vec); - filter.r0.v1.high = vaddq_s16(filter.r0.v1.high, filter_offset_vec); - filter.r0.v2.low = vaddq_s16(filter.r0.v2.low, filter_offset_vec); - filter.r0.v2.high = vaddq_s16(filter.r0.v2.high, filter_offset_vec); - - filter.r1.v0.low = vaddq_s16(filter.r1.v0.low, filter_offset_vec); - filter.r1.v0.high = vaddq_s16(filter.r1.v0.high, filter_offset_vec); - filter.r1.v1.low = vaddq_s16(filter.r1.v1.low, filter_offset_vec); - filter.r1.v1.high = vaddq_s16(filter.r1.v1.high, filter_offset_vec); - filter.r1.v2.low = vaddq_s16(filter.r1.v2.low, filter_offset_vec); - filter.r1.v2.high = vaddq_s16(filter.r1.v2.high, filter_offset_vec); - - filter.r2.v0.low = vaddq_s16(filter.r2.v0.low, filter_offset_vec); - filter.r2.v0.high = vaddq_s16(filter.r2.v0.high, filter_offset_vec); - filter.r2.v1.low = vaddq_s16(filter.r2.v1.low, filter_offset_vec); - filter.r2.v1.high = vaddq_s16(filter.r2.v1.high, filter_offset_vec); - filter.r2.v2.low = vaddq_s16(filter.r2.v2.low, filter_offset_vec); - filter.r2.v2.high = vaddq_s16(filter.r2.v2.high, filter_offset_vec); + const int16x8_t input_offset_vec = vdupq_n_s16(input_offset); + int16x8_t input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8; + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5, temp_6, temp_7, + temp_8; - return filter; -} + const uint8* ptr = input_ptr; -// Loads 3 input cells of depth 16 and adds input offsets. -inline Int16x16x3 LoadInputRowDepth16(const uint8* ptr, int input_depth, - int32 input_offset, - Int16x16x3 input_row) { - uint8x8_t temp_0, temp_1; - int16x8_t offset_vec = vdupq_n_s16(input_offset); - - temp_0 = vld1_u8(ptr + 0 * input_depth); - temp_1 = vld1_u8(ptr + 0 * input_depth + 8); - input_row.v0.low = vreinterpretq_s16_u16(vmovl_u8(temp_0)); - input_row.v0.high = vreinterpretq_s16_u16(vmovl_u8(temp_1)); - input_row.v0.low = vaddq_s16(input_row.v0.low, offset_vec); - input_row.v0.high = vaddq_s16(input_row.v0.high, offset_vec); - - temp_0 = vld1_u8(ptr + 1 * input_depth); - temp_1 = vld1_u8(ptr + 1 * input_depth + 8); - input_row.v1.low = vreinterpretq_s16_u16(vmovl_u8(temp_0)); - input_row.v1.high = vreinterpretq_s16_u16(vmovl_u8(temp_1)); - input_row.v1.low = vaddq_s16(input_row.v1.low, offset_vec); - input_row.v1.high = vaddq_s16(input_row.v1.high, offset_vec); - - temp_0 = vld1_u8(ptr + 2 * input_depth); - temp_1 = vld1_u8(ptr + 2 * input_depth + 8); - input_row.v2.low = vreinterpretq_s16_u16(vmovl_u8(temp_0)); - input_row.v2.high = vreinterpretq_s16_u16(vmovl_u8(temp_1)); - input_row.v2.low = vaddq_s16(input_row.v2.low, offset_vec); - input_row.v2.high = vaddq_s16(input_row.v2.high, offset_vec); - - return input_row; -} + // Load all inputs for top output. + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + ptr += input_row_size; + temp_3 = vld1_u8(ptr); + temp_4 = vld1_u8(ptr + input_depth); + temp_5 = vld1_u8(ptr + 2 * input_depth); + ptr += input_row_size; + temp_6 = vld1_u8(ptr); + temp_7 = vld1_u8(ptr + input_depth); + temp_8 = vld1_u8(ptr + 2 * input_depth); -// Performs multiply accumulate on 3 inputs of depth 16. -inline Int32x16 MultiplyAccumulateRowDepth16(Int32x16 output, - const Int16x16x3& filter_row, - const Int16x16x3& input_row) { - output.v0 = vmlal_s16(output.v0, vget_low_s16(filter_row.v0.low), - vget_low_s16(input_row.v0.low)); - output.v1 = vmlal_s16(output.v1, vget_high_s16(filter_row.v0.low), - vget_high_s16(input_row.v0.low)); - output.v2 = vmlal_s16(output.v2, vget_low_s16(filter_row.v0.high), - vget_low_s16(input_row.v0.high)); - output.v3 = vmlal_s16(output.v3, vget_high_s16(filter_row.v0.high), - vget_high_s16(input_row.v0.high)); - - output.v0 = vmlal_s16(output.v0, vget_low_s16(filter_row.v1.low), - vget_low_s16(input_row.v1.low)); - output.v1 = vmlal_s16(output.v1, vget_high_s16(filter_row.v1.low), - vget_high_s16(input_row.v1.low)); - output.v2 = vmlal_s16(output.v2, vget_low_s16(filter_row.v1.high), - vget_low_s16(input_row.v1.high)); - output.v3 = vmlal_s16(output.v3, vget_high_s16(filter_row.v1.high), - vget_high_s16(input_row.v1.high)); - - output.v0 = vmlal_s16(output.v0, vget_low_s16(filter_row.v2.low), - vget_low_s16(input_row.v2.low)); - output.v1 = vmlal_s16(output.v1, vget_high_s16(filter_row.v2.low), - vget_high_s16(input_row.v2.low)); - output.v2 = vmlal_s16(output.v2, vget_low_s16(filter_row.v2.high), - vget_low_s16(input_row.v2.high)); - output.v3 = vmlal_s16(output.v3, vget_high_s16(filter_row.v2.high), - vget_high_s16(input_row.v2.high)); - - return output; -} + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_6)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_7)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_8)); -// Applies activation, offset and downquantize on a set of accumulator -// registers of depth 16. Stores results to output. -inline void DownquantizeAndStoreDepth16(Int32x16 acc, int32 output_multiplier, - int output_shift, - int32x4_t output_offset_vec, - int32x4_t output_activation_min_vec, - int32x4_t output_activation_max_vec, - uint8* output_ptr) { - // Fixed-point multiplication. - acc.v0 = vqrdmulhq_n_s32(acc.v0, output_multiplier); - acc.v1 = vqrdmulhq_n_s32(acc.v1, output_multiplier); - acc.v2 = vqrdmulhq_n_s32(acc.v2, output_multiplier); - acc.v3 = vqrdmulhq_n_s32(acc.v3, output_multiplier); + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); - using gemmlowp::RoundingDivideByPOT; - acc.v0 = RoundingDivideByPOT(acc.v0, output_shift); - acc.v1 = RoundingDivideByPOT(acc.v1, output_shift); - acc.v2 = RoundingDivideByPOT(acc.v2, output_shift); - acc.v3 = RoundingDivideByPOT(acc.v3, output_shift); + DotProductAndStore( + filter, input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, output_ptr); - // Add the output offset. - acc.v0 = vaddq_s32(acc.v0, output_offset_vec); - acc.v1 = vaddq_s32(acc.v1, output_offset_vec); - acc.v2 = vaddq_s32(acc.v2, output_offset_vec); - acc.v3 = vaddq_s32(acc.v3, output_offset_vec); + // Second output. + output_ptr += output_row_size; - // Apply the activation function. - acc.v0 = vmaxq_s32(acc.v0, output_activation_min_vec); - acc.v1 = vmaxq_s32(acc.v1, output_activation_min_vec); - acc.v2 = vmaxq_s32(acc.v2, output_activation_min_vec); - acc.v3 = vmaxq_s32(acc.v3, output_activation_min_vec); + ptr += input_row_size; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + ptr += input_row_size; + temp_3 = vld1_u8(ptr); + temp_4 = vld1_u8(ptr + input_depth); + temp_5 = vld1_u8(ptr + 2 * input_depth); - acc.v0 = vminq_s32(acc.v0, output_activation_max_vec); - acc.v1 = vminq_s32(acc.v1, output_activation_max_vec); - acc.v2 = vminq_s32(acc.v2, output_activation_max_vec); - acc.v3 = vminq_s32(acc.v3, output_activation_max_vec); + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); - // Saturating cast to uint8 and store to destination. - int16x4_t acc_tlla_s16 = vqmovn_s32(acc.v0); - int16x4_t acc_tllb_s16 = vqmovn_s32(acc.v1); - int16x4_t acc_tlha_s16 = vqmovn_s32(acc.v2); - int16x4_t acc_tlhb_s16 = vqmovn_s32(acc.v3); - - int16x8_t res_s16_0 = vcombine_s16(acc_tlla_s16, acc_tllb_s16); - int16x8_t res_s16_1 = vcombine_s16(acc_tlha_s16, acc_tlhb_s16); - uint8x8_t res_u8_0 = vqmovun_s16(res_s16_0); - uint8x8_t res_u8_1 = vqmovun_s16(res_s16_1); - vst1q_u8(output_ptr, vcombine_u8(res_u8_0, res_u8_1)); + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + + DotProductAndStore( + filter, input_6, input_7, input_8, input_0, input_1, input_2, input_3, + input_4, input_5, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, output_ptr); + } +}; + +template <> +struct ConvKernel3x3FilterDepth8<1, 2, 2> { + static inline void Run(const uint8* input_ptr, int input_depth, + int32 input_offset, int input_row_size, + const uint8* filter_ptr, int32 filter_offset, + const int32* bias_ptr, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_ptr, + int output_depth, int output_width) { + Filter3x3x8 filter = Load3x3Filter(filter_ptr, filter_offset, output_depth); + + const int16x8_t input_offset_vec = vdupq_n_s16(input_offset); + int16x8_t input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8; + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5, temp_6, temp_7, + temp_8; + + const uint8* ptr = input_ptr; + + // Load all inputs for top output. + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + ptr += input_row_size; + temp_3 = vld1_u8(ptr); + temp_4 = vld1_u8(ptr + input_depth); + temp_5 = vld1_u8(ptr + 2 * input_depth); + ptr += input_row_size; + temp_6 = vld1_u8(ptr); + temp_7 = vld1_u8(ptr + input_depth); + temp_8 = vld1_u8(ptr + 2 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_6)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_7)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_8)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + + DotProductAndStore( + filter, input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, output_ptr); + + // Second output. + output_ptr += output_depth; + + ptr = input_ptr + 3 * input_depth; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + ptr += input_row_size; + temp_3 = vld1_u8(ptr); + temp_4 = vld1_u8(ptr + input_depth); + ptr += input_row_size; + temp_6 = vld1_u8(ptr); + temp_7 = vld1_u8(ptr + input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_6)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_7)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + + DotProductAndStore( + filter, input_2, input_0, input_1, input_5, input_3, input_4, input_8, + input_6, input_7, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, output_ptr); + } +}; + +template <> +struct ConvKernel3x3FilterDepth8<1, 4, 2> { + static inline void Run(const uint8* input_ptr, int input_depth, + int32 input_offset, int input_row_size, + const uint8* filter_ptr, int32 filter_offset, + const int32* bias_ptr, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_ptr, + int output_depth, int output_width) { + Filter3x3x8 filter = Load3x3Filter(filter_ptr, filter_offset, output_depth); + + const int16x8_t input_offset_vec = vdupq_n_s16(input_offset); + int16x8_t input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8; + uint8x8_t temp_0, temp_1, temp_2, temp_3, temp_4, temp_5, temp_6, temp_7, + temp_8; + + const uint8* ptr = input_ptr; + + // Load all inputs for top output. + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + temp_2 = vld1_u8(ptr + 2 * input_depth); + ptr += input_row_size; + temp_3 = vld1_u8(ptr); + temp_4 = vld1_u8(ptr + input_depth); + temp_5 = vld1_u8(ptr + 2 * input_depth); + ptr += input_row_size; + temp_6 = vld1_u8(ptr); + temp_7 = vld1_u8(ptr + input_depth); + temp_8 = vld1_u8(ptr + 2 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_6)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_7)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_8)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + + DotProductAndStore( + filter, input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, output_ptr); + + // Second output. + output_ptr += output_depth; + + ptr = input_ptr + 3 * input_depth; + temp_0 = vld1_u8(ptr); + temp_1 = vld1_u8(ptr + input_depth); + ptr += input_row_size; + temp_3 = vld1_u8(ptr); + temp_4 = vld1_u8(ptr + input_depth); + ptr += input_row_size; + temp_6 = vld1_u8(ptr); + temp_7 = vld1_u8(ptr + input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_6)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_7)); + + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + + DotProductAndStore( + filter, input_2, input_0, input_1, input_5, input_3, input_4, input_8, + input_6, input_7, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, output_ptr); + + // Third output. + output_ptr += output_depth; + + ptr = input_ptr + 5 * input_depth; + temp_2 = vld1_u8(ptr); + temp_0 = vld1_u8(ptr + input_depth); + ptr += input_row_size; + temp_5 = vld1_u8(ptr); + temp_3 = vld1_u8(ptr + input_depth); + ptr += input_row_size; + temp_8 = vld1_u8(ptr); + temp_6 = vld1_u8(ptr + input_depth); + + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_8)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_6)); + + input_2 = vaddq_s16(input_2, input_offset_vec); + input_0 = vaddq_s16(input_0, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + + DotProductAndStore( + filter, input_1, input_2, input_0, input_4, input_5, input_3, input_7, + input_8, input_6, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, output_ptr); + + // Fourth output. + output_ptr += output_depth; + + ptr = input_ptr + 7 * input_depth; + temp_1 = vld1_u8(ptr); + temp_2 = vld1_u8(ptr + input_depth); + ptr += input_row_size; + temp_4 = vld1_u8(ptr); + temp_5 = vld1_u8(ptr + input_depth); + ptr += input_row_size; + temp_7 = vld1_u8(ptr); + temp_8 = vld1_u8(ptr + input_depth); + + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_7)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_8)); + + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + + DotProductAndStore( + filter, input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, output_ptr); + } +}; + +template <> +struct ConvKernel3x3FilterDepth8<1, 1> { + static inline void Run(const uint8* input_ptr, int input_depth, + int32 input_offset, int input_row_size, + const uint8* filter_ptr, int32 filter_offset, + const int32* bias_ptr, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_ptr, + int output_depth, int output_width) { + Filter3x3x8 filter = Load3x3Filter(filter_ptr, filter_offset, output_depth); + + int16x8_t input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8; + + uint8x8_t temp_0 = vld1_u8(input_ptr); + uint8x8_t temp_1 = vld1_u8(input_ptr + input_depth); + uint8x8_t temp_2 = vld1_u8(input_ptr + 2 * input_depth); + + input_ptr += input_row_size; + uint8x8_t temp_3 = vld1_u8(input_ptr); + uint8x8_t temp_4 = vld1_u8(input_ptr + input_depth); + uint8x8_t temp_5 = vld1_u8(input_ptr + 2 * input_depth); + + input_ptr += input_row_size; + uint8x8_t temp_6 = vld1_u8(input_ptr); + uint8x8_t temp_7 = vld1_u8(input_ptr + input_depth); + uint8x8_t temp_8 = vld1_u8(input_ptr + 2 * input_depth); + + input_0 = vreinterpretq_s16_u16(vmovl_u8(temp_0)); + input_1 = vreinterpretq_s16_u16(vmovl_u8(temp_1)); + input_2 = vreinterpretq_s16_u16(vmovl_u8(temp_2)); + input_3 = vreinterpretq_s16_u16(vmovl_u8(temp_3)); + input_4 = vreinterpretq_s16_u16(vmovl_u8(temp_4)); + input_5 = vreinterpretq_s16_u16(vmovl_u8(temp_5)); + input_6 = vreinterpretq_s16_u16(vmovl_u8(temp_6)); + input_7 = vreinterpretq_s16_u16(vmovl_u8(temp_7)); + input_8 = vreinterpretq_s16_u16(vmovl_u8(temp_8)); + + const int16x8_t input_offset_vec = vdupq_n_s16(input_offset); + input_0 = vaddq_s16(input_0, input_offset_vec); + input_1 = vaddq_s16(input_1, input_offset_vec); + input_2 = vaddq_s16(input_2, input_offset_vec); + input_3 = vaddq_s16(input_3, input_offset_vec); + input_4 = vaddq_s16(input_4, input_offset_vec); + input_5 = vaddq_s16(input_5, input_offset_vec); + input_6 = vaddq_s16(input_6, input_offset_vec); + input_7 = vaddq_s16(input_7, input_offset_vec); + input_8 = vaddq_s16(input_8, input_offset_vec); + + DotProductAndStore( + filter, input_0, input_1, input_2, input_3, input_4, input_5, input_6, + input_7, input_8, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, output_ptr); + } +}; + +inline void ShuffleInput(const uint8* input_ptr, int input_depth, + int input_width, int input_height, int output_depth, + int output_width, int output_height, + uint8* output_ptr) { + const int input_row_size = input_depth * input_width; + + for (int y = 0; y < output_height; y++) { + const uint8* ptr = input_ptr; + for (int x = 0; x < output_width; x++) { + memcpy(output_ptr, ptr, output_depth); + output_ptr += output_depth; + ptr += input_depth; + } + input_ptr += input_row_size; + } } -// A kernel that is optimized on the number of output cells in the x and y -// direction, and the stride. Assumes 3x3 filters of 16 depth. -template -struct ConvKernel3x3FilterDepth16 {}; +template +struct ConvRow3x3FilterDepth8 {}; + +template +struct ConvRow3x3FilterDepth8<1, kFixedStrideWidth> { + static inline void Run(const uint8* input_data, int start_x, int start_y, + int input_depth, int input_width, int input_height, + int input_row_size, int32 input_offset, + const uint8* filter_data, int32 filter_offset, + const int32* bias_data, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_data, + int output_depth, int output_width, + uint8* shuffle_workspace) { + int out_x = start_x; + + // 1x4 at a time. + for (; out_x <= output_width - 4; out_x += 4) { + const int32* bias_ptr = bias_data; + const uint8* filter_ptr = filter_data; + + const uint8* input_ptr = input_data; + uint8* output_ptr = output_data; + + for (int depth = 0; depth <= output_depth - 8; depth += 8) { + ConvKernel3x3FilterDepth8<1, 4, kFixedStrideWidth>::Run( + input_ptr, input_depth, input_offset, input_row_size, filter_ptr, + filter_offset, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, + output_ptr, output_depth, output_width); + + input_ptr += 8; + output_ptr += 8; + filter_ptr += 8; + bias_ptr += 8; + } + + input_data += 4 * kFixedStrideWidth * input_depth; + output_data += 4 * output_depth; + } + + // 1x1 at a time. + for (; out_x < output_width; out_x++) { + const int32* bias_ptr = bias_data; + const uint8* filter_ptr = filter_data; + + const uint8* input_ptr = input_data; + uint8* output_ptr = output_data; + + for (int depth = 0; depth <= output_depth - 8; depth += 8) { + ConvKernel3x3FilterDepth8<1, 1>::Run( + input_ptr, input_depth, input_offset, input_row_size, filter_ptr, + filter_offset, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, + output_ptr, output_depth, output_width); + + input_ptr += 8; + output_ptr += 8; + filter_ptr += 8; + bias_ptr += 8; + } + + input_data += kFixedStrideWidth * input_depth; + output_data += output_depth; + } + } +}; + +template +struct ConvRow3x3FilterDepth8<2, kFixedStrideWidth> { + static inline void Run(const uint8* input_data, int start_x, int start_y, + int input_depth, int input_width, int input_height, + int input_row_size, int32 input_offset, + const uint8* filter_data, int32 filter_offset, + const int32* bias_data, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_data, + int output_depth, int output_width, + uint8* shuffle_workspace) { + int out_x = start_x; + + // 2x4 at a time. + for (; out_x <= output_width - 4; out_x += 4) { + const int32* bias_ptr = bias_data; + const uint8* filter_ptr = filter_data; + + const uint8* input_ptr = input_data; + uint8* output_ptr = output_data; + + for (int depth = 0; depth <= output_depth - 8; depth += 8) { + ConvKernel3x3FilterDepth8<2, 4, kFixedStrideWidth>::Run( + input_ptr, input_depth, input_offset, input_row_size, filter_ptr, + filter_offset, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, + output_ptr, output_depth, output_width); + + input_ptr += 8; + output_ptr += 8; + filter_ptr += 8; + bias_ptr += 8; + } + + input_data += 4 * kFixedStrideWidth * input_depth; + output_data += 4 * output_depth; + } + + // 2x2 at a time. + for (; out_x <= output_width - 2; out_x += 2) { + const int32* bias_ptr = bias_data; + const uint8* filter_ptr = filter_data; + + const uint8* input_ptr = input_data; + uint8* output_ptr = output_data; + + for (int depth = 0; depth <= output_depth - 8; depth += 8) { + ConvKernel3x3FilterDepth8<2, 2, kFixedStrideWidth>::Run( + input_ptr, input_depth, input_offset, input_row_size, filter_ptr, + filter_offset, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, + output_ptr, output_depth, output_width); + + input_ptr += 8; + output_ptr += 8; + filter_ptr += 8; + bias_ptr += 8; + } + + input_data += 2 * kFixedStrideWidth * input_depth; + output_data += 2 * output_depth; + } + + // 2x1 at a time. + for (; out_x < output_width; out_x++) { + const int32* bias_ptr = bias_data; + const uint8* filter_ptr = filter_data; + + const uint8* input_ptr = input_data; + uint8* output_ptr = output_data; + + for (int depth = 0; depth <= output_depth - 8; depth += 8) { + ConvKernel3x3FilterDepth8<2, 1, kFixedStrideWidth>::Run( + input_ptr, input_depth, input_offset, input_row_size, filter_ptr, + filter_offset, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, + output_ptr, output_depth, output_width); + + input_ptr += 8; + output_ptr += 8; + filter_ptr += 8; + bias_ptr += 8; + } + + input_data += kFixedStrideWidth * input_depth; + output_data += output_depth; + } + } +}; + +template <> +struct ConvRow3x3FilterDepth8<4, 1> { + static inline void Run(const uint8* input_data, int start_x, int start_y, + int input_depth, int input_width, int input_height, + int input_row_size, int32 input_offset, + const uint8* filter_data, int32 filter_offset, + const int32* bias_data, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_data, + int output_depth, int output_width, + uint8* shuffle_workspace) { + int out_x = start_x; + + // 4x4 at a time. + for (; out_x <= output_width - 4; out_x += 4) { + const int32* bias_ptr = bias_data; + const uint8* filter_ptr = filter_data; + + const uint8* input_ptr = input_data; + uint8* output_ptr = output_data; + + for (int depth = 0; depth <= output_depth - 8; depth += 8) { + ConvKernel3x3FilterDepth8<4, 4, 1>::Run( + input_ptr, input_depth, input_offset, input_row_size, filter_ptr, + filter_offset, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, + output_ptr, output_depth, output_width); + + input_ptr += 8; + output_ptr += 8; + filter_ptr += 8; + bias_ptr += 8; + } + + input_data += 4 * input_depth; + output_data += 4 * output_depth; + } + + // Handle the rest of the right side. + // 4x2 at a time. + for (; out_x <= output_width - 2; out_x += 2) { + const int32* bias_ptr = bias_data; + const uint8* filter_ptr = filter_data; + + const uint8* input_ptr = input_data; + uint8* output_ptr = output_data; + + for (int depth = 0; depth <= output_depth - 8; depth += 8) { + ConvKernel3x3FilterDepth8<4, 2, 1>::Run( + input_ptr, input_depth, input_offset, input_row_size, filter_ptr, + filter_offset, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, + output_ptr, output_depth, output_width); + + input_ptr += 8; + output_ptr += 8; + filter_ptr += 8; + bias_ptr += 8; + } + + input_data += 2 * input_depth; + output_data += 2 * output_depth; + } + + // 4x1 at a time. + for (; out_x < output_width; out_x++) { + const int32* bias_ptr = bias_data; + const uint8* filter_ptr = filter_data; + + const uint8* input_ptr = input_data; + uint8* output_ptr = output_data; + + for (int depth = 0; depth <= output_depth - 8; depth += 8) { + ConvKernel3x3FilterDepth8<4, 1, 1>::Run( + input_ptr, input_depth, input_offset, input_row_size, filter_ptr, + filter_offset, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, + output_ptr, output_depth, output_width); + + input_ptr += 8; + output_ptr += 8; + filter_ptr += 8; + bias_ptr += 8; + } + + input_data += input_depth; + output_data += output_depth; + } + } +}; template <> -struct ConvKernel3x3FilterDepth16<1, 2, 1> { - static void Run(const Filter3x3x16& filter, const uint8* input_ptr, - int input_depth, int32 input_offset, int input_row_width, - const int32* bias_ptr, int32 output_offset, - int32 output_multiplier, int output_shift, - int32 output_activation_min, int32 output_activation_max, - uint8* output_ptr, int output_depth, int output_width) { - // 16 depth accumulators for the 2 outputs. - Int32x16 acc0, acc1; - - // Accumulators for top filter. - acc0.v0 = vld1q_s32(bias_ptr); - acc0.v1 = vld1q_s32(bias_ptr + 4); - acc0.v2 = vld1q_s32(bias_ptr + 8); - acc0.v3 = vld1q_s32(bias_ptr + 12); - // Accumulators for bottom filter. - acc1.v0 = vld1q_s32(bias_ptr); - acc1.v1 = vld1q_s32(bias_ptr + 4); - acc1.v2 = vld1q_s32(bias_ptr + 8); - acc1.v3 = vld1q_s32(bias_ptr + 12); - - // Main multiply accumulate work. - { - // Load inputs for one filter row at a time. - Int16x16x3 input; - - // Do first row of top filter. - input = LoadInputRowDepth16(input_ptr, input_depth, input_offset, input); - acc0 = MultiplyAccumulateRowDepth16(acc0, filter.r0, input); - - // Do second row of top filter. - input = LoadInputRowDepth16(input_ptr + input_row_width, input_depth, - input_offset, input); - acc0 = MultiplyAccumulateRowDepth16(acc0, filter.r1, input); - - // The inputs to second row of the top filter are also the inputs to the - // first row of the bottom filter. - acc1 = MultiplyAccumulateRowDepth16(acc1, filter.r0, input); - - // Do third row of top filter. - input = LoadInputRowDepth16(input_ptr + 2 * input_row_width, input_depth, - input_offset, input); - acc0 = MultiplyAccumulateRowDepth16(acc0, filter.r2, input); - - // The inputs to third row of the top filter are also the inputs to the - // second row of the bottom filter. - acc1 = MultiplyAccumulateRowDepth16(acc1, filter.r1, input); - - // Do third row of bottom filter. - input = LoadInputRowDepth16(input_ptr + 3 * input_row_width, input_depth, - input_offset, input); - acc1 = MultiplyAccumulateRowDepth16(acc1, filter.r2, input); - } - - // Apply activation, downquantize and store. - int32x4_t output_offset_vec = vdupq_n_s32(output_offset); - int32x4_t output_activation_min_vec = vdupq_n_s32(output_activation_min); - int32x4_t output_activation_max_vec = vdupq_n_s32(output_activation_max); - - DownquantizeAndStoreDepth16(acc0, output_multiplier, output_shift, - output_offset_vec, output_activation_min_vec, - output_activation_max_vec, output_ptr); - - DownquantizeAndStoreDepth16(acc1, output_multiplier, output_shift, - output_offset_vec, output_activation_min_vec, - output_activation_max_vec, - output_ptr + output_depth * output_width); +struct ConvRow3x3FilterDepth8<4, 2> { + // The buffer size of the shuffled input. + static inline constexpr int ShuffleWorkspaceSize() { return 64 * 9 * 9; } + + static inline void Run(const uint8* input_data, int start_x, int start_y, + int input_depth, int input_width, int input_height, + int input_row_size, int32 input_offset, + const uint8* filter_data, int32 filter_offset, + const int32* bias_data, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_data, + int output_depth, int output_width, + uint8* shuffle_workspace) { + // Branch and cache misses increase substantially with stride 2 kernels. + // Adding prefetching reduces latency by as much as 2x. + const int i0 = 0; + const int i1 = input_depth; + const int i2 = 2 * input_depth; + const int i3 = 3 * input_depth; + const int i4 = 4 * input_depth; + const int i5 = 5 * input_depth; + const int i6 = 6 * input_depth; + const int i7 = 7 * input_depth; + const int i8 = 8 * input_depth; + +#define DEPTHWISECONV_PRELOAD_ROW(input_ptr, i) \ + preload_l1_keep(input_ptr + i * input_row_size + i0); \ + preload_l1_keep(input_ptr + i * input_row_size + i1); \ + preload_l1_keep(input_ptr + i * input_row_size + i2); \ + preload_l1_keep(input_ptr + i * input_row_size + i3); \ + preload_l1_keep(input_ptr + i * input_row_size + i4); \ + preload_l1_keep(input_ptr + i * input_row_size + i5); \ + preload_l1_keep(input_ptr + i * input_row_size + i6); \ + preload_l1_keep(input_ptr + i * input_row_size + i7); \ + preload_l1_keep(input_ptr + i * input_row_size + i8); + + int out_x = start_x; + // 4x4 at a time. + for (; out_x <= output_width - 4; out_x += 4) { + const int32* bias_ptr = bias_data; + const uint8* filter_ptr = filter_data; + + const uint8* input_ptr = input_data; + uint8* output_ptr = output_data; + + int depth = 0; + for (; depth <= output_depth - 64; depth += 64) { + // Preload 9x9 input. + DEPTHWISECONV_PRELOAD_ROW(input_ptr, 0); + DEPTHWISECONV_PRELOAD_ROW(input_ptr, 1); + DEPTHWISECONV_PRELOAD_ROW(input_ptr, 2); + DEPTHWISECONV_PRELOAD_ROW(input_ptr, 3); + DEPTHWISECONV_PRELOAD_ROW(input_ptr, 4); + DEPTHWISECONV_PRELOAD_ROW(input_ptr, 5); + DEPTHWISECONV_PRELOAD_ROW(input_ptr, 6); + DEPTHWISECONV_PRELOAD_ROW(input_ptr, 7); + DEPTHWISECONV_PRELOAD_ROW(input_ptr, 8); + + // For a large input window (64x9x9) that is small enough to fit in L1 + // cache, copy the input into a separate buffer and run the kernel on + // this new buffer. This reduces the likelihood of cache misses when + // the kernel is loading input data. If this size is ever changed, + // update the ShuffleWorkspaceSize() function to return the new size. + ShuffleInput(input_ptr, input_depth, input_width, input_height, 64, 9, + 9, shuffle_workspace); + const uint8* shuffled_ptr = &shuffle_workspace[0]; + + for (int micro_depth = 0; micro_depth <= 64 - 8; micro_depth += 8) { + ConvKernel3x3FilterDepth8<4, 4, 2>::Run( + shuffled_ptr, 64, input_offset, 64 * 9, filter_ptr, filter_offset, + bias_ptr, output_offset, output_multiplier, output_shift, + output_activation_min, output_activation_max, output_ptr, + output_depth, output_width); + + shuffled_ptr += 8; + output_ptr += 8; + filter_ptr += 8; + bias_ptr += 8; + } + input_ptr += 64; + } + + // Preload 9x9 input one more time for the rest of the depth. + DEPTHWISECONV_PRELOAD_ROW(input_ptr, 0); + DEPTHWISECONV_PRELOAD_ROW(input_ptr, 1); + DEPTHWISECONV_PRELOAD_ROW(input_ptr, 2); + DEPTHWISECONV_PRELOAD_ROW(input_ptr, 3); + DEPTHWISECONV_PRELOAD_ROW(input_ptr, 4); + DEPTHWISECONV_PRELOAD_ROW(input_ptr, 5); + DEPTHWISECONV_PRELOAD_ROW(input_ptr, 6); + DEPTHWISECONV_PRELOAD_ROW(input_ptr, 7); + DEPTHWISECONV_PRELOAD_ROW(input_ptr, 8); + + for (; depth <= output_depth - 8; depth += 8) { + ConvKernel3x3FilterDepth8<4, 4, 2>::Run( + input_ptr, input_depth, input_offset, input_row_size, filter_ptr, + filter_offset, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, + output_ptr, output_depth, output_width); + + input_ptr += 8; + output_ptr += 8; + filter_ptr += 8; + bias_ptr += 8; + } + + input_data += 4 * 2 * input_depth; + output_data += 4 * output_depth; + } + +#undef DEPTHWISECONV_PRELOAD_ROW + + // Handle the rest of the right side. + // 4x2 at a time. + for (; out_x <= output_width - 2; out_x += 2) { + const int32* bias_ptr = bias_data; + const uint8* filter_ptr = filter_data; + + const uint8* input_ptr = input_data; + uint8* output_ptr = output_data; + + for (int depth = 0; depth <= output_depth - 8; depth += 8) { + ConvKernel3x3FilterDepth8<4, 2, 2>::Run( + input_ptr, input_depth, input_offset, input_row_size, filter_ptr, + filter_offset, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, + output_ptr, output_depth, output_width); + + input_ptr += 8; + output_ptr += 8; + filter_ptr += 8; + bias_ptr += 8; + } + + input_data += 2 * 2 * input_depth; + output_data += 2 * output_depth; + } + + // 4x1 at a time. + for (; out_x < output_width; out_x++) { + const int32* bias_ptr = bias_data; + const uint8* filter_ptr = filter_data; + + const uint8* input_ptr = input_data; + uint8* output_ptr = output_data; + + for (int depth = 0; depth <= output_depth - 8; depth += 8) { + ConvKernel3x3FilterDepth8<4, 1, 2>::Run( + input_ptr, input_depth, input_offset, input_row_size, filter_ptr, + filter_offset, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, + output_ptr, output_depth, output_width); + + input_ptr += 8; + output_ptr += 8; + filter_ptr += 8; + bias_ptr += 8; + } + + input_data += 2 * input_depth; + output_data += output_depth; + } } }; template <> -struct ConvKernel3x3FilterDepth16<1, 2, 2> { - static void Run(const Filter3x3x16& filter, const uint8* input_ptr, - int input_depth, int32 input_offset, int input_row_width, - const int32* bias_ptr, int32 output_offset, - int32 output_multiplier, int output_shift, - int32 output_activation_min, int32 output_activation_max, - uint8* output_ptr, int output_depth, int output_width) { - // 16 depth accumulators for the 2 outputs. - Int32x16 acc0, acc1; - - // Accumulators for top filter. - acc0.v0 = vld1q_s32(bias_ptr); - acc0.v1 = vld1q_s32(bias_ptr + 4); - acc0.v2 = vld1q_s32(bias_ptr + 8); - acc0.v3 = vld1q_s32(bias_ptr + 12); - // Accumulators for bottom filter. - acc1.v0 = vld1q_s32(bias_ptr); - acc1.v1 = vld1q_s32(bias_ptr + 4); - acc1.v2 = vld1q_s32(bias_ptr + 8); - acc1.v3 = vld1q_s32(bias_ptr + 12); - - // Main multiply accumulate work. - { - // Load inputs for one filter row at a time. - Int16x16x3 input; - - // Do first row of top filter. - input = LoadInputRowDepth16(input_ptr, input_depth, input_offset, input); - acc0 = MultiplyAccumulateRowDepth16(acc0, filter.r0, input); - - // Do second row of top filter. - input = LoadInputRowDepth16(input_ptr + input_row_width, input_depth, - input_offset, input); - acc0 = MultiplyAccumulateRowDepth16(acc0, filter.r1, input); - - // Do third row of top filter. - input = LoadInputRowDepth16(input_ptr + 2 * input_row_width, input_depth, - input_offset, input); - acc0 = MultiplyAccumulateRowDepth16(acc0, filter.r2, input); - - // The inputs to third row of the top filter are also the inputs - // to first row of the bottom filter. - acc1 = MultiplyAccumulateRowDepth16(acc1, filter.r0, input); - - // Do second row of bottom filter. - input = LoadInputRowDepth16(input_ptr + 3 * input_row_width, input_depth, - input_offset, input); - acc1 = MultiplyAccumulateRowDepth16(acc1, filter.r1, input); - - // Do third row of bottom filter. - input = LoadInputRowDepth16(input_ptr + 4 * input_row_width, input_depth, - input_offset, input); - acc1 = MultiplyAccumulateRowDepth16(acc1, filter.r2, input); - } - - // Apply activation, downquantize and store. - int32x4_t output_offset_vec = vdupq_n_s32(output_offset); - int32x4_t output_activation_min_vec = vdupq_n_s32(output_activation_min); - int32x4_t output_activation_max_vec = vdupq_n_s32(output_activation_max); - - DownquantizeAndStoreDepth16(acc0, output_multiplier, output_shift, - output_offset_vec, output_activation_min_vec, - output_activation_max_vec, output_ptr); - - DownquantizeAndStoreDepth16(acc1, output_multiplier, output_shift, - output_offset_vec, output_activation_min_vec, - output_activation_max_vec, - output_ptr + output_depth * output_width); +struct ConvRow3x3FilterDepth8<8, 2> { + static inline void Run(const uint8* input_data, int start_x, int start_y, + int input_depth, int input_width, int input_height, + int input_row_size, int32 input_offset, + const uint8* filter_data, int32 filter_offset, + const int32* bias_data, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_data, + int output_depth, int output_width, + uint8* shuffle_workspace) { + // Reuse 4 row kernels twice. + ConvRow3x3FilterDepth8<4, 2>::Run( + input_data, start_x, start_y, input_depth, input_width, input_height, + input_row_size, input_offset, filter_data, filter_offset, bias_data, + output_offset, output_multiplier, output_shift, output_activation_min, + output_activation_max, output_data, output_depth, output_width, + shuffle_workspace); + + ConvRow3x3FilterDepth8<4, 2>::Run( + input_data + 2 * 4 * input_row_size, start_x, start_y + 4, input_depth, + input_width, input_height, input_row_size, input_offset, filter_data, + filter_offset, bias_data, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, + output_data + 4 * output_depth * output_width, output_depth, + output_width, shuffle_workspace); } }; template <> -struct ConvKernel3x3FilterDepth16<1, 1> { - static void Run(const Filter3x3x16& filter, const uint8* input_ptr, - int input_depth, int32 input_offset, int input_row_width, - const int32* bias_ptr, int32 output_offset, - int32 output_multiplier, int output_shift, - int32 output_activation_min, int32 output_activation_max, - uint8* output_ptr, int output_depth, int output_width) { - Int32x16 acc; - acc.v0 = vld1q_s32(bias_ptr); - acc.v1 = vld1q_s32(bias_ptr + 4); - acc.v2 = vld1q_s32(bias_ptr + 8); - acc.v3 = vld1q_s32(bias_ptr + 12); - - // Main multiply accumulate work. - { - // Load inputs for one filter row at a time. - Int16x16x3 input; - - // Do first row. - input = LoadInputRowDepth16(input_ptr, input_depth, input_offset, input); - acc = MultiplyAccumulateRowDepth16(acc, filter.r0, input); - - // Do second row. - input = LoadInputRowDepth16(input_ptr + input_row_width, input_depth, - input_offset, input); - acc = MultiplyAccumulateRowDepth16(acc, filter.r1, input); - - // Do third row. - input = LoadInputRowDepth16(input_ptr + 2 * input_row_width, input_depth, - input_offset, input); - acc = MultiplyAccumulateRowDepth16(acc, filter.r2, input); - } - - // Apply activation, downquantize and store. - int32x4_t output_offset_vec = vdupq_n_s32(output_offset); - int32x4_t output_activation_min_vec = vdupq_n_s32(output_activation_min); - int32x4_t output_activation_max_vec = vdupq_n_s32(output_activation_max); - - DownquantizeAndStoreDepth16(acc, output_multiplier, output_shift, - output_offset_vec, output_activation_min_vec, - output_activation_max_vec, output_ptr); +struct ConvRow3x3FilterDepth8<8, 1> { + // The buffer size of the shuffled input. + static inline constexpr int ShuffleWorkspaceSize() { return 64 * 10 * 10; } + + static inline void Run(const uint8* input_data, int start_x, int start_y, + int input_depth, int input_width, int input_height, + int input_row_size, int32 input_offset, + const uint8* filter_data, int32 filter_offset, + const int32* bias_data, int32 output_offset, + int32 output_multiplier, int output_shift, + int32 output_activation_min, + int32 output_activation_max, uint8* output_data, + int output_depth, int output_width, + uint8* shuffle_workspace) { + int out_x = start_x; + // 8x8 at a time. + for (; out_x <= output_width - 8; out_x += 8) { + const int32* bias_ptr = bias_data; + const uint8* filter_ptr = filter_data; + + const uint8* input_ptr = input_data; + uint8* output_ptr = output_data; + + int depth = 0; + for (; depth <= output_depth - 64; depth += 64) { + // For a large input window (64x10x10) that is small enough to fit in L1 + // cache, copy the input into a separate buffer and run the kernel on + // this new buffer. This reduces the likelihood of cache misses when + // the kernel is loading input data. If the size of the input window + // changes, update the function ShuffleWorkspaceSize() with the new + // size. + ShuffleInput(input_ptr, input_depth, input_width, input_height, 64, 10, + 10, shuffle_workspace); + const uint8* shuffled_ptr = shuffle_workspace; + + for (int micro_depth = 0; micro_depth <= 64 - 8; micro_depth += 8) { + ConvKernel3x3FilterDepth8<8, 8, 1>::Run( + shuffled_ptr, 64, input_offset, 64 * 10, filter_ptr, + filter_offset, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, + output_ptr, output_depth, output_width); + + shuffled_ptr += 8; + output_ptr += 8; + filter_ptr += 8; + bias_ptr += 8; + } + input_ptr += 64; + } + + for (; depth <= output_depth - 8; depth += 8) { + ConvKernel3x3FilterDepth8<8, 8, 1>::Run( + input_ptr, input_depth, input_offset, input_row_size, filter_ptr, + filter_offset, bias_ptr, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, + output_ptr, output_depth, output_width); + + input_ptr += 8; + output_ptr += 8; + filter_ptr += 8; + bias_ptr += 8; + } + + input_data += 8 * input_depth; + output_data += 8 * output_depth; + } + + // Handle the rest of the right side by re-using 4 row kernels twice. + ConvRow3x3FilterDepth8<4, 1>::Run( + input_data, out_x, start_y, input_depth, input_width, input_height, + input_row_size, input_offset, filter_data, filter_offset, bias_data, + output_offset, output_multiplier, output_shift, output_activation_min, + output_activation_max, output_data, output_depth, output_width, + shuffle_workspace); + + ConvRow3x3FilterDepth8<4, 1>::Run( + input_data + 4 * input_row_size, out_x, start_y + 4, input_depth, + input_width, input_height, input_row_size, input_offset, filter_data, + filter_offset, bias_data, output_offset, output_multiplier, + output_shift, output_activation_min, output_activation_max, + output_data + 4 * output_depth * output_width, output_depth, + output_width, shuffle_workspace); } }; -inline bool Fast3by3FilterKernelSupported(const Dims<4>& input_dims, - const Dims<4>& filter_dims, - int stride_width, int stride_height, - int pad_width, int pad_height, - int depth_multiplier, - const Dims<4>& output_dims) { +inline bool Fast3x3FilterKernelSupported(const Dims<4>& input_dims, + const Dims<4>& filter_dims, + int stride_width, int stride_height, + int pad_width, int pad_height, + int depth_multiplier, + const Dims<4>& output_dims) { const int input_height = ArraySize(input_dims, 2); const int input_width = ArraySize(input_dims, 1); const int input_depth = ArraySize(input_dims, 0); @@ -458,14 +4426,14 @@ inline bool Fast3by3FilterKernelSupported(const Dims<4>& input_dims, depth_multiplier == 1 && (stride_width == 1 || stride_width == 2) && (stride_height == 1 || stride_height == 2) && - pad_width == 0 && pad_height == 0 && (input_depth % 16) == 0; + pad_width == 0 && pad_height == 0 && (input_depth % 8) == 0; if (!supported) { return false; } - // Handle case where padding is zero but type is not kValid. This would - // require special boundary case handling that is not supported yet. + // Handle case where padding is zero but padding type is not kValid. + // This would require special boundary case handling that is not supported. const int out_x = output_width - 1; const int out_y = output_height - 1; @@ -481,7 +4449,7 @@ inline bool Fast3by3FilterKernelSupported(const Dims<4>& input_dims, return in_x_end <= input_width && in_y_end <= input_height; } -inline void DepthwiseConv3by3FilterDepth16( +inline void DepthwiseConv3x3Filter( const uint8* input_data, const Dims<4>& input_dims, int32 input_offset, const uint8* filter_data, const Dims<4>& filter_dims, int32 filter_offset, const int32* bias_data, const Dims<4>& bias_dims, int stride_width, @@ -500,241 +4468,108 @@ inline void DepthwiseConv3by3FilterDepth16( const int output_width = ArraySize(output_dims, 1); // Algorithm assumes below constraints. It is optimized for depth multiplier - // of 1, 3x3 filter, no padding, strides 1 and 2. + // of 1, 3x3 filter, no padding and strides 1 and 2. TFLITE_DCHECK(output_depth == input_depth * depth_multiplier); TFLITE_DCHECK(depth_multiplier == 1); TFLITE_DCHECK(filter_height == 3); TFLITE_DCHECK(filter_width == 3); TFLITE_DCHECK(pad_height == 0); TFLITE_DCHECK(pad_width == 0); - TFLITE_DCHECK(stride_width == 1 || stride_width == 2); TFLITE_DCHECK(stride_height == 1 || stride_height == 2); + TFLITE_DCHECK(stride_width == 1 || stride_width == 2); - // The number of outputs to process in the main loop. - const int num_x_outputs = 1; - const int num_y_outputs = 2; - - const int input_row_width = output_depth * (input_width + 2 * pad_width); - const int input_batch_size = - input_row_width * (input_height + 2 * pad_height); + const int input_row_size = input_depth * (input_width + 2 * pad_width); + const int output_row_size = output_depth * output_width; + const int input_batch_size = input_row_size * (input_height + 2 * pad_height); const int output_batch_size = output_depth * output_width * output_height; - const int input_ptr_x_increment = input_depth * stride_width; - // Calculate extents of non-boundary loop. - int out_x_start = 0; - for (; out_x_start < input_width; out_x_start++) { - int in_x = (out_x_start * stride_width) - pad_width; - if (in_x >= 0) { - break; - } - } - int out_x_end = output_width - 1; - for (; out_x_end >= 0; out_x_end--) { - int in_x = (out_x_end * stride_width) - pad_width; - int in_x_end = in_x + filter_width + (num_x_outputs - 1) * stride_width; - if (in_x_end <= input_width) { - out_x_end++; - break; - } - } - int out_y_start = 0; - for (; out_y_start < input_height; out_y_start++) { - int in_y = (out_y_start * stride_height) - pad_height; - if (in_y >= 0) { - break; - } - } - int out_y_end = output_height - 1; - for (; out_y_end >= 0; out_y_end--) { - int in_y = (out_y_end * stride_height) - pad_height; - int in_y_end = in_y + filter_height + (num_y_outputs - 1) * stride_height; - if (in_y_end <= input_height) { - out_y_end++; - break; - } + using conv_row_func_t = decltype(&ConvRow3x3FilterDepth8<1, 1>::Run); + conv_row_func_t conv_1_output_row = ConvRow3x3FilterDepth8<1, 1>::Run; + conv_row_func_t conv_2_output_rows = ConvRow3x3FilterDepth8<2, 1>::Run; + conv_row_func_t conv_4_output_rows = ConvRow3x3FilterDepth8<4, 1>::Run; + conv_row_func_t conv_8_output_rows = ConvRow3x3FilterDepth8<8, 1>::Run; + + if (stride_width == 2) { + conv_1_output_row = ConvRow3x3FilterDepth8<1, 2>::Run; + conv_2_output_rows = ConvRow3x3FilterDepth8<2, 2>::Run; + conv_4_output_rows = ConvRow3x3FilterDepth8<4, 2>::Run; + conv_8_output_rows = ConvRow3x3FilterDepth8<8, 2>::Run; } - using dot_product_func_t = - decltype(&ConvKernel3x3FilterDepth16<1, 2, 1>::Run); - dot_product_func_t dot_product_func = nullptr; + // Allocate maximum memory needed for shuffled input. + // TODO(mariewhite): The size of this workspace is small enough to be + // allocated on the stack. Eventually we will want to move it to the heap + // and have it allocated outside of this function, like the im2col_array used + // in gemmlowp. +#define DEPTHWISECONV_SHUFFLE_WORKSPACE_SIZE 10 * 10 * 64 + uint8 shuffle_workspace[DEPTHWISECONV_SHUFFLE_WORKSPACE_SIZE]; - if (stride_width == 1 && stride_height == 1) { - dot_product_func = ConvKernel3x3FilterDepth16<1, 2, 1>::Run; - } else { - dot_product_func = ConvKernel3x3FilterDepth16<1, 2, 2>::Run; - } + // Make sure the kernels using this buffer will not run out of bounds. + static_assert(ConvRow3x3FilterDepth8<8, 1>::ShuffleWorkspaceSize() <= + DEPTHWISECONV_SHUFFLE_WORKSPACE_SIZE, + "Shuffle workspace size is too small."); + static_assert(ConvRow3x3FilterDepth8<4, 2>::ShuffleWorkspaceSize() <= + DEPTHWISECONV_SHUFFLE_WORKSPACE_SIZE, + "Shuffle workspace size is too small."); - // Offsets for preloading inputs. - const int i0 = 0; - const int i1 = input_depth; - const int i2 = 2 * input_depth; - const int i3 = input_row_width; - const int i4 = input_row_width + input_depth; - const int i5 = input_row_width + 2 * input_depth; - const int i6 = 2 * input_row_width; - const int i7 = 2 * input_row_width + input_depth; - const int i8 = 2 * input_row_width + 2 * input_depth; - const int i9 = 3 * input_row_width; - const int i10 = 3 * input_row_width + input_depth; - const int i11 = 3 * input_row_width + 2 * input_depth; - const int i12 = 4 * input_row_width; - const int i13 = 4 * input_row_width + input_depth; - const int i14 = 4 * input_row_width + 2 * input_depth; +#undef DEPTHWISECONV_SHUFFLE_WORKSPACE_SIZE for (int b = 0; b < batches; ++b) { - const int32* bias_ptr = bias_data; - const uint8* filter_ptr = filter_data; - - const int in_batch_offset = b * input_batch_size; - const int out_batch_offset = b * output_batch_size; - - int depth = 0; - for (; depth <= output_depth - 16; depth += 16) { - Filter3x3x16 filter = - LoadFilterDepth16(filter_ptr, filter_offset, output_depth); - - // Handle 1x2 outputs. - int out_y = out_y_start; - for (; out_y < out_y_end; out_y += num_y_outputs) { - int out_x = out_x_start; - - int in_y_offset = - stride_height * input_row_width * (out_y + pad_height); - int in_x_offset = stride_width * input_depth * (out_x + pad_width); - - const uint8* input_ptr = - input_data + depth + in_x_offset + in_y_offset + in_batch_offset; - - // Preload inputs. If input depth is large, preload every value of the - // input for this depth range. Otherwise, preload only the first values - // of each row. - if (input_depth >= 32) { - preload_l1_keep(input_ptr + i0); - preload_l1_keep(input_ptr + i1); - preload_l1_keep(input_ptr + i2); - preload_l1_keep(input_ptr + i3); - preload_l1_keep(input_ptr + i4); - preload_l1_keep(input_ptr + i5); - preload_l1_keep(input_ptr + i6); - preload_l1_keep(input_ptr + i7); - preload_l1_keep(input_ptr + i8); - preload_l1_keep(input_ptr + i9); - preload_l1_keep(input_ptr + i10); - preload_l1_keep(input_ptr + i11); - - if (stride_height == 2) { - preload_l1_keep(input_ptr + i12); - preload_l1_keep(input_ptr + i13); - preload_l1_keep(input_ptr + i14); - } - } else { - preload_l1_keep(input_ptr + i0); - preload_l1_keep(input_ptr + i3); - preload_l1_keep(input_ptr + i6); - preload_l1_keep(input_ptr + i9); - - if (stride_height == 2) { - preload_l1_keep(input_ptr + i12); - } - } + const uint8* input_ptr = input_data + b * input_batch_size; + uint8* output_ptr = output_data + b * output_batch_size; - uint8* output_ptr = output_data + depth + (out_x * output_depth) + - (output_depth * output_width * out_y) + - out_batch_offset; - - for (; out_x < out_x_end; out_x += num_x_outputs) { - dot_product_func(filter, input_ptr, input_depth, input_offset, - input_row_width, bias_ptr, output_offset, - output_multiplier, output_shift, - output_activation_min, output_activation_max, - output_ptr, output_depth, output_width); - - input_ptr += input_ptr_x_increment * num_x_outputs; - output_ptr += output_depth * num_x_outputs; - - // Preload the next inputs depending on stride. - if (stride_width == 1) { - preload_l1_keep(input_ptr + i2); - preload_l1_keep(input_ptr + i5); - preload_l1_keep(input_ptr + i8); - preload_l1_keep(input_ptr + i11); - } else if (stride_width == 2) { - preload_l1_keep(input_ptr + i1); - preload_l1_keep(input_ptr + i2); - preload_l1_keep(input_ptr + i4); - preload_l1_keep(input_ptr + i5); - preload_l1_keep(input_ptr + i7); - preload_l1_keep(input_ptr + i8); - preload_l1_keep(input_ptr + i10); - preload_l1_keep(input_ptr + i11); - preload_l1_keep(input_ptr + i13); - preload_l1_keep(input_ptr + i14); - } - } + int out_y = 0; - // Handle the rest of the right side. - for (; out_x < output_width; out_x++) { - // This code path can only be reached if we're handling >1 x outputs - // at a time or support kSame padding. - } - } + // Handle 8 rows at a time. + for (; out_y <= output_height - 8; out_y += 8) { + conv_8_output_rows(input_ptr, 0, out_y, input_depth, input_width, + input_height, input_row_size, input_offset, + filter_data, filter_offset, bias_data, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr, output_depth, + output_width, shuffle_workspace); - // Handle the rest of the bottom side. - for (; out_y < output_height; out_y++) { - int out_x = out_x_start; - - int in_y_offset = - stride_height * input_row_width * (out_y + pad_height); - int in_x_offset = stride_width * input_depth * (out_x + pad_width); - - const uint8* input_ptr = - input_data + depth + in_x_offset + in_y_offset + in_batch_offset; - - if (input_depth >= 32) { - preload_l1_keep(input_ptr + i0); - preload_l1_keep(input_ptr + i1); - preload_l1_keep(input_ptr + i2); - preload_l1_keep(input_ptr + i3); - preload_l1_keep(input_ptr + i4); - preload_l1_keep(input_ptr + i5); - preload_l1_keep(input_ptr + i6); - preload_l1_keep(input_ptr + i7); - } else { - preload_l1_keep(input_ptr + i0); - preload_l1_keep(input_ptr + i3); - preload_l1_keep(input_ptr + i6); - } + input_ptr += 8 * stride_height * input_row_size; + output_ptr += 8 * output_row_size; + } - uint8* output_ptr = output_data + depth + (out_x * output_depth) + - (output_depth * output_width * out_y) + - out_batch_offset; + // Handle 4 rows at a time. + for (; out_y <= output_height - 4; out_y += 4) { + conv_4_output_rows(input_ptr, 0, out_y, input_depth, input_width, + input_height, input_row_size, input_offset, + filter_data, filter_offset, bias_data, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr, output_depth, + output_width, shuffle_workspace); - for (; out_x < output_width; out_x++) { - ConvKernel3x3FilterDepth16<1, 1>::Run( - filter, input_ptr, input_depth, input_offset, input_row_width, - bias_ptr, output_offset, output_multiplier, output_shift, - output_activation_min, output_activation_max, output_ptr, - output_depth, output_width); + input_ptr += 4 * stride_height * input_row_size; + output_ptr += 4 * output_row_size; + } - input_ptr += input_ptr_x_increment; - output_ptr += output_depth; - - if (stride_width == 1) { - preload_l1_keep(input_ptr + i2); - preload_l1_keep(input_ptr + i5); - preload_l1_keep(input_ptr + i8); - } else if (stride_width == 2) { - preload_l1_keep(input_ptr + i1); - preload_l1_keep(input_ptr + i2); - preload_l1_keep(input_ptr + i4); - preload_l1_keep(input_ptr + i5); - preload_l1_keep(input_ptr + i7); - preload_l1_keep(input_ptr + i8); - } - } - } - filter_ptr += 16; - bias_ptr += 16; + // Handle 2 rows at a time. + for (; out_y <= output_height - 2; out_y += 2) { + conv_2_output_rows(input_ptr, 0, out_y, input_depth, input_width, + input_height, input_row_size, input_offset, + filter_data, filter_offset, bias_data, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr, output_depth, + output_width, shuffle_workspace); + + input_ptr += 2 * stride_height * input_row_size; + output_ptr += 2 * output_row_size; + } + + // Handle one row at a time. + for (; out_y < output_height; out_y++) { + conv_1_output_row(input_ptr, 0, out_y, input_depth, input_width, + input_height, input_row_size, input_offset, filter_data, + filter_offset, bias_data, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr, output_depth, + output_width, shuffle_workspace); + + input_ptr += stride_height * input_row_size; + output_ptr += output_row_size; } } } -- GitLab From 317cc081d620c27df464e19aea624a1e89e30fd8 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 11 Apr 2018 13:35:51 -0700 Subject: [PATCH 178/791] Update tf.contrib.metrics with deprecations (#18335) * Update tf.contrib.metrics with deprecations This fix updates tf.contrib.metrics.streaming_mean_absolution(relative/squared)_error with deprecation notices as they have been replaces with tf.metrics. Signed-off-by: Yong Tang * Update streaming_mean_relative_error Signed-off-by: Yong Tang * Update deprecation notice for streaming_root_mean_squared_error Signed-off-by: Yong Tang * Fix the deprecation message. * Fix pylint `Line too long (81/80)` issue Signed-off-by: Yong Tang --- tensorflow/contrib/metrics/python/ops/metric_ops.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py index 088319a557..2bf281b791 100644 --- a/tensorflow/contrib/metrics/python/ops/metric_ops.py +++ b/tensorflow/contrib/metrics/python/ops/metric_ops.py @@ -2711,7 +2711,9 @@ def streaming_sparse_average_precision_at_top_k(top_k_predictions, name=name) -@deprecated(None, 'Please switch to tf.metrics.mean.') +@deprecated(None, + 'Please switch to tf.metrics.mean_absolute_error. Note that the ' + 'order of the labels and predictions arguments has been switched.') def streaming_mean_absolute_error(predictions, labels, weights=None, @@ -2830,7 +2832,9 @@ def streaming_mean_relative_error(predictions, updates_collections=updates_collections, name=name) - +@deprecated(None, + 'Please switch to tf.metrics.mean_squared_error. Note that the ' + 'order of the labels and predictions arguments has been switched.') def streaming_mean_squared_error(predictions, labels, weights=None, @@ -2888,7 +2892,10 @@ def streaming_mean_squared_error(predictions, updates_collections=updates_collections, name=name) - +@deprecated( + None, + 'Please switch to tf.metrics.root_mean_squared_error. Note that the ' + 'order of the labels and predictions arguments has been switched.') def streaming_root_mean_squared_error(predictions, labels, weights=None, -- GitLab From 744a5cc092401f3725f06498058e6ba262fd697d Mon Sep 17 00:00:00 2001 From: Francois Chollet Date: Wed, 11 Apr 2018 13:46:03 -0700 Subject: [PATCH 179/791] When not necessary, avoid the creation of a `placeholder_with_default` in BN (not yet supported by TPU compilation). PiperOrigin-RevId: 192502020 --- tensorflow/python/keras/_impl/keras/layers/normalization.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/keras/_impl/keras/layers/normalization.py b/tensorflow/python/keras/_impl/keras/layers/normalization.py index b73025a5a8..5462a95d7d 100644 --- a/tensorflow/python/keras/_impl/keras/layers/normalization.py +++ b/tensorflow/python/keras/_impl/keras/layers/normalization.py @@ -489,6 +489,7 @@ class BatchNormalization(Layer): return (r, d, new_mean, new_variance) def call(self, inputs, training=None): + original_training_value = training if training is None: training = K.learning_phase() @@ -512,7 +513,7 @@ class BatchNormalization(Layer): # Currently never reaches here since fused_batch_norm does not support # virtual batching outputs = undo_virtual_batching(outputs) - if not context.executing_eagerly() and training is K.learning_phase(): + if not context.executing_eagerly() and original_training_value is None: outputs._uses_learning_phase = True # pylint: disable=protected-access return outputs @@ -628,7 +629,7 @@ class BatchNormalization(Layer): if self.virtual_batch_size is not None: outputs = undo_virtual_batching(outputs) - if not context.executing_eagerly() and training is K.learning_phase(): + if not context.executing_eagerly() and original_training_value is None: outputs._uses_learning_phase = True # pylint: disable=protected-access return outputs -- GitLab From 3fa224a453bb9d7f7f8340231adb53ba74b79b42 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 13:47:46 -0700 Subject: [PATCH 180/791] Factor out the syntactic function scope tracking into the transformer. Choosing not to do this at static analysis because it exposes the scope to any node, making it easier to use by any specialization of a transformer. PiperOrigin-RevId: 192502309 --- tensorflow/contrib/autograph/pyct/BUILD | 11 +++ .../contrib/autograph/pyct/transformer.py | 15 +++ .../autograph/pyct/transformer_test.py | 97 +++++++++++++++++++ 3 files changed, 123 insertions(+) create mode 100644 tensorflow/contrib/autograph/pyct/transformer_test.py diff --git a/tensorflow/contrib/autograph/pyct/BUILD b/tensorflow/contrib/autograph/pyct/BUILD index c483ff68c4..796ab445c7 100644 --- a/tensorflow/contrib/autograph/pyct/BUILD +++ b/tensorflow/contrib/autograph/pyct/BUILD @@ -125,3 +125,14 @@ py_test( "@gast_archive//:gast", ], ) + +py_test( + name = "transformer_test", + srcs = ["transformer_test.py"], + srcs_version = "PY2AND3", + deps = [ + ":pyct", + "//tensorflow/python:client_testlib", + "@gast_archive//:gast", + ], +) diff --git a/tensorflow/contrib/autograph/pyct/transformer.py b/tensorflow/contrib/autograph/pyct/transformer.py index 35f114b6e1..b38d52c5b2 100644 --- a/tensorflow/contrib/autograph/pyct/transformer.py +++ b/tensorflow/contrib/autograph/pyct/transformer.py @@ -51,6 +51,11 @@ class Base(gast.NodeTransformer): self._lineno = 0 self._col_offset = 0 self.context = context + self._enclosing_entities = [] + + @property + def enclosing_entities(self): + return tuple(self._enclosing_entities) def debug_print(self, node): """Helper method useful for debugging.""" @@ -61,13 +66,20 @@ class Base(gast.NodeTransformer): def visit(self, node): source_code = self.context.source_code source_file = self.context.source_file + did_enter_function = False + try: + if isinstance(node, (gast.FunctionDef, gast.ClassDef, gast.Lambda)): + self._enclosing_entities.append(node) + did_enter_function = True + if source_code and hasattr(node, 'lineno'): self._lineno = node.lineno self._col_offset = node.col_offset if anno.hasanno(node, anno.Basic.SKIP_PROCESSING): return node return super(Base, self).visit(node) + except (ValueError, AttributeError, KeyError, NotImplementedError, AssertionError) as e: msg = '%s: %s\nOffending source:\n%s\n\nOccurred at node:\n%s' % ( @@ -82,3 +94,6 @@ class Base(gast.NodeTransformer): msg, (source_file, self._lineno, self._col_offset + 1, line)), sys.exc_info()[2]) + finally: + if did_enter_function: + self._enclosing_entities.pop() diff --git a/tensorflow/contrib/autograph/pyct/transformer_test.py b/tensorflow/contrib/autograph/pyct/transformer_test.py new file mode 100644 index 0000000000..57f1c31ef6 --- /dev/null +++ b/tensorflow/contrib/autograph/pyct/transformer_test.py @@ -0,0 +1,97 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for templates module.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.autograph.pyct import anno +from tensorflow.contrib.autograph.pyct import context +from tensorflow.contrib.autograph.pyct import parser +from tensorflow.contrib.autograph.pyct import transformer +from tensorflow.python.platform import test + + +class TransformerTest(test.TestCase): + + def test_entity_scope_tracking(self): + + class TestTransformer(transformer.Base): + + # The choice of note to assign to is arbitrary. Using Assign because it's + # easy to find in the tree. + def visit_Assign(self, node): + anno.setanno(node, 'enclosing_entities', self.enclosing_entities) + return self.generic_visit(node) + + # This will show up in the lambda function. + def visit_BinOp(self, node): + anno.setanno(node, 'enclosing_entities', self.enclosing_entities) + return self.generic_visit(node) + + tr = TestTransformer( + context.EntityContext( + namer=None, + source_code=None, + source_file=None, + namespace=None, + arg_values=None, + arg_types=None, + owner_type=None, + recursive=False)) + + def test_function(): + a = 0 + + class TestClass(object): + + def test_method(self): + b = 0 + def inner_function(x): + c = 0 + d = lambda y: (x + y) + return c, d + return b, inner_function + return a, TestClass + + node, _ = parser.parse_entity(test_function) + node = tr.visit(node) + + test_function_node = node.body[0] + test_class = test_function_node.body[1] + test_method = test_class.body[0] + inner_function = test_method.body[1] + lambda_node = inner_function.body[1].value + + a = test_function_node.body[0] + b = test_method.body[0] + c = inner_function.body[0] + lambda_expr = lambda_node.body + + self.assertEqual( + (test_function_node,), anno.getanno(a, 'enclosing_entities')) + self.assertEqual((test_function_node, test_class, test_method), + anno.getanno(b, 'enclosing_entities')) + self.assertEqual( + (test_function_node, test_class, test_method, inner_function), + anno.getanno(c, 'enclosing_entities')) + self.assertEqual((test_function_node, test_class, test_method, + inner_function, lambda_node), + anno.getanno(lambda_expr, 'enclosing_entities')) + + +if __name__ == '__main__': + test.main() -- GitLab From 1a36eb1550639b22fa884ccf7511bf8cd65cca95 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 13:48:43 -0700 Subject: [PATCH 181/791] Replace examples/image_retraining by a pointer to TensorFlow Hub. https://github.com/tensorflow/hub/tree/master/examples/image_retraining has the same tool, upgraded to use TensorFlow Hub instead of raw graph defs. PiperOrigin-RevId: 192502469 --- tensorflow/examples/image_retraining/BUILD | 51 - .../examples/image_retraining/README.md | 21 +- .../examples/image_retraining/__init__.py | 0 .../examples/image_retraining/data/labels.txt | 3 - .../examples/image_retraining/retrain.py | 1487 ----------------- .../examples/image_retraining/retrain_test.py | 148 -- 6 files changed, 12 insertions(+), 1698 deletions(-) delete mode 100644 tensorflow/examples/image_retraining/BUILD delete mode 100644 tensorflow/examples/image_retraining/__init__.py delete mode 100644 tensorflow/examples/image_retraining/data/labels.txt delete mode 100644 tensorflow/examples/image_retraining/retrain.py delete mode 100644 tensorflow/examples/image_retraining/retrain_test.py diff --git a/tensorflow/examples/image_retraining/BUILD b/tensorflow/examples/image_retraining/BUILD deleted file mode 100644 index ecd79a3b00..0000000000 --- a/tensorflow/examples/image_retraining/BUILD +++ /dev/null @@ -1,51 +0,0 @@ -# Description: -# Transfer learning example for TensorFlow. - -licenses(["notice"]) # Apache 2.0 - -exports_files(["LICENSE"]) - -load("//tensorflow:tensorflow.bzl", "py_test") - -py_binary( - name = "retrain", - srcs = [ - "retrain.py", - ], - srcs_version = "PY2AND3", - visibility = ["//tensorflow:__subpackages__"], - deps = [ - "//tensorflow:tensorflow_py", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:graph_util", - "//tensorflow/python:platform", - "//tensorflow/python:util", - "//third_party/py/numpy", - ], -) - -py_test( - name = "retrain_test", - size = "small", - srcs = [ - "retrain.py", - "retrain_test.py", - ], - data = [ - ":data/labels.txt", - "//tensorflow/examples/label_image:data/grace_hopper.jpg", - ], - srcs_version = "PY2AND3", - deps = [ - ":retrain", - "//tensorflow:tensorflow_py", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:graph_util", - "//tensorflow/python:platform", - "//tensorflow/python:platform_test", - "//tensorflow/python:tensor_shape", - "//tensorflow/python:util", - "//third_party/py/numpy", - ], -) diff --git a/tensorflow/examples/image_retraining/README.md b/tensorflow/examples/image_retraining/README.md index 8a49525c6e..3f0b3d1268 100644 --- a/tensorflow/examples/image_retraining/README.md +++ b/tensorflow/examples/image_retraining/README.md @@ -1,12 +1,15 @@ -retrain.py is an example script that shows how one can adapt a pretrained -network for other classification problems. A detailed overview of this script -can be found at: -https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/#0 +**NOTE: This code has moved to** +https://github.com/tensorflow/hub/tree/master/examples/image_retraining -The script also shows how one can train layers -with quantized weights and activations instead of taking a pre-trained floating -point model and then quantizing weights and activations. -The output graphdef produced by this script is compatible with the TensorFlow -Lite Optimizing Converter and can be converted to TFLite format. +retrain.py is an example script that shows how one can adapt a pretrained +network for other classification problems (including use with TFLite and +quantization). +As of TensorFlow 1.7, it is recommended to use a pretrained network from +TensorFlow Hub, using the new version of this example found in the location +above, as explained in TensorFlow's revised [image retraining +tutorial](https://www.tensorflow.org/tutorials/image_retraining). +Older versions of this example (using frozen GraphDefs instead of +TensorFlow Hub modules) are available in the release branches of +TensorFlow versions up to and including 1.7. diff --git a/tensorflow/examples/image_retraining/__init__.py b/tensorflow/examples/image_retraining/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tensorflow/examples/image_retraining/data/labels.txt b/tensorflow/examples/image_retraining/data/labels.txt deleted file mode 100644 index bc1131ac45..0000000000 --- a/tensorflow/examples/image_retraining/data/labels.txt +++ /dev/null @@ -1,3 +0,0 @@ -Runner-up -Winner -Loser diff --git a/tensorflow/examples/image_retraining/retrain.py b/tensorflow/examples/image_retraining/retrain.py deleted file mode 100644 index fcc191250f..0000000000 --- a/tensorflow/examples/image_retraining/retrain.py +++ /dev/null @@ -1,1487 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -r"""Simple transfer learning with Inception v3 or Mobilenet models. - -With support for TensorBoard. - -This example shows how to take a Inception v3 or Mobilenet model trained on -ImageNet images, and train a new top layer that can recognize other classes of -images. - -The top layer receives as input a 2048-dimensional vector (1001-dimensional for -Mobilenet) for each image. We train a softmax layer on top of this -representation. Assuming the softmax layer contains N labels, this corresponds -to learning N + 2048*N (or 1001*N) model parameters corresponding to the -learned biases and weights. - -Here's an example, which assumes you have a folder containing class-named -subfolders, each full of images for each label. The example folder flower_photos -should have a structure like this: - -~/flower_photos/daisy/photo1.jpg -~/flower_photos/daisy/photo2.jpg -... -~/flower_photos/rose/anotherphoto77.jpg -... -~/flower_photos/sunflower/somepicture.jpg - -The subfolder names are important, since they define what label is applied to -each image, but the filenames themselves don't matter. Once your images are -prepared, you can run the training with a command like this: - -```bash -bazel build tensorflow/examples/image_retraining:retrain && \ -bazel-bin/tensorflow/examples/image_retraining/retrain \ - --image_dir ~/flower_photos -``` - -Or, if you have a pip installation of tensorflow, `retrain.py` can be run -without bazel: - -```bash -python tensorflow/examples/image_retraining/retrain.py \ - --image_dir ~/flower_photos -``` - -You can replace the image_dir argument with any folder containing subfolders of -images. The label for each image is taken from the name of the subfolder it's -in. - -This produces a new model file that can be loaded and run by any TensorFlow -program, for example the label_image sample code. - -By default this script will use the high accuracy, but comparatively large and -slow Inception v3 model architecture. It's recommended that you start with this -to validate that you have gathered good training data, but if you want to deploy -on resource-limited platforms, you can try the `--architecture` flag with a -Mobilenet model. For example: - -Run floating-point version of mobilenet: - -```bash -python tensorflow/examples/image_retraining/retrain.py \ - --image_dir ~/flower_photos --architecture mobilenet_1.0_224 -``` - -Run mobilenet, instrumented for quantization: - -```bash -python tensorflow/examples/image_retraining/retrain.py \ - --image_dir ~/flower_photos/ --architecture mobilenet_1.0_224_quant -``` - -These instrumented models can be converted to fully quantized mobile models via -TensorFlow Lite. - -There are 32 different Mobilenet models to choose from, with a variety of file -size and latency options. The first number can be '1.0', '0.75', '0.50', or -'0.25' to control the size, and the second controls the input image size, either -'224', '192', '160', or '128', with smaller sizes running faster. See -https://research.googleblog.com/2017/06/mobilenets-open-source-models-for.html -for more information on Mobilenet. - -To use with TensorBoard: - -By default, this script will log summaries to /tmp/retrain_logs directory - -Visualize the summaries with this command: - -tensorboard --logdir /tmp/retrain_logs - -To use with Tensorflow Serving: - -```bash -tensorflow_model_server --port=9000 --model_name=inception \ - --model_base_path=/tmp/saved_models/ -``` -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -from datetime import datetime -import hashlib -import os.path -import random -import re -import sys -import tarfile - -import numpy as np -from six.moves import urllib -import tensorflow as tf - -from tensorflow.python.framework import graph_util -from tensorflow.python.framework import tensor_shape -from tensorflow.python.platform import gfile -from tensorflow.python.util import compat - -FLAGS = None - -# These are all parameters that are tied to the particular model architecture -# we're using for Inception v3. These include things like tensor names and their -# sizes. If you want to adapt this script to work with another model, you will -# need to update these to reflect the values in the network you're using. -MAX_NUM_IMAGES_PER_CLASS = 2 ** 27 - 1 # ~134M - -# The location where variable checkpoints will be stored. -CHECKPOINT_NAME = '/tmp/_retrain_checkpoint' - - -def create_image_lists(image_dir, testing_percentage, validation_percentage): - """Builds a list of training images from the file system. - - Analyzes the sub folders in the image directory, splits them into stable - training, testing, and validation sets, and returns a data structure - describing the lists of images for each label and their paths. - - Args: - image_dir: String path to a folder containing subfolders of images. - testing_percentage: Integer percentage of the images to reserve for tests. - validation_percentage: Integer percentage of images reserved for validation. - - Returns: - A dictionary containing an entry for each label subfolder, with images split - into training, testing, and validation sets within each label. - """ - if not gfile.Exists(image_dir): - tf.logging.error("Image directory '" + image_dir + "' not found.") - return None - result = {} - sub_dirs = [x[0] for x in gfile.Walk(image_dir)] - # The root directory comes first, so skip it. - is_root_dir = True - for sub_dir in sub_dirs: - if is_root_dir: - is_root_dir = False - continue - extensions = ['jpg', 'jpeg', 'JPG', 'JPEG'] - file_list = [] - dir_name = os.path.basename(sub_dir) - if dir_name == image_dir: - continue - tf.logging.info("Looking for images in '" + dir_name + "'") - for extension in extensions: - file_glob = os.path.join(image_dir, dir_name, '*.' + extension) - file_list.extend(gfile.Glob(file_glob)) - if not file_list: - tf.logging.warning('No files found') - continue - if len(file_list) < 20: - tf.logging.warning( - 'WARNING: Folder has less than 20 images, which may cause issues.') - elif len(file_list) > MAX_NUM_IMAGES_PER_CLASS: - tf.logging.warning( - 'WARNING: Folder {} has more than {} images. Some images will ' - 'never be selected.'.format(dir_name, MAX_NUM_IMAGES_PER_CLASS)) - label_name = re.sub(r'[^a-z0-9]+', ' ', dir_name.lower()) - training_images = [] - testing_images = [] - validation_images = [] - for file_name in file_list: - base_name = os.path.basename(file_name) - # We want to ignore anything after '_nohash_' in the file name when - # deciding which set to put an image in, the data set creator has a way of - # grouping photos that are close variations of each other. For example - # this is used in the plant disease data set to group multiple pictures of - # the same leaf. - hash_name = re.sub(r'_nohash_.*$', '', file_name) - # This looks a bit magical, but we need to decide whether this file should - # go into the training, testing, or validation sets, and we want to keep - # existing files in the same set even if more files are subsequently - # added. - # To do that, we need a stable way of deciding based on just the file name - # itself, so we do a hash of that and then use that to generate a - # probability value that we use to assign it. - hash_name_hashed = hashlib.sha1(compat.as_bytes(hash_name)).hexdigest() - percentage_hash = ((int(hash_name_hashed, 16) % - (MAX_NUM_IMAGES_PER_CLASS + 1)) * - (100.0 / MAX_NUM_IMAGES_PER_CLASS)) - if percentage_hash < validation_percentage: - validation_images.append(base_name) - elif percentage_hash < (testing_percentage + validation_percentage): - testing_images.append(base_name) - else: - training_images.append(base_name) - result[label_name] = { - 'dir': dir_name, - 'training': training_images, - 'testing': testing_images, - 'validation': validation_images, - } - return result - - -def get_image_path(image_lists, label_name, index, image_dir, category): - """"Returns a path to an image for a label at the given index. - - Args: - image_lists: Dictionary of training images for each label. - label_name: Label string we want to get an image for. - index: Int offset of the image we want. This will be moduloed by the - available number of images for the label, so it can be arbitrarily large. - image_dir: Root folder string of the subfolders containing the training - images. - category: Name string of set to pull images from - training, testing, or - validation. - - Returns: - File system path string to an image that meets the requested parameters. - - """ - if label_name not in image_lists: - tf.logging.fatal('Label does not exist %s.', label_name) - label_lists = image_lists[label_name] - if category not in label_lists: - tf.logging.fatal('Category does not exist %s.', category) - category_list = label_lists[category] - if not category_list: - tf.logging.fatal('Label %s has no images in the category %s.', - label_name, category) - mod_index = index % len(category_list) - base_name = category_list[mod_index] - sub_dir = label_lists['dir'] - full_path = os.path.join(image_dir, sub_dir, base_name) - return full_path - - -def get_bottleneck_path(image_lists, label_name, index, bottleneck_dir, - category, architecture): - """"Returns a path to a bottleneck file for a label at the given index. - - Args: - image_lists: Dictionary of training images for each label. - label_name: Label string we want to get an image for. - index: Integer offset of the image we want. This will be moduloed by the - available number of images for the label, so it can be arbitrarily large. - bottleneck_dir: Folder string holding cached files of bottleneck values. - category: Name string of set to pull images from - training, testing, or - validation. - architecture: The name of the model architecture. - - Returns: - File system path string to an image that meets the requested parameters. - """ - return get_image_path(image_lists, label_name, index, bottleneck_dir, - category) + '_' + architecture + '.txt' - - -def create_model_graph(model_info): - """"Creates a graph from saved GraphDef file and returns a Graph object. - - Args: - model_info: Dictionary containing information about the model architecture. - - Returns: - Graph holding the trained Inception network, and various tensors we'll be - manipulating. - """ - with tf.Graph().as_default() as graph: - model_path = os.path.join(FLAGS.model_dir, model_info['model_file_name']) - print('Model path: ', model_path) - with gfile.FastGFile(model_path, 'rb') as f: - graph_def = tf.GraphDef() - graph_def.ParseFromString(f.read()) - bottleneck_tensor, resized_input_tensor = (tf.import_graph_def( - graph_def, - name='', - return_elements=[ - model_info['bottleneck_tensor_name'], - model_info['resized_input_tensor_name'], - ])) - return graph, bottleneck_tensor, resized_input_tensor - - -def run_bottleneck_on_image(sess, image_data, image_data_tensor, - decoded_image_tensor, resized_input_tensor, - bottleneck_tensor): - """Runs inference on an image to extract the 'bottleneck' summary layer. - - Args: - sess: Current active TensorFlow Session. - image_data: String of raw JPEG data. - image_data_tensor: Input data layer in the graph. - decoded_image_tensor: Output of initial image resizing and preprocessing. - resized_input_tensor: The input node of the recognition graph. - bottleneck_tensor: Layer before the final softmax. - - Returns: - Numpy array of bottleneck values. - """ - # First decode the JPEG image, resize it, and rescale the pixel values. - resized_input_values = sess.run(decoded_image_tensor, - {image_data_tensor: image_data}) - # Then run it through the recognition network. - bottleneck_values = sess.run(bottleneck_tensor, - {resized_input_tensor: resized_input_values}) - bottleneck_values = np.squeeze(bottleneck_values) - return bottleneck_values - - -def maybe_download_and_extract(data_url): - """Download and extract model tar file. - - If the pretrained model we're using doesn't already exist, this function - downloads it from the TensorFlow.org website and unpacks it into a directory. - - Args: - data_url: Web location of the tar file containing the pretrained model. - """ - dest_directory = FLAGS.model_dir - if not os.path.exists(dest_directory): - os.makedirs(dest_directory) - filename = data_url.split('/')[-1] - filepath = os.path.join(dest_directory, filename) - if not os.path.exists(filepath): - - def _progress(count, block_size, total_size): - sys.stdout.write('\r>> Downloading %s %.1f%%' % - (filename, - float(count * block_size) / float(total_size) * 100.0)) - sys.stdout.flush() - - filepath, _ = urllib.request.urlretrieve(data_url, filepath, _progress) - print() - statinfo = os.stat(filepath) - tf.logging.info('Successfully downloaded %s %d bytes.', filename, - statinfo.st_size) - print('Extracting file from ', filepath) - tarfile.open(filepath, 'r:gz').extractall(dest_directory) - else: - print('Not extracting or downloading files, model already present in disk') - - -def ensure_dir_exists(dir_name): - """Makes sure the folder exists on disk. - - Args: - dir_name: Path string to the folder we want to create. - """ - if not os.path.exists(dir_name): - os.makedirs(dir_name) - - -bottleneck_path_2_bottleneck_values = {} - - -def create_bottleneck_file(bottleneck_path, image_lists, label_name, index, - image_dir, category, sess, jpeg_data_tensor, - decoded_image_tensor, resized_input_tensor, - bottleneck_tensor): - """Create a single bottleneck file.""" - tf.logging.info('Creating bottleneck at ' + bottleneck_path) - image_path = get_image_path(image_lists, label_name, index, - image_dir, category) - if not gfile.Exists(image_path): - tf.logging.fatal('File does not exist %s', image_path) - image_data = gfile.FastGFile(image_path, 'rb').read() - try: - bottleneck_values = run_bottleneck_on_image( - sess, image_data, jpeg_data_tensor, decoded_image_tensor, - resized_input_tensor, bottleneck_tensor) - except Exception as e: - raise RuntimeError('Error during processing file %s (%s)' % (image_path, - str(e))) - bottleneck_string = ','.join(str(x) for x in bottleneck_values) - with open(bottleneck_path, 'w') as bottleneck_file: - bottleneck_file.write(bottleneck_string) - - -def get_or_create_bottleneck(sess, image_lists, label_name, index, image_dir, - category, bottleneck_dir, jpeg_data_tensor, - decoded_image_tensor, resized_input_tensor, - bottleneck_tensor, architecture): - """Retrieves or calculates bottleneck values for an image. - - If a cached version of the bottleneck data exists on-disk, return that, - otherwise calculate the data and save it to disk for future use. - - Args: - sess: The current active TensorFlow Session. - image_lists: Dictionary of training images for each label. - label_name: Label string we want to get an image for. - index: Integer offset of the image we want. This will be modulo-ed by the - available number of images for the label, so it can be arbitrarily large. - image_dir: Root folder string of the subfolders containing the training - images. - category: Name string of which set to pull images from - training, testing, - or validation. - bottleneck_dir: Folder string holding cached files of bottleneck values. - jpeg_data_tensor: The tensor to feed loaded jpeg data into. - decoded_image_tensor: The output of decoding and resizing the image. - resized_input_tensor: The input node of the recognition graph. - bottleneck_tensor: The output tensor for the bottleneck values. - architecture: The name of the model architecture. - - Returns: - Numpy array of values produced by the bottleneck layer for the image. - """ - label_lists = image_lists[label_name] - sub_dir = label_lists['dir'] - sub_dir_path = os.path.join(bottleneck_dir, sub_dir) - ensure_dir_exists(sub_dir_path) - bottleneck_path = get_bottleneck_path(image_lists, label_name, index, - bottleneck_dir, category, architecture) - if not os.path.exists(bottleneck_path): - create_bottleneck_file(bottleneck_path, image_lists, label_name, index, - image_dir, category, sess, jpeg_data_tensor, - decoded_image_tensor, resized_input_tensor, - bottleneck_tensor) - with open(bottleneck_path, 'r') as bottleneck_file: - bottleneck_string = bottleneck_file.read() - did_hit_error = False - try: - bottleneck_values = [float(x) for x in bottleneck_string.split(',')] - except ValueError: - tf.logging.warning('Invalid float found, recreating bottleneck') - did_hit_error = True - if did_hit_error: - create_bottleneck_file(bottleneck_path, image_lists, label_name, index, - image_dir, category, sess, jpeg_data_tensor, - decoded_image_tensor, resized_input_tensor, - bottleneck_tensor) - with open(bottleneck_path, 'r') as bottleneck_file: - bottleneck_string = bottleneck_file.read() - # Allow exceptions to propagate here, since they shouldn't happen after a - # fresh creation - bottleneck_values = [float(x) for x in bottleneck_string.split(',')] - return bottleneck_values - - -def cache_bottlenecks(sess, image_lists, image_dir, bottleneck_dir, - jpeg_data_tensor, decoded_image_tensor, - resized_input_tensor, bottleneck_tensor, architecture): - """Ensures all the training, testing, and validation bottlenecks are cached. - - Because we're likely to read the same image multiple times (if there are no - distortions applied during training) it can speed things up a lot if we - calculate the bottleneck layer values once for each image during - preprocessing, and then just read those cached values repeatedly during - training. Here we go through all the images we've found, calculate those - values, and save them off. - - Args: - sess: The current active TensorFlow Session. - image_lists: Dictionary of training images for each label. - image_dir: Root folder string of the subfolders containing the training - images. - bottleneck_dir: Folder string holding cached files of bottleneck values. - jpeg_data_tensor: Input tensor for jpeg data from file. - decoded_image_tensor: The output of decoding and resizing the image. - resized_input_tensor: The input node of the recognition graph. - bottleneck_tensor: The penultimate output layer of the graph. - architecture: The name of the model architecture. - - Returns: - Nothing. - """ - how_many_bottlenecks = 0 - ensure_dir_exists(bottleneck_dir) - for label_name, label_lists in image_lists.items(): - for category in ['training', 'testing', 'validation']: - category_list = label_lists[category] - for index, unused_base_name in enumerate(category_list): - get_or_create_bottleneck( - sess, image_lists, label_name, index, image_dir, category, - bottleneck_dir, jpeg_data_tensor, decoded_image_tensor, - resized_input_tensor, bottleneck_tensor, architecture) - - how_many_bottlenecks += 1 - if how_many_bottlenecks % 100 == 0: - tf.logging.info( - str(how_many_bottlenecks) + ' bottleneck files created.') - - -def get_random_cached_bottlenecks(sess, image_lists, how_many, category, - bottleneck_dir, image_dir, jpeg_data_tensor, - decoded_image_tensor, resized_input_tensor, - bottleneck_tensor, architecture): - """Retrieves bottleneck values for cached images. - - If no distortions are being applied, this function can retrieve the cached - bottleneck values directly from disk for images. It picks a random set of - images from the specified category. - - Args: - sess: Current TensorFlow Session. - image_lists: Dictionary of training images for each label. - how_many: If positive, a random sample of this size will be chosen. - If negative, all bottlenecks will be retrieved. - category: Name string of which set to pull from - training, testing, or - validation. - bottleneck_dir: Folder string holding cached files of bottleneck values. - image_dir: Root folder string of the subfolders containing the training - images. - jpeg_data_tensor: The layer to feed jpeg image data into. - decoded_image_tensor: The output of decoding and resizing the image. - resized_input_tensor: The input node of the recognition graph. - bottleneck_tensor: The bottleneck output layer of the CNN graph. - architecture: The name of the model architecture. - - Returns: - List of bottleneck arrays, their corresponding ground truths, and the - relevant filenames. - """ - class_count = len(image_lists.keys()) - bottlenecks = [] - ground_truths = [] - filenames = [] - if how_many >= 0: - # Retrieve a random sample of bottlenecks. - for unused_i in range(how_many): - label_index = random.randrange(class_count) - label_name = list(image_lists.keys())[label_index] - image_index = random.randrange(MAX_NUM_IMAGES_PER_CLASS + 1) - image_name = get_image_path(image_lists, label_name, image_index, - image_dir, category) - bottleneck = get_or_create_bottleneck( - sess, image_lists, label_name, image_index, image_dir, category, - bottleneck_dir, jpeg_data_tensor, decoded_image_tensor, - resized_input_tensor, bottleneck_tensor, architecture) - bottlenecks.append(bottleneck) - ground_truths.append(label_index) - filenames.append(image_name) - else: - # Retrieve all bottlenecks. - for label_index, label_name in enumerate(image_lists.keys()): - for image_index, image_name in enumerate( - image_lists[label_name][category]): - image_name = get_image_path(image_lists, label_name, image_index, - image_dir, category) - bottleneck = get_or_create_bottleneck( - sess, image_lists, label_name, image_index, image_dir, category, - bottleneck_dir, jpeg_data_tensor, decoded_image_tensor, - resized_input_tensor, bottleneck_tensor, architecture) - bottlenecks.append(bottleneck) - ground_truths.append(label_index) - filenames.append(image_name) - return bottlenecks, ground_truths, filenames - - -def get_random_distorted_bottlenecks( - sess, image_lists, how_many, category, image_dir, input_jpeg_tensor, - distorted_image, resized_input_tensor, bottleneck_tensor): - """Retrieves bottleneck values for training images, after distortions. - - If we're training with distortions like crops, scales, or flips, we have to - recalculate the full model for every image, and so we can't use cached - bottleneck values. Instead we find random images for the requested category, - run them through the distortion graph, and then the full graph to get the - bottleneck results for each. - - Args: - sess: Current TensorFlow Session. - image_lists: Dictionary of training images for each label. - how_many: The integer number of bottleneck values to return. - category: Name string of which set of images to fetch - training, testing, - or validation. - image_dir: Root folder string of the subfolders containing the training - images. - input_jpeg_tensor: The input layer we feed the image data to. - distorted_image: The output node of the distortion graph. - resized_input_tensor: The input node of the recognition graph. - bottleneck_tensor: The bottleneck output layer of the CNN graph. - - Returns: - List of bottleneck arrays and their corresponding ground truths. - """ - class_count = len(image_lists.keys()) - bottlenecks = [] - ground_truths = [] - for unused_i in range(how_many): - label_index = random.randrange(class_count) - label_name = list(image_lists.keys())[label_index] - image_index = random.randrange(MAX_NUM_IMAGES_PER_CLASS + 1) - image_path = get_image_path(image_lists, label_name, image_index, image_dir, - category) - if not gfile.Exists(image_path): - tf.logging.fatal('File does not exist %s', image_path) - jpeg_data = gfile.FastGFile(image_path, 'rb').read() - # Note that we materialize the distorted_image_data as a numpy array before - # sending running inference on the image. This involves 2 memory copies and - # might be optimized in other implementations. - distorted_image_data = sess.run(distorted_image, - {input_jpeg_tensor: jpeg_data}) - bottleneck_values = sess.run(bottleneck_tensor, - {resized_input_tensor: distorted_image_data}) - bottleneck_values = np.squeeze(bottleneck_values) - bottlenecks.append(bottleneck_values) - ground_truths.append(label_index) - return bottlenecks, ground_truths - - -def should_distort_images(flip_left_right, random_crop, random_scale, - random_brightness): - """Whether any distortions are enabled, from the input flags. - - Args: - flip_left_right: Boolean whether to randomly mirror images horizontally. - random_crop: Integer percentage setting the total margin used around the - crop box. - random_scale: Integer percentage of how much to vary the scale by. - random_brightness: Integer range to randomly multiply the pixel values by. - - Returns: - Boolean value indicating whether any distortions should be applied. - """ - return (flip_left_right or (random_crop != 0) or (random_scale != 0) or - (random_brightness != 0)) - - -def add_input_distortions(flip_left_right, random_crop, random_scale, - random_brightness, input_width, input_height, - input_depth, input_mean, input_std): - """Creates the operations to apply the specified distortions. - - During training it can help to improve the results if we run the images - through simple distortions like crops, scales, and flips. These reflect the - kind of variations we expect in the real world, and so can help train the - model to cope with natural data more effectively. Here we take the supplied - parameters and construct a network of operations to apply them to an image. - - Cropping - ~~~~~~~~ - - Cropping is done by placing a bounding box at a random position in the full - image. The cropping parameter controls the size of that box relative to the - input image. If it's zero, then the box is the same size as the input and no - cropping is performed. If the value is 50%, then the crop box will be half the - width and height of the input. In a diagram it looks like this: - - < width > - +---------------------+ - | | - | width - crop% | - | < > | - | +------+ | - | | | | - | | | | - | | | | - | +------+ | - | | - | | - +---------------------+ - - Scaling - ~~~~~~~ - - Scaling is a lot like cropping, except that the bounding box is always - centered and its size varies randomly within the given range. For example if - the scale percentage is zero, then the bounding box is the same size as the - input and no scaling is applied. If it's 50%, then the bounding box will be in - a random range between half the width and height and full size. - - Args: - flip_left_right: Boolean whether to randomly mirror images horizontally. - random_crop: Integer percentage setting the total margin used around the - crop box. - random_scale: Integer percentage of how much to vary the scale by. - random_brightness: Integer range to randomly multiply the pixel values by. - graph. - input_width: Horizontal size of expected input image to model. - input_height: Vertical size of expected input image to model. - input_depth: How many channels the expected input image should have. - input_mean: Pixel value that should be zero in the image for the graph. - input_std: How much to divide the pixel values by before recognition. - - Returns: - The jpeg input layer and the distorted result tensor. - """ - - jpeg_data = tf.placeholder(tf.string, name='DistortJPGInput') - decoded_image = tf.image.decode_jpeg(jpeg_data, channels=input_depth) - decoded_image_as_float = tf.cast(decoded_image, dtype=tf.float32) - decoded_image_4d = tf.expand_dims(decoded_image_as_float, 0) - margin_scale = 1.0 + (random_crop / 100.0) - resize_scale = 1.0 + (random_scale / 100.0) - margin_scale_value = tf.constant(margin_scale) - resize_scale_value = tf.random_uniform(tensor_shape.scalar(), - minval=1.0, - maxval=resize_scale) - scale_value = tf.multiply(margin_scale_value, resize_scale_value) - precrop_width = tf.multiply(scale_value, input_width) - precrop_height = tf.multiply(scale_value, input_height) - precrop_shape = tf.stack([precrop_height, precrop_width]) - precrop_shape_as_int = tf.cast(precrop_shape, dtype=tf.int32) - precropped_image = tf.image.resize_bilinear(decoded_image_4d, - precrop_shape_as_int) - precropped_image_3d = tf.squeeze(precropped_image, squeeze_dims=[0]) - cropped_image = tf.random_crop(precropped_image_3d, - [input_height, input_width, input_depth]) - if flip_left_right: - flipped_image = tf.image.random_flip_left_right(cropped_image) - else: - flipped_image = cropped_image - brightness_min = 1.0 - (random_brightness / 100.0) - brightness_max = 1.0 + (random_brightness / 100.0) - brightness_value = tf.random_uniform(tensor_shape.scalar(), - minval=brightness_min, - maxval=brightness_max) - brightened_image = tf.multiply(flipped_image, brightness_value) - offset_image = tf.subtract(brightened_image, input_mean) - mul_image = tf.multiply(offset_image, 1.0 / input_std) - distort_result = tf.expand_dims(mul_image, 0, name='DistortResult') - return jpeg_data, distort_result - - -def variable_summaries(var): - """Attach a lot of summaries to a Tensor (for TensorBoard visualization).""" - with tf.name_scope('summaries'): - mean = tf.reduce_mean(var) - tf.summary.scalar('mean', mean) - with tf.name_scope('stddev'): - stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean))) - tf.summary.scalar('stddev', stddev) - tf.summary.scalar('max', tf.reduce_max(var)) - tf.summary.scalar('min', tf.reduce_min(var)) - tf.summary.histogram('histogram', var) - - -def add_final_retrain_ops(class_count, final_tensor_name, bottleneck_tensor, - bottleneck_tensor_size, quantize_layer, is_training): - """Adds a new softmax and fully-connected layer for training and eval. - - We need to retrain the top layer to identify our new classes, so this function - adds the right operations to the graph, along with some variables to hold the - weights, and then sets up all the gradients for the backward pass. - - The set up for the softmax and fully-connected layers is based on: - https://www.tensorflow.org/versions/master/tutorials/mnist/beginners/index.html - - Args: - class_count: Integer of how many categories of things we're trying to - recognize. - final_tensor_name: Name string for the new final node that produces results. - bottleneck_tensor: The output of the main CNN graph. - bottleneck_tensor_size: How many entries in the bottleneck vector. - quantize_layer: Boolean, specifying whether the newly added layer should be - instrumented for quantized. - is_training: Boolean, specifying whether the newly add layer is for training - or eval. - - Returns: - The tensors for the training and cross entropy results, and tensors for the - bottleneck input and ground truth input. - """ - with tf.name_scope('input'): - bottleneck_input = tf.placeholder_with_default( - bottleneck_tensor, - shape=[None, bottleneck_tensor_size], - name='BottleneckInputPlaceholder') - - ground_truth_input = tf.placeholder( - tf.int64, [None], name='GroundTruthInput') - - # Organizing the following ops so they are easier to see in TensorBoard. - layer_name = 'final_retrain_ops' - with tf.name_scope(layer_name): - with tf.name_scope('weights'): - initial_value = tf.truncated_normal( - [bottleneck_tensor_size, class_count], stddev=0.001) - layer_weights = tf.Variable(initial_value, name='final_weights') - variable_summaries(layer_weights) - - with tf.name_scope('biases'): - layer_biases = tf.Variable(tf.zeros([class_count]), name='final_biases') - variable_summaries(layer_biases) - - with tf.name_scope('Wx_plus_b'): - logits = tf.matmul(bottleneck_input, layer_weights) + layer_biases - tf.summary.histogram('pre_activations', logits) - - final_tensor = tf.nn.softmax(logits, name=final_tensor_name) - - # The tf.contrib.quantize functions rewrite the graph in place for - # quantization. The imported model graph has already been rewritten, so upon - # calling these rewrites, only the newly added final layer will be - # transformed. - if quantize_layer: - if is_training: - tf.contrib.quantize.create_training_graph() - else: - tf.contrib.quantize.create_eval_graph() - - tf.summary.histogram('activations', final_tensor) - - # If this is an eval graph, we don't need to add loss ops or an optimizer. - if not is_training: - return None, None, bottleneck_input, ground_truth_input, final_tensor - - with tf.name_scope('cross_entropy'): - cross_entropy_mean = tf.losses.sparse_softmax_cross_entropy( - labels=ground_truth_input, logits=logits) - - tf.summary.scalar('cross_entropy', cross_entropy_mean) - - with tf.name_scope('train'): - optimizer = tf.train.GradientDescentOptimizer(FLAGS.learning_rate) - train_step = optimizer.minimize(cross_entropy_mean) - - return (train_step, cross_entropy_mean, bottleneck_input, ground_truth_input, - final_tensor) - - -def add_evaluation_step(result_tensor, ground_truth_tensor): - """Inserts the operations we need to evaluate the accuracy of our results. - - Args: - result_tensor: The new final node that produces results. - ground_truth_tensor: The node we feed ground truth data - into. - - Returns: - Tuple of (evaluation step, prediction). - """ - with tf.name_scope('accuracy'): - with tf.name_scope('correct_prediction'): - prediction = tf.argmax(result_tensor, 1) - correct_prediction = tf.equal(prediction, ground_truth_tensor) - with tf.name_scope('accuracy'): - evaluation_step = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) - tf.summary.scalar('accuracy', evaluation_step) - return evaluation_step, prediction - - -def run_final_eval(sess, model_info, class_count, image_lists, jpeg_data_tensor, - decoded_image_tensor, resized_image_tensor, - bottleneck_tensor): - """Runs a final evaluation on an eval graph using the test data set. - - Args: - sess: Session for the train graph. - model_info: Model info dictionary from create_model_info() - class_count: Number of classes - image_lists: Dictionary of training images for each label. - jpeg_data_tensor: The layer to feed jpeg image data into. - decoded_image_tensor: The output of decoding and resizing the image. - resized_image_tensor: The input node of the recognition graph. - bottleneck_tensor: The bottleneck output layer of the CNN graph. - """ - test_bottlenecks, test_ground_truth, test_filenames = ( - get_random_cached_bottlenecks(sess, image_lists, FLAGS.test_batch_size, - 'testing', FLAGS.bottleneck_dir, - FLAGS.image_dir, jpeg_data_tensor, - decoded_image_tensor, resized_image_tensor, - bottleneck_tensor, FLAGS.architecture)) - - (sess, bottleneck_input, ground_truth_input, evaluation_step, - prediction) = build_eval_session(model_info, class_count) - - test_accuracy, predictions = sess.run( - [evaluation_step, prediction], - feed_dict={ - bottleneck_input: test_bottlenecks, - ground_truth_input: test_ground_truth - }) - tf.logging.info('Final test accuracy = %.1f%% (N=%d)' % - (test_accuracy * 100, len(test_bottlenecks))) - - if FLAGS.print_misclassified_test_images: - tf.logging.info('=== MISCLASSIFIED TEST IMAGES ===') - for i, test_filename in enumerate(test_filenames): - if predictions[i] != test_ground_truth[i]: - tf.logging.info('%70s %s' % (test_filename, - list(image_lists.keys())[predictions[i]])) - - -def build_eval_session(model_info, class_count): - """Builds an restored eval session without train operations for exporting. - - Args: - model_info: Model info dictionary from create_model_info() - class_count: Number of classes - - Returns: - Eval session containing the restored eval graph. - The bottleneck input, ground truth, eval step, and prediction tensors. - """ - # If quantized, we need to create the correct eval graph for exporting. - eval_graph, bottleneck_tensor, _ = create_model_graph(model_info) - - eval_sess = tf.Session(graph=eval_graph) - with eval_graph.as_default(): - # Add the new layer for exporting. - (_, _, bottleneck_input, - ground_truth_input, final_tensor) = add_final_retrain_ops( - class_count, FLAGS.final_tensor_name, bottleneck_tensor, - model_info['bottleneck_tensor_size'], model_info['quantize_layer'], - False) - - # Now we need to restore the values from the training graph to the eval - # graph. - tf.train.Saver().restore(eval_sess, CHECKPOINT_NAME) - - evaluation_step, prediction = add_evaluation_step(final_tensor, - ground_truth_input) - - return (eval_sess, bottleneck_input, ground_truth_input, evaluation_step, - prediction) - - -def save_graph_to_file(graph, graph_file_name, model_info, class_count): - """Saves an graph to file, creating a valid quantized one if necessary.""" - sess, _, _, _, _ = build_eval_session(model_info, class_count) - graph = sess.graph - - output_graph_def = graph_util.convert_variables_to_constants( - sess, graph.as_graph_def(), [FLAGS.final_tensor_name]) - - with gfile.FastGFile(graph_file_name, 'wb') as f: - f.write(output_graph_def.SerializeToString()) - - -def prepare_file_system(): - # Setup the directory we'll write summaries to for TensorBoard - if tf.gfile.Exists(FLAGS.summaries_dir): - tf.gfile.DeleteRecursively(FLAGS.summaries_dir) - tf.gfile.MakeDirs(FLAGS.summaries_dir) - if FLAGS.intermediate_store_frequency > 0: - ensure_dir_exists(FLAGS.intermediate_output_graphs_dir) - return - - -def create_model_info(architecture): - """Given the name of a model architecture, returns information about it. - - There are different base image recognition pretrained models that can be - retrained using transfer learning, and this function translates from the name - of a model to the attributes that are needed to download and train with it. - - Args: - architecture: Name of a model architecture. - - Returns: - Dictionary of information about the model, or None if the name isn't - recognized - - Raises: - ValueError: If architecture name is unknown. - """ - architecture = architecture.lower() - is_quantized = False - if architecture == 'inception_v3': - # pylint: disable=line-too-long - data_url = 'http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz' - # pylint: enable=line-too-long - bottleneck_tensor_name = 'pool_3/_reshape:0' - bottleneck_tensor_size = 2048 - input_width = 299 - input_height = 299 - input_depth = 3 - resized_input_tensor_name = 'Mul:0' - model_file_name = 'classify_image_graph_def.pb' - input_mean = 128 - input_std = 128 - elif architecture.startswith('mobilenet_'): - parts = architecture.split('_') - if len(parts) != 3 and len(parts) != 4: - tf.logging.error("Couldn't understand architecture name '%s'", - architecture) - return None - version_string = parts[1] - if (version_string != '1.0' and version_string != '0.75' and - version_string != '0.5' and version_string != '0.25'): - tf.logging.error( - """"The Mobilenet version should be '1.0', '0.75', '0.5', or '0.25', - but found '%s' for architecture '%s'""", version_string, architecture) - return None - size_string = parts[2] - if (size_string != '224' and size_string != '192' and - size_string != '160' and size_string != '128'): - tf.logging.error( - """The Mobilenet input size should be '224', '192', '160', or '128', - but found '%s' for architecture '%s'""", - size_string, architecture) - return None - if len(parts) == 3: - is_quantized = False - else: - if parts[3] != 'quant': - tf.logging.error( - "Couldn't understand architecture suffix '%s' for '%s'", parts[3], - architecture) - return None - is_quantized = True - - data_url = 'http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/' - model_name = 'mobilenet_v1_' + version_string + '_' + size_string - if is_quantized: - model_name += '_quant' - data_url += model_name + '.tgz' - bottleneck_tensor_name = 'MobilenetV1/Predictions/Reshape:0' - resized_input_tensor_name = 'input:0' - model_file_name = model_name + '_frozen.pb' - - bottleneck_tensor_size = 1001 - input_width = int(size_string) - input_height = int(size_string) - input_depth = 3 - input_mean = 127.5 - input_std = 127.5 - else: - tf.logging.error("Couldn't understand architecture name '%s'", architecture) - raise ValueError('Unknown architecture', architecture) - - return { - 'data_url': data_url, - 'bottleneck_tensor_name': bottleneck_tensor_name, - 'bottleneck_tensor_size': bottleneck_tensor_size, - 'input_width': input_width, - 'input_height': input_height, - 'input_depth': input_depth, - 'resized_input_tensor_name': resized_input_tensor_name, - 'model_file_name': model_file_name, - 'input_mean': input_mean, - 'input_std': input_std, - 'quantize_layer': is_quantized, - } - - -def add_jpeg_decoding(input_width, input_height, input_depth, input_mean, - input_std): - """Adds operations that perform JPEG decoding and resizing to the graph.. - - Args: - input_width: Desired width of the image fed into the recognizer graph. - input_height: Desired width of the image fed into the recognizer graph. - input_depth: Desired channels of the image fed into the recognizer graph. - input_mean: Pixel value that should be zero in the image for the graph. - input_std: How much to divide the pixel values by before recognition. - - Returns: - Tensors for the node to feed JPEG data into, and the output of the - preprocessing steps. - """ - jpeg_data = tf.placeholder(tf.string, name='DecodeJPGInput') - decoded_image = tf.image.decode_jpeg(jpeg_data, channels=input_depth) - decoded_image_as_float = tf.cast(decoded_image, dtype=tf.float32) - decoded_image_4d = tf.expand_dims(decoded_image_as_float, 0) - resize_shape = tf.stack([input_height, input_width]) - resize_shape_as_int = tf.cast(resize_shape, dtype=tf.int32) - resized_image = tf.image.resize_bilinear(decoded_image_4d, - resize_shape_as_int) - offset_image = tf.subtract(resized_image, input_mean) - mul_image = tf.multiply(offset_image, 1.0 / input_std) - return jpeg_data, mul_image - - -def export_model(model_info, class_count, saved_model_dir): - """Exports model for serving. - - Args: - model_info: The modelinfo for the current model. - class_count: The number of classes. - saved_model_dir: Directory in which to save exported model and variables. - """ - # The SavedModel should hold the eval graph. - sess, _, _, _, _ = build_eval_session(model_info, class_count) - graph = sess.graph - with graph.as_default(): - input_tensor = model_info['resized_input_tensor_name'] - in_image = sess.graph.get_tensor_by_name(input_tensor) - inputs = {'image': tf.saved_model.utils.build_tensor_info(in_image)} - - out_classes = sess.graph.get_tensor_by_name('final_result:0') - outputs = { - 'prediction': tf.saved_model.utils.build_tensor_info(out_classes) - } - - signature = tf.saved_model.signature_def_utils.build_signature_def( - inputs=inputs, - outputs=outputs, - method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME) - - legacy_init_op = tf.group(tf.tables_initializer(), name='legacy_init_op') - - # Save out the SavedModel. - builder = tf.saved_model.builder.SavedModelBuilder(saved_model_dir) - builder.add_meta_graph_and_variables( - sess, [tf.saved_model.tag_constants.SERVING], - signature_def_map={ - tf.saved_model.signature_constants. - DEFAULT_SERVING_SIGNATURE_DEF_KEY: - signature - }, - legacy_init_op=legacy_init_op) - builder.save() - - -def main(_): - # Needed to make sure the logging output is visible. - # See https://github.com/tensorflow/tensorflow/issues/3047 - tf.logging.set_verbosity(tf.logging.INFO) - - # Prepare necessary directories that can be used during training - prepare_file_system() - - # Gather information about the model architecture we'll be using. - model_info = create_model_info(FLAGS.architecture) - if not model_info: - tf.logging.error('Did not recognize architecture flag') - return -1 - - # Look at the folder structure, and create lists of all the images. - image_lists = create_image_lists(FLAGS.image_dir, FLAGS.testing_percentage, - FLAGS.validation_percentage) - class_count = len(image_lists.keys()) - if class_count == 0: - tf.logging.error('No valid folders of images found at ' + FLAGS.image_dir) - return -1 - if class_count == 1: - tf.logging.error('Only one valid folder of images found at ' + - FLAGS.image_dir + - ' - multiple classes are needed for classification.') - return -1 - - # See if the command-line flags mean we're applying any distortions. - do_distort_images = should_distort_images( - FLAGS.flip_left_right, FLAGS.random_crop, FLAGS.random_scale, - FLAGS.random_brightness) - - # Set up the pre-trained graph. - maybe_download_and_extract(model_info['data_url']) - graph, bottleneck_tensor, resized_image_tensor = ( - create_model_graph(model_info)) - - # Add the new layer that we'll be training. - with graph.as_default(): - (train_step, cross_entropy, bottleneck_input, - ground_truth_input, final_tensor) = add_final_retrain_ops( - class_count, FLAGS.final_tensor_name, bottleneck_tensor, - model_info['bottleneck_tensor_size'], model_info['quantize_layer'], - True) - - with tf.Session(graph=graph) as sess: - # Set up the image decoding sub-graph. - jpeg_data_tensor, decoded_image_tensor = add_jpeg_decoding( - model_info['input_width'], model_info['input_height'], - model_info['input_depth'], model_info['input_mean'], - model_info['input_std']) - - if do_distort_images: - # We will be applying distortions, so setup the operations we'll need. - (distorted_jpeg_data_tensor, - distorted_image_tensor) = add_input_distortions( - FLAGS.flip_left_right, FLAGS.random_crop, FLAGS.random_scale, - FLAGS.random_brightness, model_info['input_width'], - model_info['input_height'], model_info['input_depth'], - model_info['input_mean'], model_info['input_std']) - else: - # We'll make sure we've calculated the 'bottleneck' image summaries and - # cached them on disk. - cache_bottlenecks(sess, image_lists, FLAGS.image_dir, - FLAGS.bottleneck_dir, jpeg_data_tensor, - decoded_image_tensor, resized_image_tensor, - bottleneck_tensor, FLAGS.architecture) - - # Create the operations we need to evaluate the accuracy of our new layer. - evaluation_step, _ = add_evaluation_step(final_tensor, ground_truth_input) - - # Merge all the summaries and write them out to the summaries_dir - merged = tf.summary.merge_all() - train_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/train', - sess.graph) - - validation_writer = tf.summary.FileWriter( - FLAGS.summaries_dir + '/validation') - - # Create a train saver that is used to restore values into an eval graph - # when exporting models. - train_saver = tf.train.Saver() - - # Set up all our weights to their initial default values. - init = tf.global_variables_initializer() - sess.run(init) - - # Run the training for as many cycles as requested on the command line. - for i in range(FLAGS.how_many_training_steps): - # Get a batch of input bottleneck values, either calculated fresh every - # time with distortions applied, or from the cache stored on disk. - if do_distort_images: - (train_bottlenecks, - train_ground_truth) = get_random_distorted_bottlenecks( - sess, image_lists, FLAGS.train_batch_size, 'training', - FLAGS.image_dir, distorted_jpeg_data_tensor, - distorted_image_tensor, resized_image_tensor, bottleneck_tensor) - else: - (train_bottlenecks, - train_ground_truth, _) = get_random_cached_bottlenecks( - sess, image_lists, FLAGS.train_batch_size, 'training', - FLAGS.bottleneck_dir, FLAGS.image_dir, jpeg_data_tensor, - decoded_image_tensor, resized_image_tensor, bottleneck_tensor, - FLAGS.architecture) - # Feed the bottlenecks and ground truth into the graph, and run a training - # step. Capture training summaries for TensorBoard with the `merged` op. - train_summary, _ = sess.run( - [merged, train_step], - feed_dict={bottleneck_input: train_bottlenecks, - ground_truth_input: train_ground_truth}) - train_writer.add_summary(train_summary, i) - - # Every so often, print out how well the graph is training. - is_last_step = (i + 1 == FLAGS.how_many_training_steps) - if (i % FLAGS.eval_step_interval) == 0 or is_last_step: - train_accuracy, cross_entropy_value = sess.run( - [evaluation_step, cross_entropy], - feed_dict={bottleneck_input: train_bottlenecks, - ground_truth_input: train_ground_truth}) - tf.logging.info('%s: Step %d: Train accuracy = %.1f%%' % - (datetime.now(), i, train_accuracy * 100)) - tf.logging.info('%s: Step %d: Cross entropy = %f' % - (datetime.now(), i, cross_entropy_value)) - # TODO(suharshs): Make this use an eval graph, to avoid quantization - # moving averages being updated by the validation set, though in - # practice this makes a negligable difference. - validation_bottlenecks, validation_ground_truth, _ = ( - get_random_cached_bottlenecks( - sess, image_lists, FLAGS.validation_batch_size, 'validation', - FLAGS.bottleneck_dir, FLAGS.image_dir, jpeg_data_tensor, - decoded_image_tensor, resized_image_tensor, bottleneck_tensor, - FLAGS.architecture)) - # Run a validation step and capture training summaries for TensorBoard - # with the `merged` op. - validation_summary, validation_accuracy = sess.run( - [merged, evaluation_step], - feed_dict={bottleneck_input: validation_bottlenecks, - ground_truth_input: validation_ground_truth}) - validation_writer.add_summary(validation_summary, i) - tf.logging.info('%s: Step %d: Validation accuracy = %.1f%% (N=%d)' % - (datetime.now(), i, validation_accuracy * 100, - len(validation_bottlenecks))) - - # Store intermediate results - intermediate_frequency = FLAGS.intermediate_store_frequency - - if (intermediate_frequency > 0 and (i % intermediate_frequency == 0) - and i > 0): - # If we want to do an intermediate save, save a checkpoint of the train - # graph, to restore into the eval graph. - train_saver.save(sess, CHECKPOINT_NAME) - intermediate_file_name = (FLAGS.intermediate_output_graphs_dir + - 'intermediate_' + str(i) + '.pb') - tf.logging.info('Save intermediate result to : ' + - intermediate_file_name) - save_graph_to_file(graph, intermediate_file_name, model_info, - class_count) - - # After training is complete, force one last save of the train checkpoint. - train_saver.save(sess, CHECKPOINT_NAME) - - # We've completed all our training, so run a final test evaluation on - # some new images we haven't used before. - run_final_eval(sess, model_info, class_count, image_lists, jpeg_data_tensor, - decoded_image_tensor, resized_image_tensor, - bottleneck_tensor) - - # Write out the trained graph and labels with the weights stored as - # constants. - save_graph_to_file(graph, FLAGS.output_graph, model_info, class_count) - with gfile.FastGFile(FLAGS.output_labels, 'w') as f: - f.write('\n'.join(image_lists.keys()) + '\n') - - export_model(model_info, class_count, FLAGS.saved_model_dir) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument( - '--image_dir', - type=str, - default='', - help='Path to folders of labeled images.' - ) - parser.add_argument( - '--output_graph', - type=str, - default='/tmp/output_graph.pb', - help='Where to save the trained graph.' - ) - parser.add_argument( - '--intermediate_output_graphs_dir', - type=str, - default='/tmp/intermediate_graph/', - help='Where to save the intermediate graphs.' - ) - parser.add_argument( - '--intermediate_store_frequency', - type=int, - default=0, - help="""\ - How many steps to store intermediate graph. If "0" then will not - store.\ - """ - ) - parser.add_argument( - '--output_labels', - type=str, - default='/tmp/output_labels.txt', - help='Where to save the trained graph\'s labels.' - ) - parser.add_argument( - '--summaries_dir', - type=str, - default='/tmp/retrain_logs', - help='Where to save summary logs for TensorBoard.' - ) - parser.add_argument( - '--how_many_training_steps', - type=int, - default=4000, - help='How many training steps to run before ending.' - ) - parser.add_argument( - '--learning_rate', - type=float, - default=0.01, - help='How large a learning rate to use when training.' - ) - parser.add_argument( - '--testing_percentage', - type=int, - default=10, - help='What percentage of images to use as a test set.' - ) - parser.add_argument( - '--validation_percentage', - type=int, - default=10, - help='What percentage of images to use as a validation set.' - ) - parser.add_argument( - '--eval_step_interval', - type=int, - default=10, - help='How often to evaluate the training results.' - ) - parser.add_argument( - '--train_batch_size', - type=int, - default=100, - help='How many images to train on at a time.' - ) - parser.add_argument( - '--test_batch_size', - type=int, - default=-1, - help="""\ - How many images to test on. This test set is only used once, to evaluate - the final accuracy of the model after training completes. - A value of -1 causes the entire test set to be used, which leads to more - stable results across runs.\ - """ - ) - parser.add_argument( - '--validation_batch_size', - type=int, - default=100, - help="""\ - How many images to use in an evaluation batch. This validation set is - used much more often than the test set, and is an early indicator of how - accurate the model is during training. - A value of -1 causes the entire validation set to be used, which leads to - more stable results across training iterations, but may be slower on large - training sets.\ - """ - ) - parser.add_argument( - '--print_misclassified_test_images', - default=False, - help="""\ - Whether to print out a list of all misclassified test images.\ - """, - action='store_true' - ) - parser.add_argument( - '--model_dir', - type=str, - default='/tmp/imagenet', - help="""\ - Path to classify_image_graph_def.pb, - imagenet_synset_to_human_label_map.txt, and - imagenet_2012_challenge_label_map_proto.pbtxt.\ - """ - ) - parser.add_argument( - '--bottleneck_dir', - type=str, - default='/tmp/bottleneck', - help='Path to cache bottleneck layer values as files.' - ) - parser.add_argument( - '--final_tensor_name', - type=str, - default='final_result', - help="""\ - The name of the output classification layer in the retrained graph.\ - """ - ) - parser.add_argument( - '--flip_left_right', - default=False, - help="""\ - Whether to randomly flip half of the training images horizontally.\ - """, - action='store_true' - ) - parser.add_argument( - '--random_crop', - type=int, - default=0, - help="""\ - A percentage determining how much of a margin to randomly crop off the - training images.\ - """ - ) - parser.add_argument( - '--random_scale', - type=int, - default=0, - help="""\ - A percentage determining how much to randomly scale up the size of the - training images by.\ - """ - ) - parser.add_argument( - '--random_brightness', - type=int, - default=0, - help="""\ - A percentage determining how much to randomly multiply the training image - input pixels up or down by.\ - """ - ) - parser.add_argument( - '--architecture', - type=str, - default='inception_v3', - help="""\ - Which model architecture to use. 'inception_v3' is the most accurate, but - also the slowest. For faster or smaller models, chose a MobileNet with the - form 'mobilenet__[_quantized]'. For example, - 'mobilenet_1.0_224' will pick a model that is 17 MB in size and takes 224 - pixel input images, while 'mobilenet_0.25_128_quantized' will choose a much - smaller and less accurate model, taking 128x128 images, and instrumented - for eventual quantization via TensorFlow Lite. - See https://research.googleblog.com/2017/06/mobilenets-open-source-models-for.html - for more information on Mobilenet.\ - """) - parser.add_argument( - '--saved_model_dir', - type=str, - default='/tmp/saved_models/1/', - help='Where to save the exported graph.') - FLAGS, unparsed = parser.parse_known_args() - tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/examples/image_retraining/retrain_test.py b/tensorflow/examples/image_retraining/retrain_test.py deleted file mode 100644 index fb7324c58a..0000000000 --- a/tensorflow/examples/image_retraining/retrain_test.py +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# pylint: disable=g-bad-import-order,unused-import -"""Tests the graph freezing tool.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf -import os - -from tensorflow.examples.image_retraining import retrain -from tensorflow.python.framework import test_util - - -class ImageRetrainingTest(test_util.TensorFlowTestCase): - - def dummyImageLists(self): - return {'label_one': {'dir': 'somedir', 'training': ['image_one.jpg', - 'image_two.jpg'], - 'testing': ['image_three.jpg', 'image_four.jpg'], - 'validation': ['image_five.jpg', 'image_six.jpg']}, - 'label_two': {'dir': 'otherdir', 'training': ['image_one.jpg', - 'image_two.jpg'], - 'testing': ['image_three.jpg', 'image_four.jpg'], - 'validation': ['image_five.jpg', 'image_six.jpg']}} - - def testGetImagePath(self): - image_lists = self.dummyImageLists() - self.assertEqual('image_dir/somedir/image_one.jpg', retrain.get_image_path( - image_lists, 'label_one', 0, 'image_dir', 'training')) - self.assertEqual('image_dir/otherdir/image_four.jpg', - retrain.get_image_path(image_lists, 'label_two', 1, - 'image_dir', 'testing')) - - def testGetBottleneckPath(self): - image_lists = self.dummyImageLists() - self.assertEqual('bottleneck_dir/somedir/image_five.jpg_imagenet_v3.txt', - retrain.get_bottleneck_path( - image_lists, 'label_one', 0, 'bottleneck_dir', - 'validation', 'imagenet_v3')) - - def testShouldDistortImage(self): - self.assertEqual(False, retrain.should_distort_images(False, 0, 0, 0)) - self.assertEqual(True, retrain.should_distort_images(True, 0, 0, 0)) - self.assertEqual(True, retrain.should_distort_images(False, 10, 0, 0)) - self.assertEqual(True, retrain.should_distort_images(False, 0, 1, 0)) - self.assertEqual(True, retrain.should_distort_images(False, 0, 0, 50)) - - def testAddInputDistortions(self): - with tf.Graph().as_default(): - with tf.Session() as sess: - retrain.add_input_distortions(True, 10, 10, 10, 299, 299, 3, 128, 128) - self.assertIsNotNone(sess.graph.get_tensor_by_name('DistortJPGInput:0')) - self.assertIsNotNone(sess.graph.get_tensor_by_name('DistortResult:0')) - - @tf.test.mock.patch.object(retrain, 'FLAGS', learning_rate=0.01) - def testAddFinalRetrainOps(self, flags_mock): - with tf.Graph().as_default(): - with tf.Session() as sess: - bottleneck = tf.placeholder(tf.float32, [1, 1024], name='bottleneck') - # Test creating final training op with quantization. - retrain.add_final_retrain_ops(5, 'final', bottleneck, 1024, False, - False) - self.assertIsNotNone(sess.graph.get_tensor_by_name('final:0')) - - @tf.test.mock.patch.object(retrain, 'FLAGS', learning_rate=0.01) - def testAddFinalRetrainOpsQuantized(self, flags_mock): - # Ensure that the training and eval graph for quantized models are correctly - # created. - with tf.Graph().as_default() as g: - with tf.Session() as sess: - bottleneck = tf.placeholder(tf.float32, [1, 1024], name='bottleneck') - # Test creating final training op with quantization, set is_training to - # true. - retrain.add_final_retrain_ops(5, 'final', bottleneck, 1024, True, True) - self.assertIsNotNone(sess.graph.get_tensor_by_name('final:0')) - found_fake_quant = 0 - for op in g.get_operations(): - if op.type == 'FakeQuantWithMinMaxVars': - found_fake_quant += 1 - # Ensure that the inputs of each FakeQuant operations has 2 Assign - # operations in the training graph (Assign[Min,Max]Last, - # Assign[Min,Max]Ema) - self.assertEqual(2, - len([i for i in op.inputs if 'Assign' in i.name])) - self.assertEqual(found_fake_quant, 2) - with tf.Graph().as_default() as g: - with tf.Session() as sess: - bottleneck = tf.placeholder(tf.float32, [1, 1024], name='bottleneck') - # Test creating final training op with quantization, set is_training to - # false. - retrain.add_final_retrain_ops(5, 'final', bottleneck, 1024, True, False) - self.assertIsNotNone(sess.graph.get_tensor_by_name('final:0')) - found_fake_quant = 0 - for op in g.get_operations(): - if op.type == 'FakeQuantWithMinMaxVars': - found_fake_quant += 1 - for i in op.inputs: - # Ensure that no operations are Assign operation since this is the - # evaluation graph. - self.assertTrue('Assign' not in i.name) - self.assertEqual(found_fake_quant, 2) - - def testAddEvaluationStep(self): - with tf.Graph().as_default(): - final = tf.placeholder(tf.float32, [1], name='final') - gt = tf.placeholder(tf.int64, [1], name='gt') - self.assertIsNotNone(retrain.add_evaluation_step(final, gt)) - - def testAddJpegDecoding(self): - with tf.Graph().as_default(): - jpeg_data, mul_image = retrain.add_jpeg_decoding(10, 10, 3, 0, 255) - self.assertIsNotNone(jpeg_data) - self.assertIsNotNone(mul_image) - - def testCreateModelInfo(self): - did_raise_value_error = False - try: - retrain.create_model_info('no_such_model_name') - except ValueError: - did_raise_value_error = True - self.assertTrue(did_raise_value_error) - model_info = retrain.create_model_info('inception_v3') - self.assertIsNotNone(model_info) - self.assertEqual(299, model_info['input_width']) - - def testCreateModelInfoQuantized(self): - # Test for mobilenet_quantized - model_info = retrain.create_model_info('mobilenet_1.0_224') - self.assertIsNotNone(model_info) - self.assertEqual(224, model_info['input_width']) - - -if __name__ == '__main__': - tf.test.main() -- GitLab From 73aef57c451a13e07e48933d0bae3ad3ed2c64bd Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 13:53:01 -0700 Subject: [PATCH 182/791] Support for removing unfused quantized activation functions and min/max. PiperOrigin-RevId: 192503204 --- tensorflow/contrib/lite/toco/BUILD | 3 + .../graph_transformations.h | 1 + .../quantization_util.cc | 173 ++++++++++++++++++ .../graph_transformations/quantization_util.h | 50 +++++ .../toco/graph_transformations/quantize.cc | 75 +------- .../remove_trivial_passthrough.cc | 29 +-- ...emove_trivial_quantized_activation_func.cc | 116 +++++++----- .../remove_trivial_quantized_min_max.cc | 90 +++++++++ tensorflow/contrib/lite/toco/toco_tooling.cc | 11 +- 9 files changed, 413 insertions(+), 135 deletions(-) create mode 100644 tensorflow/contrib/lite/toco/graph_transformations/quantization_util.cc create mode 100644 tensorflow/contrib/lite/toco/graph_transformations/quantization_util.h create mode 100644 tensorflow/contrib/lite/toco/graph_transformations/remove_trivial_quantized_min_max.cc diff --git a/tensorflow/contrib/lite/toco/BUILD b/tensorflow/contrib/lite/toco/BUILD index 8a35fb9034..a05d71985f 100644 --- a/tensorflow/contrib/lite/toco/BUILD +++ b/tensorflow/contrib/lite/toco/BUILD @@ -238,6 +238,8 @@ cc_library( "graph_transformations/propagate_activation_function_into_constants.cc", "graph_transformations/propagate_array_data_types.cc", "graph_transformations/propagate_fixed_sizes.cc", + "graph_transformations/quantization_util.cc", + "graph_transformations/quantization_util.h", "graph_transformations/quantize.cc", "graph_transformations/read_fake_quant_min_max.cc", "graph_transformations/remove_final_dequantize_op.cc", @@ -249,6 +251,7 @@ cc_library( "graph_transformations/remove_trivial_passthrough.cc", "graph_transformations/remove_trivial_passthrough.h", "graph_transformations/remove_trivial_quantized_activation_func.cc", + "graph_transformations/remove_trivial_quantized_min_max.cc", "graph_transformations/remove_trivial_reshape.cc", "graph_transformations/remove_trivial_slice.cc", "graph_transformations/remove_unused_op.cc", diff --git a/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h b/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h index 27c5044bb3..80463ce8f8 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h +++ b/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h @@ -146,6 +146,7 @@ DECLARE_GRAPH_TRANSFORMATION(RemoveTrivialConcatenation) DECLARE_GRAPH_TRANSFORMATION(RemoveTrivialConcatenationInput) DECLARE_GRAPH_TRANSFORMATION(RemoveTrivialSlice) DECLARE_GRAPH_TRANSFORMATION(RemoveTrivialQuantizedActivationFunc) +DECLARE_GRAPH_TRANSFORMATION(RemoveTrivialQuantizedMinMax) DECLARE_GRAPH_TRANSFORMATION(RemoveUnusedOp) DECLARE_GRAPH_TRANSFORMATION(ResolveBatchNormalization) DECLARE_GRAPH_TRANSFORMATION(ResolveConstantBinaryOperator) diff --git a/tensorflow/contrib/lite/toco/graph_transformations/quantization_util.cc b/tensorflow/contrib/lite/toco/graph_transformations/quantization_util.cc new file mode 100644 index 0000000000..e080df4bed --- /dev/null +++ b/tensorflow/contrib/lite/toco/graph_transformations/quantization_util.cc @@ -0,0 +1,173 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include + +#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h" +#include "tensorflow/contrib/lite/toco/graph_transformations/quantization_util.h" +#include "tensorflow/contrib/lite/toco/model.h" +#include "tensorflow/contrib/lite/toco/tooling_util.h" +#include "tensorflow/core/platform/logging.h" + +namespace toco { + +bool GetQuantizedDataTypeNumericalRange(ArrayDataType data_type, + double* out_min_value, + double* out_max_value) { + switch (data_type) { + case ArrayDataType::kUint8: + *out_min_value = 0; + *out_max_value = 255; + return true; + case ArrayDataType::kInt16: + *out_min_value = -32768; + *out_max_value = 32767; + return true; + default: + return false; + } +} + +ArrayDataType GetQuantizedDataType(const Array& array, + ArrayDataType default_type) { + switch (array.final_data_type) { + case ArrayDataType::kInt8: + case ArrayDataType::kUint8: + case ArrayDataType::kInt16: + case ArrayDataType::kUint16: + case ArrayDataType::kInt32: + case ArrayDataType::kUint32: + case ArrayDataType::kInt64: + case ArrayDataType::kUint64: + return array.final_data_type; + case ArrayDataType::kFloat: + case ArrayDataType::kNone: + return default_type; + default: + LOG(FATAL) << "Unhandled final quantization type " + << static_cast(array.final_data_type); + } +} + +void GetQuantizationParams(ArrayDataType data_type, const MinMax& minmax, + QuantizationParams* quantization_params) { + switch (data_type) { + case ArrayDataType::kInt8: + GetQuantizationParamsFromMinMax( + minmax, quantization_params); + break; + case ArrayDataType::kUint8: + GetQuantizationParamsFromMinMax( + minmax, quantization_params); + break; + case ArrayDataType::kInt16: + GetQuantizationParamsFromMinMax( + minmax, quantization_params); + break; + case ArrayDataType::kUint16: + GetQuantizationParamsFromMinMax( + minmax, quantization_params); + break; + case ArrayDataType::kInt32: + GetQuantizationParamsFromMinMax( + minmax, quantization_params); + break; + case ArrayDataType::kUint32: + GetQuantizationParamsFromMinMax( + minmax, quantization_params); + break; + case ArrayDataType::kInt64: + GetQuantizationParamsFromMinMax( + minmax, quantization_params); + break; + case ArrayDataType::kUint64: + GetQuantizationParamsFromMinMax( + minmax, quantization_params); + break; + case ArrayDataType::kFloat: + case ArrayDataType::kNone: + default: + LOG(FATAL) << "Unhandled final quantization type " + << static_cast(data_type); + } +} + +bool IsArrayQuantizedRangeSubset(GraphTransformation* transformation, + const Array& array, double clamp_min, + double clamp_max) { + ArrayDataType quantized_data_type = + GetQuantizedDataType(array, array.data_type); + if (quantized_data_type == ArrayDataType::kNone || + quantized_data_type == ArrayDataType::kFloat) { + // The array is not (or never will be) quantized. + return false; + } + + QuantizationParams quantization_params; + if (!array.quantization_params) { + if (!array.minmax) { + transformation->AddMessageF("No quantization params and no minmax"); + return false; + } else { + // Work around cases where we are asking for this prior to the Quantize + // transformation having added the quantization_params. + GetQuantizationParams(quantized_data_type, *array.minmax, + &quantization_params); + transformation->AddMessageF( + "No quantization params - infering from data type %s with minmax " + "%g,%g as zero_point=%g, scale=%g", + ArrayDataTypeName(quantized_data_type), array.minmax->min, + array.minmax->max, quantization_params.zero_point, + quantization_params.scale); + } + } else { + quantization_params = array.GetQuantizationParams(); + } + + double quantized_min, quantized_max; + CHECK(GetQuantizedDataTypeNumericalRange(quantized_data_type, &quantized_min, + &quantized_max)) + << "Type is not quantized"; + + bool has_nontrivial_min_bound = false; + bool has_nontrivial_max_bound = false; + + double lowest_representable_output = + (quantized_min - quantization_params.zero_point) * + quantization_params.scale; + if (lowest_representable_output < clamp_min) { + has_nontrivial_min_bound = true; + transformation->AddMessageF( + "Quantized activation function is not trivial: " + "the lowest representable output value %g" + " less than the clamp min bound %g.", + lowest_representable_output, clamp_min); + } + + double highest_representable_output = + (quantized_max - quantization_params.zero_point) * + quantization_params.scale; + if (highest_representable_output > clamp_max) { + has_nontrivial_max_bound = true; + transformation->AddMessageF( + "Quantized activation function is not trivial: " + "the highest representable output value %g" + " is greater than the clamp max bound %g.", + highest_representable_output, clamp_max); + } + + return !has_nontrivial_min_bound && !has_nontrivial_max_bound; +} + +} // namespace toco diff --git a/tensorflow/contrib/lite/toco/graph_transformations/quantization_util.h b/tensorflow/contrib/lite/toco/graph_transformations/quantization_util.h new file mode 100644 index 0000000000..35fb310777 --- /dev/null +++ b/tensorflow/contrib/lite/toco/graph_transformations/quantization_util.h @@ -0,0 +1,50 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_LITE_TOCO_GRAPH_TRANSFORMATIONS_QUANTIZATION_UTIL_H_ +#define TENSORFLOW_CONTRIB_LITE_TOCO_GRAPH_TRANSFORMATIONS_QUANTIZATION_UTIL_H_ + +#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h" +#include "tensorflow/contrib/lite/toco/model.h" + +namespace toco { + +// Gets the min/max numerical range for the given quantized data type. +// For example, kUint8 will return [0,255]. +// Returns true if the ranges were set and false if the type is not quantized. +bool GetQuantizedDataTypeNumericalRange(ArrayDataType data_type, + double* out_min_value, + double* out_max_value); + +// Returns the quantized data type of an array, falling back to the provided +// default data type. +ArrayDataType GetQuantizedDataType(const Array& array, + ArrayDataType default_type); + +// Gets the quantization params for the array with the given data type and +// minmax. +void GetQuantizationParams(ArrayDataType data_type, const MinMax& minmax, + QuantizationParams* quantization_params); + +// Returns true if the given array, when quantized, contains only values between +// the provided clamp min/max. +// Either clamp_min or clamp_max may be +/-infinity to indicate that the value +// is unbounded on that side. +bool IsArrayQuantizedRangeSubset(GraphTransformation* transformation, + const Array& array, double clamp_min, + double clamp_max); + +} // namespace toco + +#endif // TENSORFLOW_CONTRIB_LITE_TOCO_GRAPH_TRANSFORMATIONS_QUANTIZATION_UTIL_H_ diff --git a/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc b/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc index f50830ae60..d6cae3cdbf 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc +++ b/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc @@ -21,6 +21,7 @@ limitations under the License. #include #include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h" +#include "tensorflow/contrib/lite/toco/graph_transformations/quantization_util.h" #include "tensorflow/contrib/lite/toco/model.h" #include "tensorflow/contrib/lite/toco/model_flags.pb.h" #include "tensorflow/contrib/lite/toco/tooling_util.h" @@ -205,70 +206,6 @@ QuantizationPoints GetQuantizationPoints(ArrayDataType data_type) { } } -ArrayDataType GetQuantizedDataType(const Array& array, - ArrayDataType default_type) { - switch (array.final_data_type) { - case ArrayDataType::kInt8: - case ArrayDataType::kUint8: - case ArrayDataType::kInt16: - case ArrayDataType::kUint16: - case ArrayDataType::kInt32: - case ArrayDataType::kUint32: - case ArrayDataType::kInt64: - case ArrayDataType::kUint64: - return array.final_data_type; - case ArrayDataType::kFloat: - case ArrayDataType::kNone: - return default_type; - default: - LOG(FATAL) << "Unhandled final quantization type " - << static_cast(array.final_data_type); - } -} - -void GetQuantizationParams(ArrayDataType data_type, const MinMax& minmax, - QuantizationParams* quantization_params) { - switch (data_type) { - case ArrayDataType::kInt8: - GetQuantizationParamsFromMinMax( - minmax, quantization_params); - break; - case ArrayDataType::kUint8: - GetQuantizationParamsFromMinMax( - minmax, quantization_params); - break; - case ArrayDataType::kInt16: - GetQuantizationParamsFromMinMax( - minmax, quantization_params); - break; - case ArrayDataType::kUint16: - GetQuantizationParamsFromMinMax( - minmax, quantization_params); - break; - case ArrayDataType::kInt32: - GetQuantizationParamsFromMinMax( - minmax, quantization_params); - break; - case ArrayDataType::kUint32: - GetQuantizationParamsFromMinMax( - minmax, quantization_params); - break; - case ArrayDataType::kInt64: - GetQuantizationParamsFromMinMax( - minmax, quantization_params); - break; - case ArrayDataType::kUint64: - GetQuantizationParamsFromMinMax( - minmax, quantization_params); - break; - case ArrayDataType::kFloat: - case ArrayDataType::kNone: - default: - LOG(FATAL) << "Unhandled final quantization type " - << static_cast(data_type); - } -} - bool ChooseQuantizationForOperatorInput( GraphTransformation* transformation, Model* model, const Operator& op, std::size_t input_index, ArrayDataType* quantized_data_type, @@ -336,12 +273,11 @@ bool ChooseQuantizationForOperatorInput( *quantized_data_type = GetQuantizedDataType(array, ArrayDataType::kUint8); GetQuantizationParams(*quantized_data_type, minmax, quantization_params); transformation->AddMessageF( - "For input array %s with min=%g" - ", max=%g" - ", chose to quantize as %s with zero_point=%d" - ", scale=%g", + "For input array %s with min=%g, max=%g, chose to quantize as %s (f=%s) " + "with zero_point=%d, scale=%g", input, minmax.min, minmax.max, ArrayDataTypeName(*quantized_data_type), - quantization_params->zero_point, quantization_params->scale); + ArrayDataTypeName(array.final_data_type), quantization_params->zero_point, + quantization_params->scale); return true; } @@ -525,6 +461,7 @@ void FixMinMaxPostQuantization(ArrayDataType quantized_data_type, minmax->max = max; } } + } // namespace bool Quantize::Run(Model* model, std::size_t op_index) { diff --git a/tensorflow/contrib/lite/toco/graph_transformations/remove_trivial_passthrough.cc b/tensorflow/contrib/lite/toco/graph_transformations/remove_trivial_passthrough.cc index aa93ace03a..3e021b819f 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/remove_trivial_passthrough.cc +++ b/tensorflow/contrib/lite/toco/graph_transformations/remove_trivial_passthrough.cc @@ -82,22 +82,13 @@ bool RemoveTrivialPassthroughOp(GraphTransformation* transformation, if (IsDiscardableArray(*model, output_name)) { transformation->AddMessageF( - "Removing %s, keeping its non-constant input array", - LogName(*passthru_op)); - for (const string& input : passthru_op->inputs) { - if (IsDiscardableArray(*model, input) && input != main_input_name && - CountOpsWithInput(*model, input) == 1) { - } - } + "Removing %s, keeping its non-constant input array %s and removing %s", + LogName(*passthru_op), main_input_name, output_name); RerouteEdges(output_name, main_input_name, model); } else if (IsDiscardableArray(*model, main_input_name)) { - transformation->AddMessageF("Removing %s, keeping its output array", - LogName(*passthru_op)); - for (const string& input : passthru_op->inputs) { - if (IsDiscardableArray(*model, input) && - (input == main_input_name || CountOpsWithInput(*model, input) == 1)) { - } - } + transformation->AddMessageF( + "Removing %s, keeping its output array %s and removing input %s", + LogName(*passthru_op), output_name, main_input_name); RerouteEdges(main_input_name, output_name, model); } else { transformation->AddMessageF( @@ -113,6 +104,16 @@ bool RemoveTrivialPassthroughOp(GraphTransformation* transformation, // Remove any array that is no longer used. for (const string& removal_candidate : removal_candidates) { bool is_referenced = false; + for (const auto& array : model->flags.input_arrays()) { + if (array.name() == removal_candidate) { + is_referenced = true; + } + } + for (const auto& array_name : model->flags.output_arrays()) { + if (array_name == removal_candidate) { + is_referenced = true; + } + } for (const auto& op : model->operators) { for (const string& input : op->inputs) { if (input == removal_candidate) { diff --git a/tensorflow/contrib/lite/toco/graph_transformations/remove_trivial_quantized_activation_func.cc b/tensorflow/contrib/lite/toco/graph_transformations/remove_trivial_quantized_activation_func.cc index 9b65feaa64..752560e075 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/remove_trivial_quantized_activation_func.cc +++ b/tensorflow/contrib/lite/toco/graph_transformations/remove_trivial_quantized_activation_func.cc @@ -18,6 +18,8 @@ limitations under the License. #include #include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h" +#include "tensorflow/contrib/lite/toco/graph_transformations/quantization_util.h" +#include "tensorflow/contrib/lite/toco/graph_transformations/remove_trivial_passthrough.h" #include "tensorflow/contrib/lite/toco/model.h" #include "tensorflow/contrib/lite/toco/runtime/types.h" #include "tensorflow/contrib/lite/toco/toco_types.h" @@ -26,27 +28,44 @@ limitations under the License. namespace toco { -bool RemoveTrivialQuantizedActivationFunc::Run(Model* model, - std::size_t op_index) { - const auto it = model->operators.begin() + op_index; - auto* op = it->get(); - if (op->fused_activation_function != FusedActivationFunctionType::kRelu && - op->fused_activation_function != FusedActivationFunctionType::kRelu1 && - op->fused_activation_function != FusedActivationFunctionType::kRelu6) { - return false; - } - const auto& output_array = model->GetArray(op->outputs[0]); - if (!output_array.quantization_params) { - return false; - } - if (output_array.data_type != ArrayDataType::kUint8) { - return false; +namespace { + +bool IsTrivialUnfusedActivationFunc(GraphTransformation* transformation, + const Model& model, OperatorType op_type, + const string& input_array_name) { + double clamp_min; + double clamp_max; + switch (op_type) { + case OperatorType::kRelu: + clamp_min = 0.0; + clamp_max = std::numeric_limits::infinity(); + break; + case OperatorType::kRelu1: + clamp_min = -1.0; + clamp_max = 1.0; + break; + case OperatorType::kRelu6: + clamp_min = 0.0; + clamp_max = 6.0; + break; + default: + return false; } - const auto& quantization_params = output_array.GetQuantizationParams(); + const auto& input_array = model.GetArray(input_array_name); + return IsArrayQuantizedRangeSubset(transformation, input_array, clamp_min, + clamp_max); +} + +bool IsTrivialFusedActivationFunc( + GraphTransformation* transformation, const Model& model, + FusedActivationFunctionType activation_function, + const string& output_array_name) { double clamp_min; double clamp_max; - switch (op->fused_activation_function) { + switch (activation_function) { + case FusedActivationFunctionType::kNone: + return false; case FusedActivationFunctionType::kRelu: clamp_min = 0.0; clamp_max = std::numeric_limits::infinity(); @@ -61,45 +80,46 @@ bool RemoveTrivialQuantizedActivationFunc::Run(Model* model, break; default: LOG(FATAL) << "Unsupported fused activation type: " - << static_cast(op->fused_activation_function); + << static_cast(activation_function); return false; } - bool has_nontrivial_min_bound = false; - bool has_nontrivial_max_bound = false; + const auto& output_array = model.GetArray(output_array_name); + return IsArrayQuantizedRangeSubset(transformation, output_array, clamp_min, + clamp_max); +} - double lowest_representable_output = - (0. - quantization_params.zero_point) * quantization_params.scale; - if (lowest_representable_output < clamp_min) { - has_nontrivial_min_bound = true; - AddMessageF( - "Quantized activation function is not trivial: " - "the lowest representable output value %g" - " less than the clamp min bound %g.", - lowest_representable_output, clamp_min); - } - double highest_representable_output = - (255. - quantization_params.zero_point) * quantization_params.scale; - if (highest_representable_output > clamp_max) { - has_nontrivial_max_bound = true; - AddMessageF( - "Quantized activation function is not trivial: " - "the highest representable output value %g" - " is greater than the clamp max bound %g.", - highest_representable_output, clamp_max); - } +} // namespace - if (has_nontrivial_min_bound || has_nontrivial_max_bound) { +// Attempts to remove both fused and unfused activation functions if the +// quantization params indicate that the representable values fall inside the +// activation range. +bool RemoveTrivialQuantizedActivationFunc::Run(Model* model, + std::size_t op_index) { + const auto it = model->operators.begin() + op_index; + auto* op = it->get(); + if (op->inputs.empty()) { return false; } - op->fused_activation_function = FusedActivationFunctionType::kNone; - AddMessageF( - "Removing trivial quantized activation function on %s" - " because the output quantization parameters imply at least as tight" - " a clamp anyway.", - LogName(*op)); - return true; + if (IsTrivialUnfusedActivationFunc(this, *model, op->type, op->inputs[0])) { + AddMessageF( + "Removing trivial unfused activation function %s because the input " + "minmax imply at least as tight a clamp anyway.", + LogName(*op)); + return RemoveTrivialPassthroughOp(this, model, op_index); + } + if (IsTrivialFusedActivationFunc(this, *model, op->fused_activation_function, + op->outputs[0])) { + op->fused_activation_function = FusedActivationFunctionType::kNone; + AddMessageF( + "Removing trivial quantized activation function on %s " + "because the output quantization parameters imply at least as tight " + "a clamp anyway.", + LogName(*op)); + return true; + } + return false; } } // namespace toco diff --git a/tensorflow/contrib/lite/toco/graph_transformations/remove_trivial_quantized_min_max.cc b/tensorflow/contrib/lite/toco/graph_transformations/remove_trivial_quantized_min_max.cc new file mode 100644 index 0000000000..eaee1c662b --- /dev/null +++ b/tensorflow/contrib/lite/toco/graph_transformations/remove_trivial_quantized_min_max.cc @@ -0,0 +1,90 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include +#include +#include + +#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h" +#include "tensorflow/contrib/lite/toco/graph_transformations/quantization_util.h" +#include "tensorflow/contrib/lite/toco/graph_transformations/remove_trivial_passthrough.h" +#include "tensorflow/contrib/lite/toco/model.h" +#include "tensorflow/contrib/lite/toco/runtime/types.h" +#include "tensorflow/contrib/lite/toco/toco_types.h" +#include "tensorflow/contrib/lite/toco/tooling_util.h" +#include "tensorflow/core/platform/logging.h" + +namespace toco { + +namespace { + +bool IsTrivialMinMax(GraphTransformation* transformation, const Model& model, + OperatorType op_type, const string& input_array_name, + const string& clamp_value_array_name) { + const auto& clamp_value_array = model.GetArray(clamp_value_array_name); + if (!IsConstantParameterArray(model, clamp_value_array_name)) { + transformation->AddMessageF("Clip value array %s is non-constant", + clamp_value_array_name); + return false; + } + const auto& clamp_value_buffer = + clamp_value_array.GetBuffer(); + CHECK_EQ(clamp_value_buffer.Length(), 1); + float clamp_value = clamp_value_buffer.data[0]; + + double clamp_min; + double clamp_max; + switch (op_type) { + case OperatorType::kTensorFlowMinimum: + clamp_min = -std::numeric_limits::infinity(); + clamp_max = clamp_value; + break; + case OperatorType::kTensorFlowMaximum: + clamp_min = clamp_value; + clamp_max = std::numeric_limits::infinity(); + break; + default: + CHECK(false); + return false; + } + + const auto& input_array = model.GetArray(input_array_name); + return IsArrayQuantizedRangeSubset(transformation, input_array, clamp_min, + clamp_max); +} + +} // namespace + +// Attempts to remove min/max functions if the quantization params indicate that +// the representable values fall inside the clip range. +bool RemoveTrivialQuantizedMinMax::Run(Model* model, std::size_t op_index) { + const auto it = model->operators.begin() + op_index; + auto* op = it->get(); + if ((op->type != OperatorType::kTensorFlowMinimum && + op->type != OperatorType::kTensorFlowMaximum) || + op->inputs.size() != 2) { + return false; + } + if (IsTrivialMinMax(this, *model, op->type, op->inputs[0], op->inputs[1])) { + AddMessageF( + "Removing trivial min/max %s because the quantization parameters imply " + "at least as tight a clamp anyway.", + LogName(*op)); + return RemoveTrivialPassthroughOp(this, model, op_index); + } + return false; +} + +} // namespace toco diff --git a/tensorflow/contrib/lite/toco/toco_tooling.cc b/tensorflow/contrib/lite/toco/toco_tooling.cc index 96c5ebd64f..1ab0a6f058 100644 --- a/tensorflow/contrib/lite/toco/toco_tooling.cc +++ b/tensorflow/contrib/lite/toco/toco_tooling.cc @@ -279,10 +279,13 @@ void Transform(const TocoFlags& toco_flags, Model* model) { {new HardcodeMinMax}); } CheckIsReadyForQuantization(*model); - RunGraphTransformations( - model, "quantization graph transformations", - {new Quantize, new RemoveTrivialQuantizedActivationFunc, - new RemoveFinalDequantizeOp}); + RunGraphTransformations(model, "quantization graph transformations", + { + new RemoveTrivialQuantizedActivationFunc, + new RemoveTrivialQuantizedMinMax, + new Quantize, + new RemoveFinalDequantizeOp, + }); } else { GraphTransformationsSet dequantization_transformations{new Dequantize}; // Dequantize creates FakeQuant nodes. We may want to discard -- GitLab From 1e283d64816b92de6c398bee6df2122409c87d73 Mon Sep 17 00:00:00 2001 From: Jiri Simsa Date: Wed, 11 Apr 2018 13:59:58 -0700 Subject: [PATCH 183/791] Porting tests for the `decode_proto` and `encode_proto` to OS. PiperOrigin-RevId: 192504411 --- tensorflow/contrib/proto/BUILD | 16 + .../contrib/proto/python/kernel_tests/BUILD | 81 +++++ .../proto/python/kernel_tests/build_defs.bzl | 78 +++++ .../kernel_tests/decode_proto_fail_test.py | 68 ++++ .../kernel_tests/decode_proto_op_test.py | 300 ++++++++++++++++++ .../kernel_tests/encode_proto_op_test.py | 179 +++++++++++ .../python/kernel_tests/minmax.TestCase.pbtxt | 161 ++++++++++ .../python/kernel_tests/nested.TestCase.pbtxt | 16 + .../kernel_tests/optional.TestCase.pbtxt | 20 ++ .../promote_unsigned.TestCase.pbtxt | 21 ++ .../python/kernel_tests/ragged.TestCase.pbtxt | 32 ++ .../kernel_tests/shaped_batch.TestCase.pbtxt | 62 ++++ .../python/kernel_tests/simple.TestCase.pbtxt | 21 ++ .../proto/python/kernel_tests/test_case.py | 35 ++ .../python/kernel_tests/test_example.proto | 149 +++++++++ tensorflow/tools/pip_package/BUILD | 1 + 16 files changed, 1240 insertions(+) create mode 100644 tensorflow/contrib/proto/python/kernel_tests/BUILD create mode 100644 tensorflow/contrib/proto/python/kernel_tests/build_defs.bzl create mode 100644 tensorflow/contrib/proto/python/kernel_tests/decode_proto_fail_test.py create mode 100644 tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test.py create mode 100644 tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test.py create mode 100644 tensorflow/contrib/proto/python/kernel_tests/minmax.TestCase.pbtxt create mode 100644 tensorflow/contrib/proto/python/kernel_tests/nested.TestCase.pbtxt create mode 100644 tensorflow/contrib/proto/python/kernel_tests/optional.TestCase.pbtxt create mode 100644 tensorflow/contrib/proto/python/kernel_tests/promote_unsigned.TestCase.pbtxt create mode 100644 tensorflow/contrib/proto/python/kernel_tests/ragged.TestCase.pbtxt create mode 100644 tensorflow/contrib/proto/python/kernel_tests/shaped_batch.TestCase.pbtxt create mode 100644 tensorflow/contrib/proto/python/kernel_tests/simple.TestCase.pbtxt create mode 100644 tensorflow/contrib/proto/python/kernel_tests/test_case.py create mode 100644 tensorflow/contrib/proto/python/kernel_tests/test_example.proto diff --git a/tensorflow/contrib/proto/BUILD b/tensorflow/contrib/proto/BUILD index 046652cbc5..3e9b1a0b8d 100644 --- a/tensorflow/contrib/proto/BUILD +++ b/tensorflow/contrib/proto/BUILD @@ -4,6 +4,8 @@ licenses(["notice"]) # Apache 2.0 exports_files(["LICENSE"]) +load("//tensorflow/core:platform/default/build_config_root.bzl", "if_static") + py_library( name = "proto", srcs = [ @@ -14,3 +16,17 @@ py_library( "//tensorflow/contrib/proto/python/ops:encode_proto_op_py", ], ) + +py_library( + name = "proto_pip", + data = [ + "//tensorflow/contrib/proto/python/kernel_tests:test_messages", + ] + if_static( + [], + otherwise = ["//tensorflow/contrib/proto/python/kernel_tests:libtestexample.so"], + ), + deps = [ + ":proto", + "//tensorflow/contrib/proto/python/kernel_tests:py_test_deps", + ], +) diff --git a/tensorflow/contrib/proto/python/kernel_tests/BUILD b/tensorflow/contrib/proto/python/kernel_tests/BUILD new file mode 100644 index 0000000000..4125ea8a2a --- /dev/null +++ b/tensorflow/contrib/proto/python/kernel_tests/BUILD @@ -0,0 +1,81 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # Apache 2.0 + +exports_files(["LICENSE"]) + +# Much of the work in this BUILD file actually happens in the corresponding +# build_defs.bzl, which creates an individual testcase for each example .pbtxt +# file in this directory. +# +load(":build_defs.bzl", "decode_proto_test_suite") +load(":build_defs.bzl", "encode_proto_test_suite") + +# This expands to a tf_py_test for each test file. +# It defines the test_suite :decode_proto_op_tests. +decode_proto_test_suite( + name = "decode_proto_tests", + examples = glob(["*.pbtxt"]), +) + +# This expands to a tf_py_test for each test file. +# It defines the test_suite :encode_proto_op_tests. +encode_proto_test_suite( + name = "encode_proto_tests", + examples = glob(["*.pbtxt"]), +) + +# Below here are tests that are not tied to an example text proto. +filegroup( + name = "test_messages", + srcs = glob(["*.pbtxt"]), +) + +load("//tensorflow:tensorflow.bzl", "tf_py_test") +load("//tensorflow:tensorflow.bzl", "tf_cc_shared_object") +load("//tensorflow/core:platform/default/build_config_root.bzl", "if_static") +load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library") + +tf_py_test( + name = "decode_proto_fail_test", + size = "small", + srcs = ["decode_proto_fail_test.py"], + additional_deps = [ + ":py_test_deps", + "//third_party/py/numpy", + "//tensorflow/contrib/proto:proto", + ], + data = if_static( + [], + otherwise = [":libtestexample.so"], + ), +) + +py_library( + name = "test_case", + srcs = ["test_case.py"], + deps = ["//tensorflow/python:client_testlib"], +) + +py_library( + name = "py_test_deps", + deps = [ + ":test_case", + ":test_example_proto_py", + ], +) + +tf_proto_library( + name = "test_example_proto", + srcs = ["test_example.proto"], + cc_api_version = 2, + protodeps = ["//tensorflow/core:protos_all"], +) + +tf_cc_shared_object( + name = "libtestexample.so", + linkstatic = 1, + deps = [ + ":test_example_proto_cc", + ], +) diff --git a/tensorflow/contrib/proto/python/kernel_tests/build_defs.bzl b/tensorflow/contrib/proto/python/kernel_tests/build_defs.bzl new file mode 100644 index 0000000000..6fe48ae807 --- /dev/null +++ b/tensorflow/contrib/proto/python/kernel_tests/build_defs.bzl @@ -0,0 +1,78 @@ +"""BUILD rules for generating file-driven proto test cases. + +The decode_proto_test_suite() and encode_proto_test_suite() rules take a list +of text protos and generates a tf_py_test() for each one. +""" + +load("//tensorflow:tensorflow.bzl", "tf_py_test") +load("//tensorflow:tensorflow.bzl", "register_extension_info") +load("//tensorflow/core:platform/default/build_config_root.bzl", "if_static") + +def _test_name(test, path): + return "%s_%s_test" % (test, path.split("/")[-1].split(".")[0]) + +def decode_proto_test_suite(name, examples): + """Build the decode_proto py_test for each test filename.""" + for test_filename in examples: + tf_py_test( + name = _test_name("decode_proto", test_filename), + srcs = ["decode_proto_op_test.py"], + size = "small", + data = [test_filename] + if_static( + [], + otherwise = [":libtestexample.so"], + ), + main = "decode_proto_op_test.py", + args = [ + "--message_text_file=\"%s/%s\"" % (native.package_name(), test_filename), + ], + additional_deps = [ + ":py_test_deps", + "//third_party/py/numpy", + "//tensorflow/contrib/proto:proto", + ], + ) + native.test_suite( + name = name, + tests = [":" + _test_name("decode_proto", test_filename) + for test_filename in examples], + ) + +def encode_proto_test_suite(name, examples): + """Build the encode_proto py_test for each test filename.""" + for test_filename in examples: + tf_py_test( + name = _test_name("encode_proto", test_filename), + srcs = ["encode_proto_op_test.py"], + size = "small", + data = [test_filename] + if_static( + [], + otherwise = [":libtestexample.so"], + ), + main = "encode_proto_op_test.py", + args = [ + "--message_text_file=\"%s/%s\"" % (native.package_name(), test_filename), + ], + additional_deps = [ + ":py_test_deps", + "//third_party/py/numpy", + "//tensorflow/contrib/proto:proto", + ], + ) + native.test_suite( + name = name, + tests = [":" + _test_name("encode_proto", test_filename) + for test_filename in examples], + ) + +register_extension_info( + extension_name = "decode_proto_test_suite", + label_regex_map = { + "deps": "deps:decode_example_.*", + }) + +register_extension_info( + extension_name = "encode_proto_test_suite", + label_regex_map = { + "deps": "deps:encode_example_.*", + }) diff --git a/tensorflow/contrib/proto/python/kernel_tests/decode_proto_fail_test.py b/tensorflow/contrib/proto/python/kernel_tests/decode_proto_fail_test.py new file mode 100644 index 0000000000..f019833905 --- /dev/null +++ b/tensorflow/contrib/proto/python/kernel_tests/decode_proto_fail_test.py @@ -0,0 +1,68 @@ +# ============================================================================= +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +# Python3 preparedness imports. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.contrib import proto +from tensorflow.contrib.proto.python.kernel_tests import test_case +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors +from tensorflow.python.platform import test + + +class DecodeProtoFailTest(test_case.ProtoOpTestCase): + """Test failure cases for DecodeToProto.""" + + def _TestCorruptProtobuf(self, sanitize): + """Test failure cases for DecodeToProto.""" + + # The goal here is to check the error reporting. + # Testing against a variety of corrupt protobufs is + # done by fuzzing. + corrupt_proto = 'This is not a binary protobuf' + + # Numpy silently truncates the strings if you don't specify dtype=object. + batch = np.array(corrupt_proto, dtype=object) + msg_type = 'tensorflow.contrib.proto.TestCase' + field_names = ['sizes'] + field_types = [dtypes.int32] + + with self.test_session() as sess: + ctensor, vtensor = proto.decode_proto( + batch, + message_type=msg_type, + field_names=field_names, + output_types=field_types, + sanitize=sanitize) + with self.assertRaisesRegexp(errors.DataLossError, + 'Unable to parse binary protobuf' + '|Failed to consume entire buffer'): + _ = sess.run([ctensor] + vtensor) + + def testCorrupt(self): + self._TestCorruptProtobuf(sanitize=False) + + def testSanitizerCorrupt(self): + self._TestCorruptProtobuf(sanitize=True) + + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test.py b/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test.py new file mode 100644 index 0000000000..30ceac5f5f --- /dev/null +++ b/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test.py @@ -0,0 +1,300 @@ +# ============================================================================= +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +"""Table-driven test for decode_proto op. + +This test is run once with each of the *.TestCase.pbtxt files +in the test directory. +""" +# Python3 preparedness imports. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from google.protobuf import text_format + +from tensorflow.contrib import proto +from tensorflow.contrib.proto.python.kernel_tests import test_case +from tensorflow.contrib.proto.python.kernel_tests import test_example_pb2 +from tensorflow.python.framework import dtypes +from tensorflow.python.platform import flags +from tensorflow.python.platform import test + +FLAGS = flags.FLAGS + +flags.DEFINE_string('message_text_file', None, + 'A file containing a text serialized TestCase protobuf.') + + +class DecodeProtoOpTest(test_case.ProtoOpTestCase): + + def _compareValues(self, fd, vs, evs): + """Compare lists/arrays of field values.""" + + if len(vs) != len(evs): + self.fail('Field %s decoded %d outputs, expected %d' % + (fd.name, len(vs), len(evs))) + for i, ev in enumerate(evs): + # Special case fuzzy match for float32. TensorFlow seems to mess with + # MAX_FLT slightly and the test doesn't work otherwise. + # TODO(nix): ask on TF list about why MAX_FLT doesn't pass through. + if fd.cpp_type == fd.CPPTYPE_FLOAT: + # Numpy isclose() is better than assertIsClose() which uses an absolute + # value comparison. + self.assertTrue( + np.isclose(vs[i], ev), 'expected %r, actual %r' % (ev, vs[i])) + elif fd.cpp_type == fd.CPPTYPE_STRING: + # In Python3 string tensor values will be represented as bytes, so we + # reencode the proto values to match that. + self.assertEqual(vs[i], ev.encode('ascii')) + else: + # Doubles and other types pass through unscathed. + self.assertEqual(vs[i], ev) + + def _compareRepeatedPrimitiveValue(self, batch_shape, sizes, fields, + field_dict): + """Compare protos of type RepeatedPrimitiveValue. + + Args: + batch_shape: the shape of the input tensor of serialized messages. + sizes: int matrix of repeat counts returned by decode_proto + fields: list of test_example_pb2.FieldSpec (types and expected values) + field_dict: map from field names to decoded numpy tensors of values + """ + + # Check that expected values match. + for field in fields: + values = field_dict[field.name] + self.assertEqual(dtypes.as_dtype(values.dtype), field.dtype) + + fd = field.expected.DESCRIPTOR.fields_by_name[field.name] + + # Values has the same shape as the input plus an extra + # dimension for repeats. + self.assertEqual(list(values.shape)[:-1], batch_shape) + + # Nested messages are represented as TF strings, requiring + # some special handling. + if field.name == 'message_value': + vs = [] + for buf in values.flat: + msg = test_example_pb2.PrimitiveValue() + msg.ParseFromString(buf) + vs.append(msg) + evs = getattr(field.expected, field.name) + if len(vs) != len(evs): + self.fail('Field %s decoded %d outputs, expected %d' % + (fd.name, len(vs), len(evs))) + for v, ev in zip(vs, evs): + self.assertEqual(v, ev) + continue + + # This can be a little confusing. For testing we are using + # RepeatedPrimitiveValue in two ways: it's the proto that we + # decode for testing, and it's used in the expected value as a + # union type. The two cases are slightly different: this is the + # second case. + # We may be fetching the uint64_value from the test proto, but + # in the expected proto we store it in the int64_value field + # because TensorFlow doesn't support unsigned int64. + tf_type_to_primitive_value_field = { + dtypes.float32: + 'float_value', + dtypes.float64: + 'double_value', + dtypes.int32: + 'int32_value', + dtypes.uint8: + 'uint8_value', + dtypes.int8: + 'int8_value', + dtypes.string: + 'string_value', + dtypes.int64: + 'int64_value', + dtypes.bool: + 'bool_value', + # Unhandled TensorFlow types: + # DT_INT16 DT_COMPLEX64 DT_QINT8 DT_QUINT8 DT_QINT32 + # DT_BFLOAT16 DT_QINT16 DT_QUINT16 DT_UINT16 + } + tf_field_name = tf_type_to_primitive_value_field.get(field.dtype) + if tf_field_name is None: + self.fail('Unhandled tensorflow type %d' % field.dtype) + + self._compareValues(fd, values.flat, + getattr(field.expected, tf_field_name)) + + def _runDecodeProtoTests(self, fields, case_sizes, batch_shape, batch, + message_type, message_format, sanitize, + force_disordered=False): + """Run decode tests on a batch of messages. + + Args: + fields: list of test_example_pb2.FieldSpec (types and expected values) + case_sizes: expected sizes array + batch_shape: the shape of the input tensor of serialized messages + batch: list of serialized messages + message_type: descriptor name for messages + message_format: format of messages, 'text' or 'binary' + sanitize: whether to sanitize binary protobuf inputs + force_disordered: whether to force fields encoded out of order. + """ + + if force_disordered: + # Exercise code path that handles out-of-order fields by prepending extra + # fields with tag numbers higher than any real field. Note that this won't + # work with sanitization because that forces reserialization using a + # trusted decoder and encoder. + assert not sanitize + extra_fields = test_example_pb2.ExtraFields() + extra_fields.string_value = 'IGNORE ME' + extra_fields.bool_value = False + extra_msg = extra_fields.SerializeToString() + batch = [extra_msg + msg for msg in batch] + + # Numpy silently truncates the strings if you don't specify dtype=object. + batch = np.array(batch, dtype=object) + batch = np.reshape(batch, batch_shape) + + field_names = [f.name for f in fields] + output_types = [f.dtype for f in fields] + + with self.test_session() as sess: + sizes, vtensor = proto.decode_proto( + batch, + message_type=message_type, + field_names=field_names, + output_types=output_types, + message_format=message_format, + sanitize=sanitize) + + vlist = sess.run([sizes] + vtensor) + sizes = vlist[0] + # Values is a list of tensors, one for each field. + value_tensors = vlist[1:] + + # Check that the repeat sizes are correct. + self.assertTrue( + np.all(np.array(sizes.shape) == batch_shape + [len(field_names)])) + + # Check that the decoded sizes match the expected sizes. + self.assertEqual(len(sizes.flat), len(case_sizes)) + self.assertTrue( + np.all(sizes.flat == np.array( + case_sizes, dtype=np.int32))) + + field_dict = dict(zip(field_names, value_tensors)) + + self._compareRepeatedPrimitiveValue(batch_shape, sizes, fields, + field_dict) + + def testBinary(self): + with open(FLAGS.message_text_file, 'r') as fp: + case = text_format.Parse(fp.read(), test_example_pb2.TestCase()) + + batch = [primitive.SerializeToString() for primitive in case.primitive] + self._runDecodeProtoTests( + case.field, + case.sizes, + list(case.shape), + batch, + 'tensorflow.contrib.proto.RepeatedPrimitiveValue', + 'binary', + sanitize=False) + + def testBinaryDisordered(self): + with open(FLAGS.message_text_file, 'r') as fp: + case = text_format.Parse(fp.read(), test_example_pb2.TestCase()) + + batch = [primitive.SerializeToString() for primitive in case.primitive] + self._runDecodeProtoTests( + case.field, + case.sizes, + list(case.shape), + batch, + 'tensorflow.contrib.proto.RepeatedPrimitiveValue', + 'binary', + sanitize=False, + force_disordered=True) + + def testPacked(self): + with open(FLAGS.message_text_file, 'r') as fp: + case = text_format.Parse(fp.read(), test_example_pb2.TestCase()) + + # Now try with the packed serialization. + # We test the packed representations by loading the same test cases + # using PackedPrimitiveValue instead of RepeatedPrimitiveValue. + # To do this we rely on the text format being the same for packed and + # unpacked fields, and reparse the test message using the packed version + # of the proto. + packed_batch = [ + # Note: float_format='.17g' is necessary to ensure preservation of + # doubles and floats in text format. + text_format.Parse( + text_format.MessageToString( + primitive, float_format='.17g'), + test_example_pb2.PackedPrimitiveValue()).SerializeToString() + for primitive in case.primitive + ] + + self._runDecodeProtoTests( + case.field, + case.sizes, + list(case.shape), + packed_batch, + 'tensorflow.contrib.proto.PackedPrimitiveValue', + 'binary', + sanitize=False) + + def testText(self): + with open(FLAGS.message_text_file, 'r') as fp: + case = text_format.Parse(fp.read(), test_example_pb2.TestCase()) + + # Note: float_format='.17g' is necessary to ensure preservation of + # doubles and floats in text format. + text_batch = [ + text_format.MessageToString( + primitive, float_format='.17g') for primitive in case.primitive + ] + + self._runDecodeProtoTests( + case.field, + case.sizes, + list(case.shape), + text_batch, + 'tensorflow.contrib.proto.RepeatedPrimitiveValue', + 'text', + sanitize=False) + + def testSanitizerGood(self): + with open(FLAGS.message_text_file, 'r') as fp: + case = text_format.Parse(fp.read(), test_example_pb2.TestCase()) + + batch = [primitive.SerializeToString() for primitive in case.primitive] + self._runDecodeProtoTests( + case.field, + case.sizes, + list(case.shape), + batch, + 'tensorflow.contrib.proto.RepeatedPrimitiveValue', + 'binary', + sanitize=True) + + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test.py b/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test.py new file mode 100644 index 0000000000..2a24c3b8ce --- /dev/null +++ b/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test.py @@ -0,0 +1,179 @@ +# ============================================================================= +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +"""Table-driven test for encode_proto op. + +This test is run once with each of the *.TestCase.pbtxt files +in the test directory. + +It tests that encode_proto is a lossless inverse of decode_proto +(for the specified fields). +""" +# Python3 readiness boilerplate +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from google.protobuf import text_format + +from tensorflow.contrib import proto +from tensorflow.contrib.proto.python.kernel_tests import test_case +from tensorflow.contrib.proto.python.kernel_tests import test_example_pb2 +from tensorflow.python.framework import dtypes +from tensorflow.python.ops import array_ops +from tensorflow.python.platform import flags +from tensorflow.python.platform import test + +FLAGS = flags.FLAGS + +flags.DEFINE_string('message_text_file', None, + 'A file containing a text serialized TestCase protobuf.') + + +class EncodeProtoOpTest(test_case.ProtoOpTestCase): + + def testBadInputs(self): + # Invalid field name + with self.test_session(): + with self.assertRaisesOpError('Unknown field: non_existent_field'): + proto.encode_proto( + sizes=[[1]], + values=[np.array([[0.0]], dtype=np.int32)], + message_type='tensorflow.contrib.proto.RepeatedPrimitiveValue', + field_names=['non_existent_field']).eval() + + # Incorrect types. + with self.test_session(): + with self.assertRaisesOpError( + 'Incompatible type for field double_value.'): + proto.encode_proto( + sizes=[[1]], + values=[np.array([[0.0]], dtype=np.int32)], + message_type='tensorflow.contrib.proto.RepeatedPrimitiveValue', + field_names=['double_value']).eval() + + # Incorrect shapes of sizes. + with self.test_session(): + with self.assertRaisesOpError( + r'sizes should be batch_size \+ \[len\(field_names\)\]'): + sizes = array_ops.placeholder(dtypes.int32) + values = array_ops.placeholder(dtypes.float64) + proto.encode_proto( + sizes=sizes, + values=[values], + message_type='tensorflow.contrib.proto.RepeatedPrimitiveValue', + field_names=['double_value']).eval(feed_dict={ + sizes: [[[0, 0]]], + values: [[0.0]] + }) + + # Inconsistent shapes of values. + with self.test_session(): + with self.assertRaisesOpError( + 'Values must match up to the last dimension'): + sizes = array_ops.placeholder(dtypes.int32) + values1 = array_ops.placeholder(dtypes.float64) + values2 = array_ops.placeholder(dtypes.int32) + (proto.encode_proto( + sizes=[[1, 1]], + values=[values1, values2], + message_type='tensorflow.contrib.proto.RepeatedPrimitiveValue', + field_names=['double_value', 'int32_value']).eval(feed_dict={ + values1: [[0.0]], + values2: [[0], [0]] + })) + + def _testRoundtrip(self, in_bufs, message_type, fields): + + field_names = [f.name for f in fields] + out_types = [f.dtype for f in fields] + + with self.test_session() as sess: + sizes, field_tensors = proto.decode_proto( + in_bufs, + message_type=message_type, + field_names=field_names, + output_types=out_types) + + out_tensors = proto.encode_proto( + sizes, + field_tensors, + message_type=message_type, + field_names=field_names) + + out_bufs, = sess.run([out_tensors]) + + # Check that the re-encoded tensor has the same shape. + self.assertEqual(in_bufs.shape, out_bufs.shape) + + # Compare the input and output. + for in_buf, out_buf in zip(in_bufs.flat, out_bufs.flat): + in_obj = test_example_pb2.RepeatedPrimitiveValue() + in_obj.ParseFromString(in_buf) + + out_obj = test_example_pb2.RepeatedPrimitiveValue() + out_obj.ParseFromString(out_buf) + + # Check that the deserialized objects are identical. + self.assertEqual(in_obj, out_obj) + + # Check that the input and output serialized messages are identical. + # If we fail here, there is a difference in the serialized + # representation but the new serialization still parses. This could + # be harmless (a change in map ordering?) or it could be bad (e.g. + # loss of packing in the encoding). + self.assertEqual(in_buf, out_buf) + + def testRoundtrip(self): + with open(FLAGS.message_text_file, 'r') as fp: + case = text_format.Parse(fp.read(), test_example_pb2.TestCase()) + + in_bufs = [primitive.SerializeToString() for primitive in case.primitive] + + # np.array silently truncates strings if you don't specify dtype=object. + in_bufs = np.reshape(np.array(in_bufs, dtype=object), list(case.shape)) + return self._testRoundtrip( + in_bufs, 'tensorflow.contrib.proto.RepeatedPrimitiveValue', case.field) + + def testRoundtripPacked(self): + with open(FLAGS.message_text_file, 'r') as fp: + case = text_format.Parse(fp.read(), test_example_pb2.TestCase()) + + # Now try with the packed serialization. + # We test the packed representations by loading the same test cases + # using PackedPrimitiveValue instead of RepeatedPrimitiveValue. + # To do this we rely on the text format being the same for packed and + # unpacked fields, and reparse the test message using the packed version + # of the proto. + in_bufs = [ + # Note: float_format='.17g' is necessary to ensure preservation of + # doubles and floats in text format. + text_format.Parse( + text_format.MessageToString( + primitive, float_format='.17g'), + test_example_pb2.PackedPrimitiveValue()).SerializeToString() + for primitive in case.primitive + ] + + # np.array silently truncates strings if you don't specify dtype=object. + in_bufs = np.reshape(np.array(in_bufs, dtype=object), list(case.shape)) + return self._testRoundtrip( + in_bufs, 'tensorflow.contrib.proto.PackedPrimitiveValue', case.field) + + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/contrib/proto/python/kernel_tests/minmax.TestCase.pbtxt b/tensorflow/contrib/proto/python/kernel_tests/minmax.TestCase.pbtxt new file mode 100644 index 0000000000..b170f89c0f --- /dev/null +++ b/tensorflow/contrib/proto/python/kernel_tests/minmax.TestCase.pbtxt @@ -0,0 +1,161 @@ +primitive { + double_value: -1.7976931348623158e+308 + double_value: 2.2250738585072014e-308 + double_value: 1.7976931348623158e+308 + float_value: -3.402823466e+38 + float_value: 1.175494351e-38 + float_value: 3.402823466e+38 + int64_value: -9223372036854775808 + int64_value: 9223372036854775807 + uint64_value: 0 + uint64_value: 18446744073709551615 + int32_value: -2147483648 + int32_value: 2147483647 + fixed64_value: 0 + fixed64_value: 18446744073709551615 + fixed32_value: 0 + fixed32_value: 4294967295 + bool_value: false + bool_value: true + string_value: "" + string_value: "I refer to the infinite." + uint32_value: 0 + uint32_value: 4294967295 + sfixed32_value: -2147483648 + sfixed32_value: 2147483647 + sfixed64_value: -9223372036854775808 + sfixed64_value: 9223372036854775807 + sint32_value: -2147483648 + sint32_value: 2147483647 + sint64_value: -9223372036854775808 + sint64_value: 9223372036854775807 +} +shape: 1 +sizes: 3 +sizes: 3 +sizes: 2 +sizes: 2 +sizes: 2 +sizes: 2 +sizes: 2 +sizes: 2 +sizes: 2 +sizes: 2 +sizes: 2 +sizes: 2 +sizes: 2 +sizes: 2 +field { + name: "double_value" + dtype: DT_DOUBLE + expected { + double_value: -1.7976931348623158e+308 + double_value: 2.2250738585072014e-308 + double_value: 1.7976931348623158e+308 + } +} +field { + name: "float_value" + dtype: DT_FLOAT + expected { + float_value: -3.402823466e+38 + float_value: 1.175494351e-38 + float_value: 3.402823466e+38 + } +} +field { + name: "int64_value" + dtype: DT_INT64 + expected { + int64_value: -9223372036854775808 + int64_value: 9223372036854775807 + } +} +field { + name: "uint64_value" + dtype: DT_INT64 + expected { + int64_value: 0 + int64_value: -1 + } +} +field { + name: "int32_value" + dtype: DT_INT32 + expected { + int32_value: -2147483648 + int32_value: 2147483647 + } +} +field { + name: "fixed64_value" + dtype: DT_INT64 + expected { + int64_value: 0 + int64_value: -1 # unsigned is 18446744073709551615 + } +} +field { + name: "fixed32_value" + dtype: DT_INT32 + expected { + int32_value: 0 + int32_value: -1 # unsigned is 4294967295 + } +} +field { + name: "bool_value" + dtype: DT_BOOL + expected { + bool_value: false + bool_value: true + } +} +field { + name: "string_value" + dtype: DT_STRING + expected { + string_value: "" + string_value: "I refer to the infinite." + } +} +field { + name: "uint32_value" + dtype: DT_INT32 + expected { + int32_value: 0 + int32_value: -1 # unsigned is 4294967295 + } +} +field { + name: "sfixed32_value" + dtype: DT_INT32 + expected { + int32_value: -2147483648 + int32_value: 2147483647 + } +} +field { + name: "sfixed64_value" + dtype: DT_INT64 + expected { + int64_value: -9223372036854775808 + int64_value: 9223372036854775807 + } +} +field { + name: "sint32_value" + dtype: DT_INT32 + expected { + int32_value: -2147483648 + int32_value: 2147483647 + } +} +field { + name: "sint64_value" + dtype: DT_INT64 + expected { + int64_value: -9223372036854775808 + int64_value: 9223372036854775807 + } +} diff --git a/tensorflow/contrib/proto/python/kernel_tests/nested.TestCase.pbtxt b/tensorflow/contrib/proto/python/kernel_tests/nested.TestCase.pbtxt new file mode 100644 index 0000000000..c664e52851 --- /dev/null +++ b/tensorflow/contrib/proto/python/kernel_tests/nested.TestCase.pbtxt @@ -0,0 +1,16 @@ +primitive { + message_value { + double_value: 23.5 + } +} +shape: 1 +sizes: 1 +field { + name: "message_value" + dtype: DT_STRING + expected { + message_value { + double_value: 23.5 + } + } +} diff --git a/tensorflow/contrib/proto/python/kernel_tests/optional.TestCase.pbtxt b/tensorflow/contrib/proto/python/kernel_tests/optional.TestCase.pbtxt new file mode 100644 index 0000000000..125651d7ea --- /dev/null +++ b/tensorflow/contrib/proto/python/kernel_tests/optional.TestCase.pbtxt @@ -0,0 +1,20 @@ +primitive { + bool_value: true +} +shape: 1 +sizes: 1 +sizes: 0 +field { + name: "bool_value" + dtype: DT_BOOL + expected { + bool_value: true + } +} +field { + name: "double_value" + dtype: DT_DOUBLE + expected { + double_value: 0.0 + } +} diff --git a/tensorflow/contrib/proto/python/kernel_tests/promote_unsigned.TestCase.pbtxt b/tensorflow/contrib/proto/python/kernel_tests/promote_unsigned.TestCase.pbtxt new file mode 100644 index 0000000000..db7555bf2d --- /dev/null +++ b/tensorflow/contrib/proto/python/kernel_tests/promote_unsigned.TestCase.pbtxt @@ -0,0 +1,21 @@ +primitive { + fixed32_value: 4294967295 + uint32_value: 4294967295 +} +shape: 1 +sizes: 1 +sizes: 1 +field { + name: "fixed32_value" + dtype: DT_INT64 + expected { + int64_value: 4294967295 + } +} +field { + name: "uint32_value" + dtype: DT_INT64 + expected { + int64_value: 4294967295 + } +} diff --git a/tensorflow/contrib/proto/python/kernel_tests/ragged.TestCase.pbtxt b/tensorflow/contrib/proto/python/kernel_tests/ragged.TestCase.pbtxt new file mode 100644 index 0000000000..61c7ac53f7 --- /dev/null +++ b/tensorflow/contrib/proto/python/kernel_tests/ragged.TestCase.pbtxt @@ -0,0 +1,32 @@ +primitive { + double_value: 23.5 + double_value: 123.0 + bool_value: true +} +primitive { + double_value: 3.1 + bool_value: false +} +shape: 2 +sizes: 2 +sizes: 1 +sizes: 1 +sizes: 1 +field { + name: "double_value" + dtype: DT_DOUBLE + expected { + double_value: 23.5 + double_value: 123.0 + double_value: 3.1 + double_value: 0.0 + } +} +field { + name: "bool_value" + dtype: DT_BOOL + expected { + bool_value: true + bool_value: false + } +} diff --git a/tensorflow/contrib/proto/python/kernel_tests/shaped_batch.TestCase.pbtxt b/tensorflow/contrib/proto/python/kernel_tests/shaped_batch.TestCase.pbtxt new file mode 100644 index 0000000000..f4828076d5 --- /dev/null +++ b/tensorflow/contrib/proto/python/kernel_tests/shaped_batch.TestCase.pbtxt @@ -0,0 +1,62 @@ +primitive { + double_value: 23.5 + bool_value: true +} +primitive { + double_value: 44.0 + bool_value: false +} +primitive { + double_value: 3.14159 + bool_value: true +} +primitive { + double_value: 1.414 + bool_value: true +} +primitive { + double_value: -32.2 + bool_value: false +} +primitive { + double_value: 0.0001 + bool_value: true +} +shape: 3 +shape: 2 +sizes: 1 +sizes: 1 +sizes: 1 +sizes: 1 +sizes: 1 +sizes: 1 +sizes: 1 +sizes: 1 +sizes: 1 +sizes: 1 +sizes: 1 +sizes: 1 +field { + name: "double_value" + dtype: DT_DOUBLE + expected { + double_value: 23.5 + double_value: 44.0 + double_value: 3.14159 + double_value: 1.414 + double_value: -32.2 + double_value: 0.0001 + } +} +field { + name: "bool_value" + dtype: DT_BOOL + expected { + bool_value: true + bool_value: false + bool_value: true + bool_value: true + bool_value: false + bool_value: true + } +} diff --git a/tensorflow/contrib/proto/python/kernel_tests/simple.TestCase.pbtxt b/tensorflow/contrib/proto/python/kernel_tests/simple.TestCase.pbtxt new file mode 100644 index 0000000000..dc20ac147b --- /dev/null +++ b/tensorflow/contrib/proto/python/kernel_tests/simple.TestCase.pbtxt @@ -0,0 +1,21 @@ +primitive { + double_value: 23.5 + bool_value: true +} +shape: 1 +sizes: 1 +sizes: 1 +field { + name: "double_value" + dtype: DT_DOUBLE + expected { + double_value: 23.5 + } +} +field { + name: "bool_value" + dtype: DT_BOOL + expected { + bool_value: true + } +} diff --git a/tensorflow/contrib/proto/python/kernel_tests/test_case.py b/tensorflow/contrib/proto/python/kernel_tests/test_case.py new file mode 100644 index 0000000000..b95202c5df --- /dev/null +++ b/tensorflow/contrib/proto/python/kernel_tests/test_case.py @@ -0,0 +1,35 @@ +# ============================================================================= +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +"""Test case base for testing proto operations.""" + +# Python3 preparedness imports. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import ctypes as ct +import os + +from tensorflow.python.platform import test + + +class ProtoOpTestCase(test.TestCase): + + def __init__(self, methodName='runTest'): # pylint: disable=invalid-name + super(ProtoOpTestCase, self).__init__(methodName) + lib = os.path.join(os.path.dirname(__file__), 'libtestexample.so') + if os.path.isfile(lib): + ct.cdll.LoadLibrary(lib) diff --git a/tensorflow/contrib/proto/python/kernel_tests/test_example.proto b/tensorflow/contrib/proto/python/kernel_tests/test_example.proto new file mode 100644 index 0000000000..dc495034ff --- /dev/null +++ b/tensorflow/contrib/proto/python/kernel_tests/test_example.proto @@ -0,0 +1,149 @@ +// Test description and protos to work with it. +// +// Many of the protos in this file are for unit tests that haven't been written yet. + +syntax = "proto2"; + +import "tensorflow/core/framework/types.proto"; + +package tensorflow.contrib.proto; + +// A TestCase holds a proto and a bunch of assertions +// about how it should decode. +message TestCase { + // A batch of primitives to be serialized and decoded. + repeated RepeatedPrimitiveValue primitive = 1; + // The shape of the batch. + repeated int32 shape = 2; + // Expected sizes for each field. + repeated int32 sizes = 3; + // Expected values for each field. + repeated FieldSpec field = 4; +}; + +// FieldSpec describes the expected output for a single field. +message FieldSpec { + optional string name = 1; + optional tensorflow.DataType dtype = 2; + optional RepeatedPrimitiveValue expected = 3; +}; + +message TestValue { + optional PrimitiveValue primitive_value = 1; + optional EnumValue enum_value = 2; + optional MessageValue message_value = 3; + optional RepeatedMessageValue repeated_message_value = 4; + optional RepeatedPrimitiveValue repeated_primitive_value = 6; +} + +message PrimitiveValue { + optional double double_value = 1; + optional float float_value = 2; + optional int64 int64_value = 3; + optional uint64 uint64_value = 4; + optional int32 int32_value = 5; + optional fixed64 fixed64_value = 6; + optional fixed32 fixed32_value = 7; + optional bool bool_value = 8; + optional string string_value = 9; + optional bytes bytes_value = 12; + optional uint32 uint32_value = 13; + optional sfixed32 sfixed32_value = 15; + optional sfixed64 sfixed64_value = 16; + optional sint32 sint32_value = 17; + optional sint64 sint64_value = 18; +} + +// NOTE: This definition must be kept in sync with PackedPrimitiveValue. +message RepeatedPrimitiveValue { + repeated double double_value = 1; + repeated float float_value = 2; + repeated int64 int64_value = 3; + repeated uint64 uint64_value = 4; + repeated int32 int32_value = 5; + repeated fixed64 fixed64_value = 6; + repeated fixed32 fixed32_value = 7; + repeated bool bool_value = 8; + repeated string string_value = 9; + repeated bytes bytes_value = 12; + repeated uint32 uint32_value = 13; + repeated sfixed32 sfixed32_value = 15; + repeated sfixed64 sfixed64_value = 16; + repeated sint32 sint32_value = 17; + repeated sint64 sint64_value = 18; + repeated PrimitiveValue message_value = 19; +} + +// A PackedPrimitiveValue looks exactly the same as a RepeatedPrimitiveValue +// in the text format, but the binary serializion is different. +// We test the packed representations by loading the same test cases +// using this definition instead of RepeatedPrimitiveValue. +// NOTE: This definition must be kept in sync with RepeatedPrimitiveValue +// in every way except the packed=true declaration. +message PackedPrimitiveValue { + repeated double double_value = 1 [packed = true]; + repeated float float_value = 2 [packed = true]; + repeated int64 int64_value = 3 [packed = true]; + repeated uint64 uint64_value = 4 [packed = true]; + repeated int32 int32_value = 5 [packed = true]; + repeated fixed64 fixed64_value = 6 [packed = true]; + repeated fixed32 fixed32_value = 7 [packed = true]; + repeated bool bool_value = 8 [packed = true]; + repeated string string_value = 9; + repeated bytes bytes_value = 12; + repeated uint32 uint32_value = 13 [packed = true]; + repeated sfixed32 sfixed32_value = 15 [packed = true]; + repeated sfixed64 sfixed64_value = 16 [packed = true]; + repeated sint32 sint32_value = 17 [packed = true]; + repeated sint64 sint64_value = 18 [packed = true]; + repeated PrimitiveValue message_value = 19; +} + +message EnumValue { + enum Color { + RED = 0; + ORANGE = 1; + YELLOW = 2; + GREEN = 3; + BLUE = 4; + INDIGO = 5; + VIOLET = 6; + }; + optional Color enum_value = 14; + repeated Color repeated_enum_value = 15; +} + + +message InnerMessageValue { + optional float float_value = 2; + repeated bytes bytes_values = 8; +} + +message MiddleMessageValue { + repeated int32 int32_values = 5; + optional InnerMessageValue message_value = 11; + optional uint32 uint32_value = 13; +} + +message MessageValue { + optional double double_value = 1; + optional MiddleMessageValue message_value = 11; +} + +message RepeatedMessageValue { + message NestedMessageValue { + optional float float_value = 2; + repeated bytes bytes_values = 8; + } + + repeated NestedMessageValue message_values = 11; +} + +// Message containing fields with field numbers higher than any field above. An +// instance of this message is prepended to each binary message in the test to +// exercise the code path that handles fields encoded out of order of field +// number. +message ExtraFields { + optional string string_value = 1776; + optional bool bool_value = 1777; +} diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD index 376644718f..a0bae23a7c 100644 --- a/tensorflow/tools/pip_package/BUILD +++ b/tensorflow/tools/pip_package/BUILD @@ -74,6 +74,7 @@ COMMON_PIP_DEPS = [ "//tensorflow/contrib/labeled_tensor:labeled_tensor_pip", "//tensorflow/contrib/nn:nn_py", "//tensorflow/contrib/predictor:predictor_pip", + "//tensorflow/contrib/proto:proto_pip", "//tensorflow/contrib/receptive_field:receptive_field_pip", "//tensorflow/contrib/session_bundle:session_bundle_pip", "//tensorflow/contrib/signal:signal_py", -- GitLab From eed6828acf19260279b38a7fbaf79141c813f795 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 14:02:49 -0700 Subject: [PATCH 184/791] BREAKING_CHANGE: Remove event_ndims in Bijector, and require `log_det_jacobian` methods to take event_ndims. The class level event_ndims parameter is being deprecated in favor of passing it in to the `log_det_jacobian` methods. Specific changes: - `log_det_jacobian` signatures are now `log_det_jacobian(input, event_ndims)` - Constructors no long have event_ndims passed in (e.g. Affine() vs. Affine(event_ndims=0)). - All bijectors must specify a subset of [forward_min_event_ndims, inverse_min_event_ndims]. This is the minimal dimensionality the bijector operates on, with it being "broadcasted" to any passed in event_ndims (e.g. Exp has forward_min_event_ndims = 0. That means it operates on scalars. However, we can use the bijector on any event_ndims > 0 (i.e. we've broadcasted the transformation to work on any amount of event_ndims > 0), and jacobian reduction will work in those cases. As a result of this change, all bijectors should "broadcast" (e.g. Sigmoid now works on any number of event_ndims). Other changes (internal and documentation): - Added clarifications on Jacobian Determinant vs. Jacobian Matrix. - Added clarifications on min_event_ndims, and what the jacobian reduction is over. - Changed caching of ildj to be keyed on event_ndims. - Several bug fixes to bugs unearthed while writing this code (e.g. transformed distribution shape computation being incorrect) PiperOrigin-RevId: 192504919 --- .../bijectors/absolute_value_test.py | 35 +- .../bijectors/affine_linear_operator_test.py | 30 +- .../bijectors/affine_scalar_test.py | 65 +-- .../kernel_tests/bijectors/affine_test.py | 231 ++++++---- .../bijectors/batch_normalization_test.py | 5 +- .../kernel_tests/bijectors/chain_test.py | 132 +++++- .../bijectors/cholesky_outer_product_test.py | 9 +- .../bijectors/conditional_bijector_test.py | 12 +- .../python/kernel_tests/bijectors/exp_test.py | 18 +- .../kernel_tests/bijectors/gumbel_test.py | 16 +- .../kernel_tests/bijectors/inline_test.py | 18 +- .../kernel_tests/bijectors/invert_test.py | 12 +- .../bijectors/kumaraswamy_bijector_test.py | 15 +- .../bijectors/masked_autoregressive_test.py | 5 +- .../kernel_tests/bijectors/permute_test.py | 11 +- .../bijectors/power_transform_test.py | 17 +- .../kernel_tests/bijectors/real_nvp_test.py | 12 +- .../kernel_tests/bijectors/reshape_test.py | 7 +- .../kernel_tests/bijectors/sigmoid_test.py | 16 +- .../bijectors/sinh_arcsinh_bijector_test.py | 22 +- .../bijectors/softmax_centered_test.py | 14 +- .../kernel_tests/bijectors/softplus_test.py | 40 +- .../kernel_tests/bijectors/square_test.py | 7 +- .../kernel_tests/bijectors/weibull_test.py | 16 +- ...nditional_transformed_distribution_test.py | 3 +- .../python/kernel_tests/mvn_diag_test.py | 2 +- .../transformed_distribution_test.py | 121 ++++- .../kernel_tests/vector_laplace_diag_test.py | 2 +- .../python/ops/bijectors/absolute_value.py | 29 +- .../python/ops/bijectors/affine.py | 10 +- .../ops/bijectors/affine_linear_operator.py | 36 +- .../python/ops/bijectors/affine_scalar.py | 13 +- .../ops/bijectors/batch_normalization.py | 6 +- .../python/ops/bijectors/chain.py | 157 ++++++- .../ops/bijectors/cholesky_outer_product.py | 2 +- .../ops/bijectors/conditional_bijector.py | 12 +- .../distributions/python/ops/bijectors/exp.py | 10 +- .../python/ops/bijectors/gumbel.py | 15 +- .../python/ops/bijectors/inline.py | 15 +- .../python/ops/bijectors/invert.py | 3 +- .../python/ops/bijectors/kumaraswamy.py | 27 +- .../ops/bijectors/masked_autoregressive.py | 3 +- .../python/ops/bijectors/permute.py | 8 +- .../python/ops/bijectors/power_transform.py | 16 +- .../python/ops/bijectors/real_nvp.py | 4 +- .../python/ops/bijectors/reshape.py | 8 +- .../python/ops/bijectors/sigmoid.py | 4 +- .../python/ops/bijectors/sinh_arcsinh.py | 29 +- .../python/ops/bijectors/softmax_centered.py | 12 +- .../python/ops/bijectors/softplus.py | 11 +- .../python/ops/bijectors/square.py | 2 +- .../python/ops/bijectors/weibull.py | 17 +- .../conditional_transformed_distribution.py | 21 +- .../python/ops/poisson_lognormal.py | 2 +- .../python/ops/relaxed_onehot_categorical.py | 2 +- .../distributions/python/ops/sinh_arcsinh.py | 4 +- .../python/ops/vector_diffeomixture.py | 10 +- .../python/ops/vector_sinh_arcsinh_diag.py | 4 +- .../distributions/bijector_test.py | 181 ++++++-- .../distributions/identity_bijector_test.py | 21 +- .../python/ops/distributions/bijector_impl.py | 429 +++++++++++++----- .../ops/distributions/bijector_test_util.py | 23 +- .../python/ops/distributions/bijectors.py | 31 -- .../python/ops/distributions/distributions.py | 2 - .../ops/distributions/identity_bijector.py | 8 +- .../distributions/transformed_distribution.py | 58 ++- ...ow.distributions.bijectors.-bijector.pbtxt | 65 --- ...ow.distributions.bijectors.-identity.pbtxt | 66 --- .../tensorflow.distributions.bijectors.pbtxt | 11 - .../api/golden/tensorflow.distributions.pbtxt | 4 - 70 files changed, 1412 insertions(+), 872 deletions(-) delete mode 100644 tensorflow/python/ops/distributions/bijectors.py delete mode 100644 tensorflow/tools/api/golden/tensorflow.distributions.bijectors.-bijector.pbtxt delete mode 100644 tensorflow/tools/api/golden/tensorflow.distributions.bijectors.-identity.pbtxt delete mode 100644 tensorflow/tools/api/golden/tensorflow.distributions.bijectors.pbtxt diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/absolute_value_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/absolute_value_test.py index e0d65c79b2..042c8ebd51 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/absolute_value_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/absolute_value_test.py @@ -18,11 +18,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import numpy as np - # pylint: disable=g-importing-member from tensorflow.contrib.distributions.python.ops.bijectors.absolute_value import AbsoluteValue -from tensorflow.python.framework import dtypes from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.platform import test @@ -35,50 +32,38 @@ class AbsoluteValueTest(test.TestCase): def testBijectorVersusNumpyRewriteOfBasicFunctionsEventNdims0(self): with self.test_session() as sess: - bijector = AbsoluteValue(event_ndims=0, validate_args=True) + bijector = AbsoluteValue(validate_args=True) self.assertEqual("absolute_value", bijector.name) x = array_ops.constant([[0., 1., -1], [0., -5., 3.]]) # Shape [2, 3] y = math_ops.abs(x) y_ = y.eval() - zeros = np.zeros((2, 3)) self.assertAllClose(y_, bijector.forward(x).eval()) self.assertAllClose((-y_, y_), sess.run(bijector.inverse(y))) - self.assertAllClose((zeros, zeros), - sess.run(bijector.inverse_log_det_jacobian(y))) + self.assertAllClose((0., 0.), + sess.run(bijector.inverse_log_det_jacobian( + y, event_ndims=0))) # Run things twice to make sure there are no issues in caching the tuples # returned by .inverse* self.assertAllClose(y_, bijector.forward(x).eval()) self.assertAllClose((-y_, y_), sess.run(bijector.inverse(y))) - self.assertAllClose((zeros, zeros), - sess.run(bijector.inverse_log_det_jacobian(y))) - - def testEventNdimsMustBeZeroOrRaiseStatic(self): - with self.test_session(): - with self.assertRaisesRegexp(ValueError, "event_ndims.*was not 0"): - AbsoluteValue(event_ndims=1) - - def testEventNdimsMustBeZeroOrRaiseDynamic(self): - with self.test_session() as sess: - event_ndims = array_ops.placeholder(dtypes.int32) - abs_bijector = AbsoluteValue(event_ndims=event_ndims, validate_args=True) - with self.assertRaisesOpError("event_ndims was not 0"): - sess.run(abs_bijector.inverse_log_det_jacobian([1.]), - feed_dict={event_ndims: 1}) + self.assertAllClose((0., 0.), + sess.run(bijector.inverse_log_det_jacobian( + y, event_ndims=0))) def testNegativeYRaisesForInverseIfValidateArgs(self): with self.test_session() as sess: - bijector = AbsoluteValue(event_ndims=0, validate_args=True) + bijector = AbsoluteValue(validate_args=True) with self.assertRaisesOpError("y was negative"): sess.run(bijector.inverse(-1.)) def testNegativeYRaisesForILDJIfValidateArgs(self): with self.test_session() as sess: - bijector = AbsoluteValue(event_ndims=0, validate_args=True) + bijector = AbsoluteValue(validate_args=True) with self.assertRaisesOpError("y was negative"): - sess.run(bijector.inverse_log_det_jacobian(-1.)) + sess.run(bijector.inverse_log_det_jacobian(-1., event_ndims=0)) if __name__ == "__main__": diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_linear_operator_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_linear_operator_test.py index 405ddd292c..1e4ad724d0 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_linear_operator_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_linear_operator_test.py @@ -38,9 +38,11 @@ class AffineLinearOperatorTest(test.TestCase): self.assertEqual(affine.name, "affine_linear_operator") self.assertAllClose(y, affine.forward(x).eval()) self.assertAllClose(x, affine.inverse(y).eval()) - self.assertAllClose(ildj, affine.inverse_log_det_jacobian(y).eval()) - self.assertAllClose(-affine.inverse_log_det_jacobian(y).eval(), - affine.forward_log_det_jacobian(x).eval()) + self.assertAllClose(ildj, affine.inverse_log_det_jacobian( + y, event_ndims=2).eval()) + self.assertAllClose( + -affine.inverse_log_det_jacobian(y, event_ndims=2).eval(), + affine.forward_log_det_jacobian(x, event_ndims=2).eval()) def testDiag(self): with self.test_session(): @@ -58,14 +60,16 @@ class AffineLinearOperatorTest(test.TestCase): self.assertEqual(affine.name, "affine_linear_operator") self.assertAllClose(y, affine.forward(x).eval()) self.assertAllClose(x, affine.inverse(y).eval()) - self.assertAllClose(ildj, affine.inverse_log_det_jacobian(y).eval()) - self.assertAllClose(-affine.inverse_log_det_jacobian(y).eval(), - affine.forward_log_det_jacobian(x).eval()) + self.assertAllClose( + ildj, affine.inverse_log_det_jacobian(y, event_ndims=1).eval()) + self.assertAllClose( + -affine.inverse_log_det_jacobian(y, event_ndims=1).eval(), + affine.forward_log_det_jacobian(x, event_ndims=1).eval()) def testTriL(self): with self.test_session(): shift = np.array([-1, 0, 1], dtype=np.float32) - tril = np.array([[[1, 0, 0], + tril = np.array([[[3, 0, 0], [2, -1, 0], [3, 2, 1]], [[2, 0, 0], @@ -85,15 +89,17 @@ class AffineLinearOperatorTest(test.TestCase): # y = np.matmul(x, tril) + shift. y = np.squeeze(np.matmul(tril, np.expand_dims(x, -1)), -1) + shift ildj = -np.sum(np.log(np.abs(np.diagonal( - tril, axis1=-2, axis2=-1))), - axis=-1) + tril, axis1=-2, axis2=-1)))) self.assertEqual(affine.name, "affine_linear_operator") self.assertAllClose(y, affine.forward(x).eval()) self.assertAllClose(x, affine.inverse(y).eval()) - self.assertAllClose(ildj, affine.inverse_log_det_jacobian(y).eval()) - self.assertAllClose(-affine.inverse_log_det_jacobian(y).eval(), - affine.forward_log_det_jacobian(x).eval()) + self.assertAllClose( + ildj, affine.inverse_log_det_jacobian( + y, event_ndims=2).eval()) + self.assertAllClose( + -affine.inverse_log_det_jacobian(y, event_ndims=2).eval(), + affine.forward_log_det_jacobian(x, event_ndims=2).eval()) if __name__ == "__main__": diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_scalar_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_scalar_test.py index 16173a166f..d2533620be 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_scalar_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_scalar_test.py @@ -40,13 +40,13 @@ class AffineScalarBijectorTest(test.TestCase): def testNoBatchScalar(self): with self.test_session() as sess: - def static_run(fun, x): - return fun(x).eval() + def static_run(fun, x, **kwargs): + return fun(x, **kwargs).eval() - def dynamic_run(fun, x_value): + def dynamic_run(fun, x_value, **kwargs): x_value = np.array(x_value) x = array_ops.placeholder(dtypes.float32, name="x") - return sess.run(fun(x), feed_dict={x: x_value}) + return sess.run(fun(x, **kwargs), feed_dict={x: x_value}) for run in (static_run, dynamic_run): mu = -1. @@ -55,19 +55,20 @@ class AffineScalarBijectorTest(test.TestCase): x = [1., 2, 3] # Three scalar samples (no batches). self.assertAllClose([1., 3, 5], run(bijector.forward, x)) self.assertAllClose([1., 1.5, 2.], run(bijector.inverse, x)) - self.assertAllClose([-np.log(2.)] * 3, - run(bijector.inverse_log_det_jacobian, x)) + self.assertAllClose( + -np.log(2.), + run(bijector.inverse_log_det_jacobian, x, event_ndims=0)) def testOneBatchScalarViaIdentityIn64BitUserProvidesShiftOnly(self): with self.test_session() as sess: - def static_run(fun, x): - return fun(x).eval() + def static_run(fun, x, **kwargs): + return fun(x, **kwargs).eval() - def dynamic_run(fun, x_value): + def dynamic_run(fun, x_value, **kwargs): x_value = np.array(x_value).astype(np.float64) x = array_ops.placeholder(dtypes.float64, name="x") - return sess.run(fun(x), feed_dict={x: x_value}) + return sess.run(fun(x, **kwargs), feed_dict={x: x_value}) for run in (static_run, dynamic_run): mu = np.float64([1.]) @@ -77,18 +78,20 @@ class AffineScalarBijectorTest(test.TestCase): x = np.float64([1.]) # One sample from one batches. self.assertAllClose([2.], run(bijector.forward, x)) self.assertAllClose([0.], run(bijector.inverse, x)) - self.assertAllClose([0.], run(bijector.inverse_log_det_jacobian, x)) + self.assertAllClose( + 0., + run(bijector.inverse_log_det_jacobian, x, event_ndims=0)) def testOneBatchScalarViaIdentityIn64BitUserProvidesScaleOnly(self): with self.test_session() as sess: - def static_run(fun, x): - return fun(x).eval() + def static_run(fun, x, **kwargs): + return fun(x, **kwargs).eval() - def dynamic_run(fun, x_value): + def dynamic_run(fun, x_value, **kwargs): x_value = np.array(x_value).astype(np.float64) x = array_ops.placeholder(dtypes.float64, name="x") - return sess.run(fun(x), feed_dict={x: x_value}) + return sess.run(fun(x, **kwargs), feed_dict={x: x_value}) for run in (static_run, dynamic_run): multiplier = np.float64([2.]) @@ -98,19 +101,20 @@ class AffineScalarBijectorTest(test.TestCase): x = np.float64([1.]) # One sample from one batches. self.assertAllClose([2.], run(bijector.forward, x)) self.assertAllClose([0.5], run(bijector.inverse, x)) - self.assertAllClose([np.log(0.5)], - run(bijector.inverse_log_det_jacobian, x)) + self.assertAllClose( + [np.log(0.5)], + run(bijector.inverse_log_det_jacobian, x, event_ndims=0)) def testTwoBatchScalarIdentityViaIdentity(self): with self.test_session() as sess: - def static_run(fun, x): - return fun(x).eval() + def static_run(fun, x, **kwargs): + return fun(x, **kwargs).eval() - def dynamic_run(fun, x_value): - x_value = np.array(x_value) + def dynamic_run(fun, x_value, **kwargs): + x_value = np.array(x_value).astype(np.float32) x = array_ops.placeholder(dtypes.float32, name="x") - return sess.run(fun(x), feed_dict={x: x_value}) + return sess.run(fun(x, **kwargs), feed_dict={x: x_value}) for run in (static_run, dynamic_run): mu = [1., -1] @@ -120,18 +124,20 @@ class AffineScalarBijectorTest(test.TestCase): x = [1., 1] # One sample from each of two batches. self.assertAllClose([2., 0], run(bijector.forward, x)) self.assertAllClose([0., 2], run(bijector.inverse, x)) - self.assertAllClose([0., 0.], run(bijector.inverse_log_det_jacobian, x)) + self.assertAllClose( + 0., + run(bijector.inverse_log_det_jacobian, x, event_ndims=0)) def testTwoBatchScalarIdentityViaScale(self): with self.test_session() as sess: - def static_run(fun, x): - return fun(x).eval() + def static_run(fun, x, **kwargs): + return fun(x, **kwargs).eval() - def dynamic_run(fun, x_value): - x_value = np.array(x_value) + def dynamic_run(fun, x_value, **kwargs): + x_value = np.array(x_value).astype(np.float32) x = array_ops.placeholder(dtypes.float32, name="x") - return sess.run(fun(x), feed_dict={x: x_value}) + return sess.run(fun(x, **kwargs), feed_dict={x: x_value}) for run in (static_run, dynamic_run): mu = [1., -1] @@ -142,7 +148,8 @@ class AffineScalarBijectorTest(test.TestCase): self.assertAllClose([3., 0], run(bijector.forward, x)) self.assertAllClose([0., 2], run(bijector.inverse, x)) self.assertAllClose( - [-np.log(2), 0.], run(bijector.inverse_log_det_jacobian, x)) + [-np.log(2), 0.], + run(bijector.inverse_log_det_jacobian, x, event_ndims=0)) def testScalarCongruency(self): with self.test_session(): diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_test.py index 077e6176b4..9e14b9a53e 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_test.py @@ -40,14 +40,15 @@ class AffineBijectorTest(test.TestCase): def testNoBatchMultivariateIdentity(self): with self.test_session() as sess: + placeholder = array_ops.placeholder(dtypes.float32, name="x") - def static_run(fun, x): - return fun(x).eval() + def static_run(fun, x, **kwargs): + return fun(x, **kwargs).eval() - def dynamic_run(fun, x_value): + def dynamic_run(fun, x_value, **kwargs): x_value = np.array(x_value) - x = array_ops.placeholder(dtypes.float32, name="x") - return sess.run(fun(x), feed_dict={x: x_value}) + return sess.run( + fun(placeholder, **kwargs), feed_dict={placeholder: x_value}) for run in (static_run, dynamic_run): mu = [1., -1] @@ -66,18 +67,20 @@ class AffineBijectorTest(test.TestCase): x = [[1., 1], [-1., -1]] self.assertAllClose([[2., 0], [0., -2]], run(bijector.forward, x)) self.assertAllClose([[0., 2], [-2., 0]], run(bijector.inverse, x)) - self.assertAllClose(0., run(bijector.inverse_log_det_jacobian, x)) + self.assertAllClose( + 0., run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) def testNoBatchMultivariateDiag(self): with self.test_session() as sess: + placeholder = array_ops.placeholder(dtypes.float32, name="x") - def static_run(fun, x): - return fun(x).eval() + def static_run(fun, x, **kwargs): + return fun(x, **kwargs).eval() - def dynamic_run(fun, x_value): + def dynamic_run(fun, x_value, **kwargs): x_value = np.array(x_value) - x = array_ops.placeholder(dtypes.float32, name="x") - return sess.run(fun(x), feed_dict={x: x_value}) + return sess.run( + fun(placeholder, **kwargs), feed_dict={placeholder: x_value}) for run in (static_run, dynamic_run): mu = [1., -1] @@ -89,9 +92,12 @@ class AffineBijectorTest(test.TestCase): # = [-1, -1] + [1, -1] self.assertAllClose([3., 0], run(bijector.forward, x)) self.assertAllClose([0., 2], run(bijector.inverse, x)) - self.assertAllClose(-np.log(2.), - run(bijector.inverse_log_det_jacobian, x)) + self.assertAllClose( + -np.log(2.), + run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) + # Reset bijector. + bijector = Affine(shift=mu, scale_diag=[2., 1]) # x is a 2-batch of 2-vectors. # The first vector is [1, 1], the second is [-1, -1]. # Each undergoes matmul(sigma, x) + shift. @@ -103,8 +109,9 @@ class AffineBijectorTest(test.TestCase): self.assertAllClose([[0., 2], [-1., 0]], run(bijector.inverse, x)) - self.assertAllClose(-np.log(2.), - run(bijector.inverse_log_det_jacobian, x)) + self.assertAllClose( + -np.log(2.), + run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) def testNoBatchMultivariateFullDynamic(self): with self.test_session() as sess: @@ -126,18 +133,20 @@ class AffineBijectorTest(test.TestCase): self.assertAllClose([[0., 1]], sess.run(bijector.inverse(x), feed_dict)) self.assertAllClose( -np.log(4), - sess.run(bijector.inverse_log_det_jacobian(x), feed_dict)) + sess.run(bijector.inverse_log_det_jacobian(x, event_ndims=1), + feed_dict)) def testBatchMultivariateIdentity(self): with self.test_session() as sess: + placeholder = array_ops.placeholder(dtypes.float32, name="x") - def static_run(fun, x): - return fun(x).eval() + def static_run(fun, x, **kwargs): + return fun(x, **kwargs).eval() - def dynamic_run(fun, x_value): - x_value = np.array(x_value, dtype=np.float32) - x = array_ops.placeholder(dtypes.float32, name="x") - return sess.run(fun(x), feed_dict={x: x_value}) + def dynamic_run(fun, x_value, **kwargs): + x_value = np.array(x_value) + return sess.run( + fun(placeholder, **kwargs), feed_dict={placeholder: x_value}) for run in (static_run, dynamic_run): mu = [[1., -1]] @@ -147,19 +156,21 @@ class AffineBijectorTest(test.TestCase): x = [[[1., 1]]] self.assertAllClose([[[3., 1]]], run(bijector.forward, x)) self.assertAllClose([[[0., 1]]], run(bijector.inverse, x)) - self.assertAllClose(-np.log(4), - run(bijector.inverse_log_det_jacobian, x)) + self.assertAllClose( + -np.log(4), + run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) def testBatchMultivariateDiag(self): with self.test_session() as sess: + placeholder = array_ops.placeholder(dtypes.float32, name="x") - def static_run(fun, x): - return fun(x).eval() + def static_run(fun, x, **kwargs): + return fun(x, **kwargs).eval() - def dynamic_run(fun, x_value): - x_value = np.array(x_value, dtype=np.float32) - x = array_ops.placeholder(dtypes.float32, name="x") - return sess.run(fun(x), feed_dict={x: x_value}) + def dynamic_run(fun, x_value, **kwargs): + x_value = np.array(x_value) + return sess.run( + fun(placeholder, **kwargs), feed_dict={placeholder: x_value}) for run in (static_run, dynamic_run): mu = [[1., -1]] @@ -169,8 +180,9 @@ class AffineBijectorTest(test.TestCase): x = [[[1., 1]]] self.assertAllClose([[[3., 1]]], run(bijector.forward, x)) self.assertAllClose([[[0., 1]]], run(bijector.inverse, x)) - self.assertAllClose([-np.log(4)], - run(bijector.inverse_log_det_jacobian, x)) + self.assertAllClose( + [-np.log(4)], + run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) def testBatchMultivariateFullDynamic(self): with self.test_session() as sess: @@ -191,20 +203,22 @@ class AffineBijectorTest(test.TestCase): bijector = Affine(shift=mu, scale_diag=scale_diag) self.assertAllClose([[[3., 1]]], sess.run(bijector.forward(x), feed_dict)) self.assertAllClose([[[0., 1]]], sess.run(bijector.inverse(x), feed_dict)) - self.assertAllClose([-np.log(4)], - sess.run( - bijector.inverse_log_det_jacobian(x), feed_dict)) + self.assertAllClose( + [-np.log(4)], + sess.run(bijector.inverse_log_det_jacobian( + x, event_ndims=1), feed_dict)) def testIdentityWithDiagUpdate(self): with self.test_session() as sess: + placeholder = array_ops.placeholder(dtypes.float32, name="x") - def static_run(fun, x): - return fun(x).eval() + def static_run(fun, x, **kwargs): + return fun(x, **kwargs).eval() - def dynamic_run(fun, x_value): + def dynamic_run(fun, x_value, **kwargs): x_value = np.array(x_value) - x = array_ops.placeholder(dtypes.float32, name="x") - return sess.run(fun(x), feed_dict={x: x_value}) + return sess.run( + fun(placeholder, **kwargs), feed_dict={placeholder: x_value}) for run in (static_run, dynamic_run): mu = -1. @@ -216,19 +230,21 @@ class AffineBijectorTest(test.TestCase): x = [1., 2, 3] # Three scalar samples (no batches). self.assertAllClose([1., 3, 5], run(bijector.forward, x)) self.assertAllClose([1., 1.5, 2.], run(bijector.inverse, x)) - self.assertAllClose(-np.log(2.**3), - run(bijector.inverse_log_det_jacobian, x)) + self.assertAllClose( + -np.log(2.**3), + run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) def testIdentityWithTriL(self): with self.test_session() as sess: + placeholder = array_ops.placeholder(dtypes.float32, name="x") - def static_run(fun, x): - return fun(x).eval() + def static_run(fun, x, **kwargs): + return fun(x, **kwargs).eval() - def dynamic_run(fun, x_value): + def dynamic_run(fun, x_value, **kwargs): x_value = np.array(x_value) - x = array_ops.placeholder(dtypes.float32, name="x") - return sess.run(fun(x), feed_dict={x: x_value}) + return sess.run( + fun(placeholder, **kwargs), feed_dict={placeholder: x_value}) for run in (static_run, dynamic_run): mu = -1. @@ -240,19 +256,21 @@ class AffineBijectorTest(test.TestCase): x = [[1., 2]] # One multivariate sample. self.assertAllClose([[1., 5]], run(bijector.forward, x)) self.assertAllClose([[1., 0.5]], run(bijector.inverse, x)) - self.assertAllClose(-np.log(4.), - run(bijector.inverse_log_det_jacobian, x)) + self.assertAllClose( + -np.log(4.), + run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) def testDiagWithTriL(self): with self.test_session() as sess: + placeholder = array_ops.placeholder(dtypes.float32, name="x") - def static_run(fun, x): - return fun(x).eval() + def static_run(fun, x, **kwargs): + return fun(x, **kwargs).eval() - def dynamic_run(fun, x_value): + def dynamic_run(fun, x_value, **kwargs): x_value = np.array(x_value) - x = array_ops.placeholder(dtypes.float32, name="x") - return sess.run(fun(x), feed_dict={x: x_value}) + return sess.run( + fun(placeholder, **kwargs), feed_dict={placeholder: x_value}) for run in (static_run, dynamic_run): mu = -1. @@ -262,19 +280,21 @@ class AffineBijectorTest(test.TestCase): x = [[1., 2]] # One multivariate sample. self.assertAllClose([[1., 7]], run(bijector.forward, x)) self.assertAllClose([[1., 1 / 3.]], run(bijector.inverse, x)) - self.assertAllClose(-np.log(6.), - run(bijector.inverse_log_det_jacobian, x)) + self.assertAllClose( + -np.log(6.), + run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) def testIdentityAndDiagWithTriL(self): with self.test_session() as sess: + placeholder = array_ops.placeholder(dtypes.float32, name="x") - def static_run(fun, x): - return fun(x).eval() + def static_run(fun, x, **kwargs): + return fun(x, **kwargs).eval() - def dynamic_run(fun, x_value): + def dynamic_run(fun, x_value, **kwargs): x_value = np.array(x_value) - x = array_ops.placeholder(dtypes.float32, name="x") - return sess.run(fun(x), feed_dict={x: x_value}) + return sess.run( + fun(placeholder, **kwargs), feed_dict={placeholder: x_value}) for run in (static_run, dynamic_run): mu = -1. @@ -287,19 +307,21 @@ class AffineBijectorTest(test.TestCase): x = [[1., 2]] # One multivariate sample. self.assertAllClose([[2., 9]], run(bijector.forward, x)) self.assertAllClose([[2 / 3., 5 / 12.]], run(bijector.inverse, x)) - self.assertAllClose(-np.log(12.), - run(bijector.inverse_log_det_jacobian, x)) + self.assertAllClose( + -np.log(12.), + run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) def testIdentityWithVDVTUpdate(self): with self.test_session() as sess: + placeholder = array_ops.placeholder(dtypes.float32, name="x") - def static_run(fun, x): - return fun(x).eval() + def static_run(fun, x, **kwargs): + return fun(x, **kwargs).eval() - def dynamic_run(fun, x_value): + def dynamic_run(fun, x_value, **kwargs): x_value = np.array(x_value) - x = array_ops.placeholder(dtypes.float32, name="x") - return sess.run(fun(x), feed_dict={x: x_value}) + return sess.run( + fun(placeholder, **kwargs), feed_dict={placeholder: x_value}) for run in (static_run, dynamic_run): mu = -1. @@ -319,22 +341,24 @@ class AffineBijectorTest(test.TestCase): self.assertAllClose([0.2, 1.5, 4 / 3.], run(bijector.inverse, x)) self.assertAllClose( run(bijector_ref.inverse, x), run(bijector.inverse, x)) - self.assertAllClose(-np.log(60.), - run(bijector.inverse_log_det_jacobian, x)) self.assertAllClose( - run(bijector.inverse_log_det_jacobian, x), - run(bijector_ref.inverse_log_det_jacobian, x)) + -np.log(60.), + run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) + self.assertAllClose( + run(bijector.inverse_log_det_jacobian, x, event_ndims=1), + run(bijector_ref.inverse_log_det_jacobian, x, event_ndims=1)) def testDiagWithVDVTUpdate(self): with self.test_session() as sess: + placeholder = array_ops.placeholder(dtypes.float32, name="x") - def static_run(fun, x): - return fun(x).eval() + def static_run(fun, x, **kwargs): + return fun(x, **kwargs).eval() - def dynamic_run(fun, x_value): + def dynamic_run(fun, x_value, **kwargs): x_value = np.array(x_value) - x = array_ops.placeholder(dtypes.float32, name="x") - return sess.run(fun(x), feed_dict={x: x_value}) + return sess.run( + fun(placeholder, **kwargs), feed_dict={placeholder: x_value}) for run in (static_run, dynamic_run): mu = -1. @@ -353,22 +377,24 @@ class AffineBijectorTest(test.TestCase): self.assertAllClose([0.2, 1., 0.8], run(bijector.inverse, x)) self.assertAllClose( run(bijector_ref.inverse, x), run(bijector.inverse, x)) - self.assertAllClose(-np.log(150.), - run(bijector.inverse_log_det_jacobian, x)) self.assertAllClose( - run(bijector.inverse_log_det_jacobian, x), - run(bijector_ref.inverse_log_det_jacobian, x)) + -np.log(150.), + run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) + self.assertAllClose( + run(bijector.inverse_log_det_jacobian, x, event_ndims=1), + run(bijector_ref.inverse_log_det_jacobian, x, event_ndims=1)) def testTriLWithVDVTUpdate(self): with self.test_session() as sess: + placeholder = array_ops.placeholder(dtypes.float32, name="x") - def static_run(fun, x): - return fun(x).eval() + def static_run(fun, x, **kwargs): + return fun(x, **kwargs).eval() - def dynamic_run(fun, x_value): + def dynamic_run(fun, x_value, **kwargs): x_value = np.array(x_value) - x = array_ops.placeholder(dtypes.float32, name="x") - return sess.run(fun(x), feed_dict={x: x_value}) + return sess.run( + fun(placeholder, **kwargs), feed_dict={placeholder: x_value}) for run in (static_run, dynamic_run): mu = -1. @@ -388,22 +414,24 @@ class AffineBijectorTest(test.TestCase): self.assertAllClose([0.2, 14 / 15., 4 / 25.], run(bijector.inverse, x)) self.assertAllClose( run(bijector_ref.inverse, x), run(bijector.inverse, x)) - self.assertAllClose(-np.log(150.), - run(bijector.inverse_log_det_jacobian, x)) self.assertAllClose( - run(bijector.inverse_log_det_jacobian, x), - run(bijector_ref.inverse_log_det_jacobian, x)) + -np.log(150.), + run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) + self.assertAllClose( + run(bijector.inverse_log_det_jacobian, x, event_ndims=1), + run(bijector_ref.inverse_log_det_jacobian, x, event_ndims=1)) def testTriLWithVDVTUpdateNoDiagonal(self): with self.test_session() as sess: + placeholder = array_ops.placeholder(dtypes.float32, name="x") - def static_run(fun, x): - return fun(x).eval() + def static_run(fun, x, **kwargs): + return fun(x, **kwargs).eval() - def dynamic_run(fun, x_value): + def dynamic_run(fun, x_value, **kwargs): x_value = np.array(x_value) - x = array_ops.placeholder(dtypes.float32, name="x") - return sess.run(fun(x), feed_dict={x: x_value}) + return sess.run( + fun(placeholder, **kwargs), feed_dict={placeholder: x_value}) for run in (static_run, dynamic_run): mu = -1. @@ -423,11 +451,12 @@ class AffineBijectorTest(test.TestCase): self.assertAllClose([1 / 3., 8 / 9., 4 / 30.], run(bijector.inverse, x)) self.assertAllClose( run(bijector_ref.inverse, x), run(bijector.inverse, x)) - self.assertAllClose(-np.log(90.), - run(bijector.inverse_log_det_jacobian, x)) self.assertAllClose( - run(bijector.inverse_log_det_jacobian, x), - run(bijector_ref.inverse_log_det_jacobian, x)) + -np.log(90.), + run(bijector.inverse_log_det_jacobian, x, event_ndims=1)) + self.assertAllClose( + run(bijector.inverse_log_det_jacobian, x, event_ndims=1), + run(bijector_ref.inverse_log_det_jacobian, x, event_ndims=1)) def testNoBatchMultivariateRaisesWhenSingular(self): with self.test_session(): @@ -530,6 +559,7 @@ class AffineBijectorTest(test.TestCase): backward = np.squeeze(backward, axis=-1) self.assertAllClose(backward, bijector.inverse(x).eval()) + scale *= np.ones(shape=x.shape[:-1], dtype=scale.dtype) ildj = -np.log(np.abs(np.linalg.det(scale))) # TODO(jvdillon): We need to make it so the scale_identity_multiplier # case does not deviate in expected shape. Fixing this will get rid of @@ -540,7 +570,8 @@ class AffineBijectorTest(test.TestCase): ildj = np.squeeze(ildj[0]) elif ildj.ndim < scale.ndim - 2: ildj = np.reshape(ildj, scale.shape[0:-2]) - self.assertAllClose(ildj, bijector.inverse_log_det_jacobian(x).eval()) + self.assertAllClose( + ildj, bijector.inverse_log_det_jacobian(x, event_ndims=1).eval()) def testLegalInputs(self): self._testLegalInputs( diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/batch_normalization_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/batch_normalization_test.py index a215a4a2b1..c832fcaa68 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/batch_normalization_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/batch_normalization_test.py @@ -83,10 +83,11 @@ class BatchNormTest(test_util.VectorDistributionTestHelpers, moving_mean = array_ops.identity(batch_norm.batchnorm.moving_mean) moving_var = array_ops.identity(batch_norm.batchnorm.moving_variance) denorm_x = batch_norm.forward(array_ops.identity(norm_x)) - fldj = batch_norm.forward_log_det_jacobian(x) + fldj = batch_norm.forward_log_det_jacobian( + x, event_ndims=len(event_dims)) # Use identity to invalidate cache. ildj = batch_norm.inverse_log_det_jacobian( - array_ops.identity(denorm_x)) + array_ops.identity(denorm_x), event_ndims=len(event_dims)) variables.global_variables_initializer().run() # Update variables. norm_x_ = sess.run(norm_x) diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/chain_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/chain_test.py index a748acd667..ca20442c39 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/chain_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/chain_test.py @@ -20,21 +20,33 @@ from __future__ import print_function import numpy as np +from tensorflow.contrib.distributions.python.ops.bijectors.affine import Affine from tensorflow.contrib.distributions.python.ops.bijectors.chain import Chain from tensorflow.contrib.distributions.python.ops.bijectors.exp import Exp from tensorflow.contrib.distributions.python.ops.bijectors.softmax_centered import SoftmaxCentered from tensorflow.contrib.distributions.python.ops.bijectors.softplus import Softplus from tensorflow.python.framework import tensor_shape +from tensorflow.python.ops.distributions import bijector from tensorflow.python.ops.distributions.bijector_test_util import assert_scalar_congruency from tensorflow.python.platform import test +class ShapeChanging(bijector.Bijector): + """Only used for op_ndims manipulation.""" + + def __init__(self, forward_min_event_ndims=0, inverse_min_event_ndims=3): + super(ShapeChanging, self).__init__( + forward_min_event_ndims=forward_min_event_ndims, + inverse_min_event_ndims=inverse_min_event_ndims, + validate_args=False, name="shape_changer") + + class ChainBijectorTest(test.TestCase): """Tests the correctness of the Y = Chain(bij1, bij2, bij3) transformation.""" def testBijector(self): with self.test_session(): - chain = Chain((Exp(event_ndims=1), Softplus(event_ndims=1))) + chain = Chain((Exp(), Softplus())) self.assertEqual("chain_of_exp_of_softplus", chain.name) x = np.asarray([[[1., 2.], [2., 3.]]]) @@ -42,9 +54,10 @@ class ChainBijectorTest(test.TestCase): self.assertAllClose(np.log(x - 1.), chain.inverse(x).eval()) self.assertAllClose( -np.sum(np.log(x - 1.), axis=2), - chain.inverse_log_det_jacobian(x).eval()) + chain.inverse_log_det_jacobian(x, event_ndims=1).eval()) self.assertAllClose( - np.sum(x, axis=2), chain.forward_log_det_jacobian(x).eval()) + np.sum(x, axis=2), + chain.forward_log_det_jacobian(x, event_ndims=1).eval()) def testBijectorIdentity(self): with self.test_session(): @@ -54,31 +67,126 @@ class ChainBijectorTest(test.TestCase): [2., 3.]]]) self.assertAllClose(x, chain.forward(x).eval()) self.assertAllClose(x, chain.inverse(x).eval()) - self.assertAllClose(0., chain.inverse_log_det_jacobian(x).eval()) - self.assertAllClose(0., chain.forward_log_det_jacobian(x).eval()) + self.assertAllClose( + 0., chain.inverse_log_det_jacobian(x, event_ndims=1).eval()) + self.assertAllClose( + 0., chain.forward_log_det_jacobian(x, event_ndims=1).eval()) def testScalarCongruency(self): with self.test_session(): - bijector = Chain((Exp(), Softplus())) + chain = Chain((Exp(), Softplus())) assert_scalar_congruency( - bijector, lower_x=1e-3, upper_x=1.5, rtol=0.05) + chain, lower_x=1e-3, upper_x=1.5, rtol=0.05) def testShapeGetters(self): with self.test_session(): - bijector = Chain([ + chain = Chain([ SoftmaxCentered(validate_args=True), SoftmaxCentered(validate_args=True), ]) x = tensor_shape.TensorShape([1]) y = tensor_shape.TensorShape([2 + 1]) - self.assertAllEqual(y, bijector.forward_event_shape(x)) + self.assertAllEqual(y, chain.forward_event_shape(x)) self.assertAllEqual( y.as_list(), - bijector.forward_event_shape_tensor(x.as_list()).eval()) - self.assertAllEqual(x, bijector.inverse_event_shape(y)) + chain.forward_event_shape_tensor(x.as_list()).eval()) + self.assertAllEqual(x, chain.inverse_event_shape(y)) self.assertAllEqual( x.as_list(), - bijector.inverse_event_shape_tensor(y.as_list()).eval()) + chain.inverse_event_shape_tensor(y.as_list()).eval()) + + def testMinEventNdimsChain(self): + chain = Chain([Exp(), Exp(), Exp()]) + self.assertEqual(0, chain.forward_min_event_ndims) + self.assertEqual(0, chain.inverse_min_event_ndims) + + chain = Chain([Affine(), Affine(), Affine()]) + self.assertEqual(1, chain.forward_min_event_ndims) + self.assertEqual(1, chain.inverse_min_event_ndims) + + chain = Chain([Exp(), Affine()]) + self.assertEqual(1, chain.forward_min_event_ndims) + self.assertEqual(1, chain.inverse_min_event_ndims) + + chain = Chain([Affine(), Exp()]) + self.assertEqual(1, chain.forward_min_event_ndims) + self.assertEqual(1, chain.inverse_min_event_ndims) + + chain = Chain([Affine(), Exp(), Softplus(), Affine()]) + self.assertEqual(1, chain.forward_min_event_ndims) + self.assertEqual(1, chain.inverse_min_event_ndims) + + def testMinEventNdimsShapeChangingAddDims(self): + chain = Chain([ShapeChanging()]) + self.assertEqual(0, chain.forward_min_event_ndims) + self.assertEqual(3, chain.inverse_min_event_ndims) + + chain = Chain([ShapeChanging(), Affine()]) + self.assertEqual(1, chain.forward_min_event_ndims) + self.assertEqual(4, chain.inverse_min_event_ndims) + + chain = Chain([Affine(), ShapeChanging()]) + self.assertEqual(0, chain.forward_min_event_ndims) + self.assertEqual(3, chain.inverse_min_event_ndims) + + chain = Chain([ShapeChanging(), ShapeChanging()]) + self.assertEqual(0, chain.forward_min_event_ndims) + self.assertEqual(6, chain.inverse_min_event_ndims) + + def testMinEventNdimsShapeChangingRemoveDims(self): + chain = Chain([ShapeChanging(3, 0)]) + self.assertEqual(3, chain.forward_min_event_ndims) + self.assertEqual(0, chain.inverse_min_event_ndims) + + chain = Chain([ShapeChanging(3, 0), Affine()]) + self.assertEqual(3, chain.forward_min_event_ndims) + self.assertEqual(0, chain.inverse_min_event_ndims) + + chain = Chain([Affine(), ShapeChanging(3, 0)]) + self.assertEqual(4, chain.forward_min_event_ndims) + self.assertEqual(1, chain.inverse_min_event_ndims) + + chain = Chain([ShapeChanging(3, 0), ShapeChanging(3, 0)]) + self.assertEqual(6, chain.forward_min_event_ndims) + self.assertEqual(0, chain.inverse_min_event_ndims) + + def testMinEventNdimsShapeChangingAddRemoveDims(self): + chain = Chain([ + ShapeChanging(2, 1), + ShapeChanging(3, 0), + ShapeChanging(1, 2)]) + self.assertEqual(4, chain.forward_min_event_ndims) + self.assertEqual(1, chain.inverse_min_event_ndims) + + def testChainExpAffine(self): + scale_diag = np.array([1., 2., 3.], dtype=np.float32) + chain = Chain([Exp(), Affine(scale_diag=scale_diag)]) + x = [0., np.log(2., dtype=np.float32), np.log(3., dtype=np.float32)] + y = [1., 4., 27.] + self.assertAllClose(y, self.evaluate(chain.forward(x))) + self.assertAllClose(x, self.evaluate(chain.inverse(y))) + self.assertAllClose( + np.log(6, dtype=np.float32) + np.sum(scale_diag * x), + self.evaluate(chain.forward_log_det_jacobian(x, event_ndims=1))) + + self.assertAllClose( + -np.log(6, dtype=np.float32) - np.sum(scale_diag * x), + self.evaluate(chain.inverse_log_det_jacobian(y, event_ndims=1))) + + def testChainAffineExp(self): + scale_diag = np.array([1., 2., 3.], dtype=np.float32) + chain = Chain([Affine(scale_diag=scale_diag), Exp()]) + x = [0., np.log(2., dtype=np.float32), np.log(3., dtype=np.float32)] + y = [1., 4., 9.] + self.assertAllClose(y, self.evaluate(chain.forward(x))) + self.assertAllClose(x, self.evaluate(chain.inverse(y))) + self.assertAllClose( + np.log(6, dtype=np.float32) + np.sum(x), + self.evaluate(chain.forward_log_det_jacobian(x, event_ndims=1))) + + self.assertAllClose( + -np.log(6, dtype=np.float32) - np.sum(x), + self.evaluate(chain.inverse_log_det_jacobian(y, event_ndims=1))) if __name__ == "__main__": diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/cholesky_outer_product_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/cholesky_outer_product_test.py index f392e83d2c..e281e81bdf 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/cholesky_outer_product_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/cholesky_outer_product_test.py @@ -51,10 +51,13 @@ class CholeskyOuterProductBijectorTest(test.TestCase): self.assertAllClose(y, bijector.forward(x).eval()) self.assertAllClose(x, bijector.inverse(y).eval()) self.assertAllClose( - ildj, bijector.inverse_log_det_jacobian(y).eval(), atol=0., rtol=1e-7) + ildj, bijector.inverse_log_det_jacobian( + y, event_ndims=2).eval(), atol=0., rtol=1e-7) self.assertAllClose( - -bijector.inverse_log_det_jacobian(y).eval(), - bijector.forward_log_det_jacobian(x).eval(), + -bijector.inverse_log_det_jacobian( + y, event_ndims=2).eval(), + bijector.forward_log_det_jacobian( + x, event_ndims=2).eval(), atol=0., rtol=1e-7) diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/conditional_bijector_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/conditional_bijector_test.py index 26e0d2a539..8b279ebcd9 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/conditional_bijector_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/conditional_bijector_test.py @@ -27,7 +27,7 @@ class _TestBijector(ConditionalBijector): def __init__(self): super(_TestBijector, self).__init__( - event_ndims=0, + forward_min_event_ndims=0, graph_parents=[], is_constant_jacobian=True, validate_args=False, @@ -51,11 +51,15 @@ class ConditionalBijectorTest(test.TestCase): def testConditionalBijector(self): b = _TestBijector() - for name in ["forward", "inverse", "inverse_log_det_jacobian", - "forward_log_det_jacobian"]: + for name in ["forward", "inverse"]: method = getattr(b, name) with self.assertRaisesRegexp(ValueError, name + ".*b1.*b2"): - method(1.0, arg1="b1", arg2="b2") + method(1., arg1="b1", arg2="b2") + + for name in ["inverse_log_det_jacobian", "forward_log_det_jacobian"]: + method = getattr(b, name) + with self.assertRaisesRegexp(ValueError, name + ".*b1.*b2"): + method(1., event_ndims=0., arg1="b1", arg2="b2") if __name__ == "__main__": diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/exp_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/exp_test.py index 9970c0b4d8..7be939cd27 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/exp_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/exp_test.py @@ -31,17 +31,21 @@ class ExpBijectorTest(test.TestCase): def testBijector(self): with self.test_session(): - bijector = Exp(event_ndims=1) + bijector = Exp() self.assertEqual("exp", bijector.name) x = [[[1.], [2.]]] y = np.exp(x) self.assertAllClose(y, bijector.forward(x).eval()) self.assertAllClose(x, bijector.inverse(y).eval()) self.assertAllClose( - -np.sum(np.log(y), axis=-1), - bijector.inverse_log_det_jacobian(y).eval()) - self.assertAllClose(-bijector.inverse_log_det_jacobian(np.exp(x)).eval(), - bijector.forward_log_det_jacobian(x).eval()) + -np.squeeze(np.log(y), axis=-1), + bijector.inverse_log_det_jacobian( + y, event_ndims=1).eval()) + self.assertAllClose( + -bijector.inverse_log_det_jacobian( + np.exp(x), event_ndims=1).eval(), + bijector.forward_log_det_jacobian( + x, event_ndims=1).eval()) def testScalarCongruency(self): with self.test_session(): @@ -51,10 +55,10 @@ class ExpBijectorTest(test.TestCase): def testBijectiveAndFinite(self): with self.test_session(): - bijector = Exp(event_ndims=0) + bijector = Exp() x = np.linspace(-10, 10, num=10).astype(np.float32) y = np.logspace(-10, 10, num=10).astype(np.float32) - assert_bijective_and_finite(bijector, x, y) + assert_bijective_and_finite(bijector, x, y, event_ndims=0) if __name__ == "__main__": diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/gumbel_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/gumbel_test.py index 9a905980c7..54e54c3296 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/gumbel_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/gumbel_test.py @@ -34,7 +34,7 @@ class GumbelBijectorTest(test.TestCase): with self.test_session(): loc = 0.3 scale = 5. - bijector = Gumbel(loc=loc, scale=scale, event_ndims=1, validate_args=True) + bijector = Gumbel(loc=loc, scale=scale, validate_args=True) self.assertEqual("gumbel", bijector.name) x = np.array([[[-3.], [0.], [0.5], [4.2], [12.]]], dtype=np.float32) # Gumbel distribution @@ -43,13 +43,11 @@ class GumbelBijectorTest(test.TestCase): self.assertAllClose(y, bijector.forward(x).eval()) self.assertAllClose(x, bijector.inverse(y).eval()) self.assertAllClose( - # We should lose a dimension from calculating the determinant of the - # jacobian. - np.squeeze(gumbel_dist.logpdf(x), axis=2), - bijector.forward_log_det_jacobian(x).eval()) + np.squeeze(gumbel_dist.logpdf(x), axis=-1), + bijector.forward_log_det_jacobian(x, event_ndims=1).eval()) self.assertAllClose( - -bijector.inverse_log_det_jacobian(y).eval(), - bijector.forward_log_det_jacobian(x).eval(), + -bijector.inverse_log_det_jacobian(y, event_ndims=1).eval(), + bijector.forward_log_det_jacobian(x, event_ndims=1).eval(), rtol=1e-4, atol=0.) @@ -60,10 +58,10 @@ class GumbelBijectorTest(test.TestCase): def testBijectiveAndFinite(self): with self.test_session(): - bijector = Gumbel(loc=0., scale=3.0, event_ndims=0, validate_args=True) + bijector = Gumbel(loc=0., scale=3.0, validate_args=True) x = np.linspace(-10., 10., num=10).astype(np.float32) y = np.linspace(0.01, 0.99, num=10).astype(np.float32) - assert_bijective_and_finite(bijector, x, y, rtol=1e-3) + assert_bijective_and_finite(bijector, x, y, event_ndims=0, rtol=1e-3) if __name__ == "__main__": diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/inline_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/inline_test.py index 739fa6d439..7d3bd758cd 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/inline_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/inline_test.py @@ -33,15 +33,13 @@ class InlineBijectorTest(test.TestCase): def testBijector(self): with self.test_session(): - exp = Exp(event_ndims=1) + exp = Exp() inline = Inline( forward_fn=math_ops.exp, inverse_fn=math_ops.log, - inverse_log_det_jacobian_fn=( - lambda y: -math_ops.reduce_sum( # pylint: disable=g-long-lambda - math_ops.log(y), reduction_indices=-1)), - forward_log_det_jacobian_fn=( - lambda x: math_ops.reduce_sum(x, reduction_indices=-1)), + inverse_log_det_jacobian_fn=lambda y: -math_ops.log(y), + forward_log_det_jacobian_fn=lambda x: x, + forward_min_event_ndims=0, name="exp") self.assertEqual(exp.name, inline.name) @@ -51,9 +49,10 @@ class InlineBijectorTest(test.TestCase): self.assertAllClose(x, inline.inverse(y).eval()) self.assertAllClose( -np.sum(np.log(y), axis=-1), - inline.inverse_log_det_jacobian(y).eval()) - self.assertAllClose(-inline.inverse_log_det_jacobian(y).eval(), - inline.forward_log_det_jacobian(x).eval()) + inline.inverse_log_det_jacobian(y, event_ndims=1).eval()) + self.assertAllClose( + -inline.inverse_log_det_jacobian(y, event_ndims=1).eval(), + inline.forward_log_det_jacobian(x, event_ndims=1).eval()) def testShapeGetters(self): with self.test_session(): @@ -62,6 +61,7 @@ class InlineBijectorTest(test.TestCase): forward_event_shape_fn=lambda x: x.as_list() + [1], inverse_event_shape_tensor_fn=lambda x: x[:-1], inverse_event_shape_fn=lambda x: x[:-1], + forward_min_event_ndims=0, name="shape_only") x = tensor_shape.TensorShape([1, 2, 3]) y = tensor_shape.TensorShape([1, 2, 3, 1]) diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/invert_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/invert_test.py index 58ba9cedb1..8b14c8327f 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/invert_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/invert_test.py @@ -34,9 +34,9 @@ class InvertBijectorTest(test.TestCase): with self.test_session(): for fwd in [ bijectors.Identity(), - bijectors.Exp(event_ndims=1), + bijectors.Exp(), bijectors.Affine(shift=[0., 1.], scale_diag=[2., 3.]), - bijectors.Softplus(event_ndims=1), + bijectors.Softplus(), bijectors.SoftmaxCentered(), ]: rev = bijectors.Invert(fwd) @@ -46,11 +46,11 @@ class InvertBijectorTest(test.TestCase): self.assertAllClose(fwd.inverse(x).eval(), rev.forward(x).eval()) self.assertAllClose(fwd.forward(x).eval(), rev.inverse(x).eval()) self.assertAllClose( - fwd.forward_log_det_jacobian(x).eval(), - rev.inverse_log_det_jacobian(x).eval()) + fwd.forward_log_det_jacobian(x, event_ndims=1).eval(), + rev.inverse_log_det_jacobian(x, event_ndims=1).eval()) self.assertAllClose( - fwd.inverse_log_det_jacobian(x).eval(), - rev.forward_log_det_jacobian(x).eval()) + fwd.inverse_log_det_jacobian(x, event_ndims=1).eval(), + rev.forward_log_det_jacobian(x, event_ndims=1).eval()) def testScalarCongruency(self): with self.test_session(): diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/kumaraswamy_bijector_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/kumaraswamy_bijector_test.py index 074b5f275d..a8089881f6 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/kumaraswamy_bijector_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/kumaraswamy_bijector_test.py @@ -34,8 +34,7 @@ class KumaraswamyBijectorTest(test.TestCase): a = 2. b = 0.3 bijector = Kumaraswamy( - concentration1=a, concentration0=b, - event_ndims=0, validate_args=True) + concentration1=a, concentration0=b, validate_args=True) self.assertEqual("kumaraswamy", bijector.name) x = np.array([[[0.1], [0.2], [0.3], [0.4], [0.5]]], dtype=np.float32) # Kumaraswamy cdf. This is the same as inverse(x). @@ -46,13 +45,11 @@ class KumaraswamyBijectorTest(test.TestCase): (b - 1) * np.log1p(-x ** a)) self.assertAllClose( - # We should lose a dimension from calculating the determinant of the - # jacobian. - kumaraswamy_log_pdf, - bijector.inverse_log_det_jacobian(x).eval()) + np.squeeze(kumaraswamy_log_pdf, axis=-1), + bijector.inverse_log_det_jacobian(x, event_ndims=1).eval()) self.assertAllClose( - -bijector.inverse_log_det_jacobian(x).eval(), - bijector.forward_log_det_jacobian(y).eval(), + -bijector.inverse_log_det_jacobian(x, event_ndims=1).eval(), + bijector.forward_log_det_jacobian(y, event_ndims=1).eval(), rtol=1e-4, atol=0.) @@ -73,7 +70,7 @@ class KumaraswamyBijectorTest(test.TestCase): # endpoints. y = np.linspace(.01, 0.99, num=10).astype(np.float32) x = 1 - (1 - y ** concentration1) ** concentration0 - assert_bijective_and_finite(bijector, x, y, rtol=1e-3) + assert_bijective_and_finite(bijector, x, y, event_ndims=0, rtol=1e-3) if __name__ == "__main__": diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py index dcfb0eb051..5ba5a2083b 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py @@ -79,9 +79,10 @@ class MaskedAutoregressiveFlowTest(test_util.VectorDistributionTestHelpers, forward_x = ma.forward(x) # Use identity to invalidate cache. inverse_y = ma.inverse(array_ops.identity(forward_x)) - fldj = ma.forward_log_det_jacobian(x) + fldj = ma.forward_log_det_jacobian(x, event_ndims=1) # Use identity to invalidate cache. - ildj = ma.inverse_log_det_jacobian(array_ops.identity(forward_x)) + ildj = ma.inverse_log_det_jacobian( + array_ops.identity(forward_x), event_ndims=1) variables.global_variables_initializer().run() [ forward_x_, diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/permute_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/permute_test.py index 54590de373..7eef4ab599 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/permute_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/permute_test.py @@ -53,8 +53,8 @@ class PermuteBijectorTest(test.TestCase): bijector.permutation, bijector.inverse(expected_y), bijector.forward(expected_x), - bijector.forward_log_det_jacobian(expected_x), - bijector.inverse_log_det_jacobian(expected_y), + bijector.forward_log_det_jacobian(expected_x, event_ndims=1), + bijector.inverse_log_det_jacobian(expected_y, event_ndims=1), ], feed_dict={permutation_ph: expected_permutation}) self.assertEqual("permute", bijector.name) self.assertAllEqual(expected_permutation, permutation_) @@ -78,10 +78,9 @@ class PermuteBijectorTest(test.TestCase): x = np.random.randn(4, 2, 3) y = x[..., permutation] with self.test_session(): - bijector = Permute( - permutation=permutation, - validate_args=True) - assert_bijective_and_finite(bijector, x, y, rtol=1e-6, atol=0) + bijector = Permute(permutation=permutation, validate_args=True) + assert_bijective_and_finite( + bijector, x, y, event_ndims=1, rtol=1e-6, atol=0) if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/power_transform_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/power_transform_test.py index de1659aa9f..85d2283013 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/power_transform_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/power_transform_test.py @@ -32,8 +32,7 @@ class PowerTransformBijectorTest(test.TestCase): def testBijector(self): with self.test_session(): c = 0.2 - bijector = PowerTransform( - power=c, event_ndims=1, validate_args=True) + bijector = PowerTransform(power=c, validate_args=True) self.assertEqual("power_transform", bijector.name) x = np.array([[[-1.], [2.], [-5. + 1e-4]]]) y = (1. + x * c)**(1. / c) @@ -41,27 +40,25 @@ class PowerTransformBijectorTest(test.TestCase): self.assertAllClose(x, bijector.inverse(y).eval()) self.assertAllClose( (c - 1.) * np.sum(np.log(y), axis=-1), - bijector.inverse_log_det_jacobian(y).eval()) + bijector.inverse_log_det_jacobian(y, event_ndims=1).eval()) self.assertAllClose( - -bijector.inverse_log_det_jacobian(y).eval(), - bijector.forward_log_det_jacobian(x).eval(), + -bijector.inverse_log_det_jacobian(y, event_ndims=1).eval(), + bijector.forward_log_det_jacobian(x, event_ndims=1).eval(), rtol=1e-4, atol=0.) def testScalarCongruency(self): with self.test_session(): - bijector = PowerTransform( - power=0.2, validate_args=True) + bijector = PowerTransform(power=0.2, validate_args=True) assert_scalar_congruency( bijector, lower_x=-2., upper_x=1.5, rtol=0.05) def testBijectiveAndFinite(self): with self.test_session(): - bijector = PowerTransform( - power=0.2, event_ndims=0, validate_args=True) + bijector = PowerTransform(power=0.2, validate_args=True) x = np.linspace(-4.999, 10, num=10).astype(np.float32) y = np.logspace(0.001, 10, num=10).astype(np.float32) - assert_bijective_and_finite(bijector, x, y, rtol=1e-3) + assert_bijective_and_finite(bijector, x, y, event_ndims=0, rtol=1e-3) if __name__ == "__main__": diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/real_nvp_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/real_nvp_test.py index 46fe779741..2d52895fbe 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/real_nvp_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/real_nvp_test.py @@ -52,24 +52,28 @@ class RealNVPTest(test_util.VectorDistributionTestHelpers, test.TestCase): forward_x = nvp.forward(x) # Use identity to invalidate cache. inverse_y = nvp.inverse(array_ops.identity(forward_x)) - fldj = nvp.forward_log_det_jacobian(x) + forward_inverse_y = nvp.forward(inverse_y) + fldj = nvp.forward_log_det_jacobian(x, event_ndims=1) # Use identity to invalidate cache. - ildj = nvp.inverse_log_det_jacobian(array_ops.identity(forward_x)) + ildj = nvp.inverse_log_det_jacobian( + array_ops.identity(forward_x), event_ndims=1) variables.global_variables_initializer().run() [ forward_x_, inverse_y_, + forward_inverse_y_, ildj_, fldj_, ] = sess.run([ forward_x, inverse_y, + forward_inverse_y, ildj, fldj, ]) self.assertEqual("real_nvp", nvp.name) - self.assertAllClose(forward_x_, forward_x_, rtol=1e-6, atol=0.) - self.assertAllClose(x_, inverse_y_, rtol=1e-5, atol=0.) + self.assertAllClose(forward_x_, forward_inverse_y_, rtol=1e-1, atol=0.) + self.assertAllClose(x_, inverse_y_, rtol=1e-1, atol=0.) self.assertAllClose(ildj_, -fldj_, rtol=1e-6, atol=0.) def testMutuallyConsistent(self): diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/reshape_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/reshape_test.py index e216d88cb1..46f2c63f9b 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/reshape_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/reshape_test.py @@ -65,8 +65,8 @@ class _ReshapeBijectorTest(object): ildj_) = sess.run(( bijector.inverse(expected_y), bijector.forward(expected_x), - bijector.forward_log_det_jacobian(expected_x), - bijector.inverse_log_det_jacobian(expected_y), + bijector.forward_log_det_jacobian(expected_x, event_ndims=2), + bijector.inverse_log_det_jacobian(expected_y, event_ndims=2), ), feed_dict=feed_dict) self.assertEqual("reshape", bijector.name) self.assertAllClose(expected_y, y_, rtol=1e-6, atol=0) @@ -301,7 +301,8 @@ class ReshapeBijectorTestStatic(test.TestCase, _ReshapeBijectorTest): event_shape_in=[2, 3], event_shape_out=[1, 2, 3], validate_args=True) - assert_bijective_and_finite(bijector, x, y, rtol=1e-6, atol=0) + assert_bijective_and_finite( + bijector, x, y, event_ndims=2, rtol=1e-6, atol=0) def testInvalidDimensionsOpError(self): if ops._USE_C_API: diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sigmoid_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sigmoid_test.py index e4f9d72785..cea4a62c22 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sigmoid_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sigmoid_test.py @@ -36,12 +36,13 @@ class SigmoidBijectorTest(test.TestCase): x = np.linspace(-10., 10., 100).reshape([2, 5, 10]).astype(np.float32) y = special.expit(x) ildj = -np.log(y) - np.log1p(-y) - self.assertAllClose(y, Sigmoid().forward(x).eval(), atol=0., rtol=1e-2) - self.assertAllClose(x, Sigmoid().inverse(y).eval(), atol=0., rtol=1e-4) - self.assertAllClose(ildj, Sigmoid().inverse_log_det_jacobian(y).eval(), - atol=0., rtol=1e-6) - self.assertAllClose(-ildj, Sigmoid().forward_log_det_jacobian(x).eval(), - atol=0., rtol=1e-4) + bijector = Sigmoid() + self.assertAllClose(y, bijector.forward(x).eval(), atol=0., rtol=1e-2) + self.assertAllClose(x, bijector.inverse(y).eval(), atol=0., rtol=1e-4) + self.assertAllClose(ildj, bijector.inverse_log_det_jacobian( + y, event_ndims=0).eval(), atol=0., rtol=1e-6) + self.assertAllClose(-ildj, bijector.forward_log_det_jacobian( + x, event_ndims=0).eval(), atol=0., rtol=1e-4) def testScalarCongruency(self): with self.test_session(): @@ -52,7 +53,8 @@ class SigmoidBijectorTest(test.TestCase): x = np.linspace(-7., 7., 100).astype(np.float32) eps = 1e-3 y = np.linspace(eps, 1. - eps, 100).astype(np.float32) - assert_bijective_and_finite(Sigmoid(), x, y, atol=0., rtol=1e-4) + assert_bijective_and_finite( + Sigmoid(), x, y, event_ndims=0, atol=0., rtol=1e-4) if __name__ == "__main__": diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sinh_arcsinh_bijector_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sinh_arcsinh_bijector_test.py index 172c180a44..45760a29ee 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sinh_arcsinh_bijector_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sinh_arcsinh_bijector_test.py @@ -39,7 +39,6 @@ class SinhArcsinhBijectorTest(test.TestCase): bijector = SinhArcsinh( skewness=skewness, tailweight=tailweight, - event_ndims=1, validate_args=True) self.assertEqual("SinhArcsinh", bijector.name) x = np.array([[[-2.01], [2.], [1e-4]]]).astype(np.float32) @@ -50,10 +49,11 @@ class SinhArcsinhBijectorTest(test.TestCase): np.sum( np.log(np.cosh(np.arcsinh(y) / tailweight - skewness)) - np.log(tailweight) - np.log(np.sqrt(y**2 + 1)), - axis=-1), bijector.inverse_log_det_jacobian(y).eval()) + axis=-1), + bijector.inverse_log_det_jacobian(y, event_ndims=1).eval()) self.assertAllClose( - -bijector.inverse_log_det_jacobian(y).eval(), - bijector.forward_log_det_jacobian(x).eval(), + -bijector.inverse_log_det_jacobian(y, event_ndims=1).eval(), + bijector.forward_log_det_jacobian(x, event_ndims=1).eval(), rtol=1e-4, atol=0.) @@ -106,14 +106,15 @@ class SinhArcsinhBijectorTest(test.TestCase): bijector = SinhArcsinh(skewness=-1., tailweight=0.5, validate_args=True) x = np.concatenate((-np.logspace(-2, 10, 1000), [0], np.logspace( -2, 10, 1000))).astype(np.float32) - assert_bijective_and_finite(bijector, x, x, rtol=1e-3) + assert_bijective_and_finite(bijector, x, x, event_ndims=0, rtol=1e-3) def testBijectiveAndFiniteSkewness1Tailweight3(self): with self.test_session(): bijector = SinhArcsinh(skewness=1., tailweight=3., validate_args=True) x = np.concatenate((-np.logspace(-2, 5, 1000), [0], np.logspace( -2, 5, 1000))).astype(np.float32) - assert_bijective_and_finite(bijector, x, x, rtol=1e-3) + assert_bijective_and_finite( + bijector, x, x, event_ndims=0, rtol=1e-3) def testBijectorEndpoints(self): with self.test_session(): @@ -124,7 +125,8 @@ class SinhArcsinhBijectorTest(test.TestCase): [np.finfo(dtype).min, np.finfo(dtype).max], dtype=dtype) # Note that the above bijector is the identity bijector. Hence, the # log_det_jacobian will be 0. Because of this we use atol. - assert_bijective_and_finite(bijector, bounds, bounds, atol=2e-6) + assert_bijective_and_finite( + bijector, bounds, bounds, event_ndims=0, atol=2e-6) def testBijectorOverRange(self): with self.test_session(): @@ -156,12 +158,12 @@ class SinhArcsinhBijectorTest(test.TestCase): np.arcsinh(y_float128) / tailweight - skewness) / np.sqrt( y_float128**2 + 1)) - np.log(tailweight), - bijector.inverse_log_det_jacobian(y).eval(), + bijector.inverse_log_det_jacobian(y, event_ndims=0).eval(), rtol=1e-4, atol=0.) self.assertAllClose( - -bijector.inverse_log_det_jacobian(y).eval(), - bijector.forward_log_det_jacobian(x).eval(), + -bijector.inverse_log_det_jacobian(y, event_ndims=0).eval(), + bijector.forward_log_det_jacobian(x, event_ndims=0).eval(), rtol=1e-4, atol=0.) diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/softmax_centered_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/softmax_centered_test.py index cad4dd1ac8..0f0a2fa531 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/softmax_centered_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/softmax_centered_test.py @@ -44,12 +44,12 @@ class SoftmaxCenteredBijectorTest(test.TestCase): self.assertAllClose(x, softmax.inverse(y).eval()) self.assertAllClose( -np.sum(np.log(y), axis=1), - softmax.inverse_log_det_jacobian(y).eval(), + softmax.inverse_log_det_jacobian(y, event_ndims=1).eval(), atol=0., rtol=1e-7) self.assertAllClose( - -softmax.inverse_log_det_jacobian(y).eval(), - softmax.forward_log_det_jacobian(x).eval(), + -softmax.inverse_log_det_jacobian(y, event_ndims=1).eval(), + softmax.forward_log_det_jacobian(x, event_ndims=1).eval(), atol=0., rtol=1e-7) @@ -67,14 +67,14 @@ class SoftmaxCenteredBijectorTest(test.TestCase): feed_dict={y: real_y})) self.assertAllClose( -np.sum(np.log(real_y), axis=1), - softmax.inverse_log_det_jacobian(y).eval( + softmax.inverse_log_det_jacobian(y, event_ndims=1).eval( feed_dict={y: real_y}), atol=0., rtol=1e-7) self.assertAllClose( - -softmax.inverse_log_det_jacobian(y).eval( + -softmax.inverse_log_det_jacobian(y, event_ndims=1).eval( feed_dict={y: real_y}), - softmax.forward_log_det_jacobian(x).eval( + softmax.forward_log_det_jacobian(x, event_ndims=1).eval( feed_dict={x: real_x}), atol=0., rtol=1e-7) @@ -104,7 +104,7 @@ class SoftmaxCenteredBijectorTest(test.TestCase): y = np.array([y_0, y_1, y_2]) y /= y.sum(axis=0) y = y.T # y.shape = [5, 3] - assert_bijective_and_finite(softmax, x, y) + assert_bijective_and_finite(softmax, x, y, event_ndims=1) if __name__ == "__main__": diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/softplus_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/softplus_test.py index d9af9aec50..3d8a0a32bb 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/softplus_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/softplus_test.py @@ -43,13 +43,13 @@ class SoftplusBijectorTest(test.TestCase): def testHingeSoftnessZeroRaises(self): with self.test_session(): - bijector = Softplus(event_ndims=0, hinge_softness=0., validate_args=True) + bijector = Softplus(hinge_softness=0., validate_args=True) with self.assertRaisesOpError("must be non-zero"): bijector.forward([1., 1.]).eval() def testBijectorForwardInverseEventDimsZero(self): with self.test_session(): - bijector = Softplus(event_ndims=0) + bijector = Softplus() self.assertEqual("softplus", bijector.name) x = 2 * rng.randn(2, 10) y = self._softplus(x) @@ -59,7 +59,7 @@ class SoftplusBijectorTest(test.TestCase): def testBijectorForwardInverseWithHingeSoftnessEventDimsZero(self): with self.test_session(): - bijector = Softplus(event_ndims=0, hinge_softness=1.5) + bijector = Softplus(hinge_softness=1.5) x = 2 * rng.randn(2, 10) y = 1.5 * self._softplus(x / 1.5) @@ -68,16 +68,17 @@ class SoftplusBijectorTest(test.TestCase): def testBijectorLogDetJacobianEventDimsZero(self): with self.test_session(): - bijector = Softplus(event_ndims=0) + bijector = Softplus() y = 2 * rng.rand(2, 10) # No reduction needed if event_dims = 0. ildj = self._softplus_ildj_before_reduction(y) - self.assertAllClose(ildj, bijector.inverse_log_det_jacobian(y).eval()) + self.assertAllClose(ildj, bijector.inverse_log_det_jacobian( + y, event_ndims=0).eval()) def testBijectorForwardInverseEventDimsOne(self): with self.test_session(): - bijector = Softplus(event_ndims=1) + bijector = Softplus() self.assertEqual("softplus", bijector.name) x = 2 * rng.randn(2, 10) y = self._softplus(x) @@ -87,58 +88,59 @@ class SoftplusBijectorTest(test.TestCase): def testBijectorLogDetJacobianEventDimsOne(self): with self.test_session(): - bijector = Softplus(event_ndims=1) + bijector = Softplus() y = 2 * rng.rand(2, 10) ildj_before = self._softplus_ildj_before_reduction(y) ildj = np.sum(ildj_before, axis=1) - self.assertAllClose(ildj, bijector.inverse_log_det_jacobian(y).eval()) + self.assertAllClose(ildj, bijector.inverse_log_det_jacobian( + y, event_ndims=1).eval()) def testScalarCongruency(self): with self.test_session(): - bijector = Softplus(event_ndims=0) + bijector = Softplus() assert_scalar_congruency( bijector, lower_x=-2., upper_x=2.) def testScalarCongruencyWithPositiveHingeSoftness(self): with self.test_session(): - bijector = Softplus(event_ndims=0, hinge_softness=1.3) + bijector = Softplus(hinge_softness=1.3) assert_scalar_congruency( bijector, lower_x=-2., upper_x=2.) def testScalarCongruencyWithNegativeHingeSoftness(self): with self.test_session(): - bijector = Softplus(event_ndims=0, hinge_softness=-1.3) + bijector = Softplus(hinge_softness=-1.3) assert_scalar_congruency( bijector, lower_x=-2., upper_x=2.) def testBijectiveAndFinite32bit(self): with self.test_session(): - bijector = Softplus(event_ndims=0) + bijector = Softplus() x = np.linspace(-20., 20., 100).astype(np.float32) y = np.logspace(-10, 10, 100).astype(np.float32) assert_bijective_and_finite( - bijector, x, y, rtol=1e-2, atol=1e-2) + bijector, x, y, event_ndims=0, rtol=1e-2, atol=1e-2) def testBijectiveAndFiniteWithPositiveHingeSoftness32Bit(self): with self.test_session(): - bijector = Softplus(event_ndims=0, hinge_softness=1.23) + bijector = Softplus(hinge_softness=1.23) x = np.linspace(-20., 20., 100).astype(np.float32) y = np.logspace(-10, 10, 100).astype(np.float32) assert_bijective_and_finite( - bijector, x, y, rtol=1e-2, atol=1e-2) + bijector, x, y, event_ndims=0, rtol=1e-2, atol=1e-2) def testBijectiveAndFiniteWithNegativeHingeSoftness32Bit(self): with self.test_session(): - bijector = Softplus(event_ndims=0, hinge_softness=-0.7) + bijector = Softplus(hinge_softness=-0.7) x = np.linspace(-20., 20., 100).astype(np.float32) y = -np.logspace(-10, 10, 100).astype(np.float32) assert_bijective_and_finite( - bijector, x, y, rtol=1e-2, atol=1e-2) + bijector, x, y, event_ndims=0, rtol=1e-2, atol=1e-2) def testBijectiveAndFinite16bit(self): with self.test_session(): - bijector = Softplus(event_ndims=0) + bijector = Softplus() # softplus(-20) is zero, so we can't use such a large range as in 32bit. x = np.linspace(-10., 20., 100).astype(np.float16) # Note that float16 is only in the open set (0, inf) for a smaller @@ -146,7 +148,7 @@ class SoftplusBijectorTest(test.TestCase): # for the test. y = np.logspace(-6, 3, 100).astype(np.float16) assert_bijective_and_finite( - bijector, x, y, rtol=1e-1, atol=1e-3) + bijector, x, y, event_ndims=0, rtol=1e-1, atol=1e-3) if __name__ == "__main__": diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/square_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/square_test.py index f03d6f1343..30c7a738c3 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/square_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/square_test.py @@ -41,10 +41,11 @@ class SquareBijectorTest(test.TestCase): self.assertAllClose(y, bijector.forward(x).eval()) self.assertAllClose(x, bijector.inverse(y).eval()) self.assertAllClose( - ildj, bijector.inverse_log_det_jacobian(y).eval(), atol=0., rtol=1e-7) + ildj, bijector.inverse_log_det_jacobian( + y, event_ndims=0).eval(), atol=0., rtol=1e-7) self.assertAllClose( - -bijector.inverse_log_det_jacobian(y).eval(), - bijector.forward_log_det_jacobian(x).eval(), + -bijector.inverse_log_det_jacobian(y, event_ndims=0).eval(), + bijector.forward_log_det_jacobian(x, event_ndims=0).eval(), atol=0., rtol=1e-7) diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/weibull_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/weibull_test.py index 7a31228d1a..f57adcda89 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/weibull_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/weibull_test.py @@ -36,7 +36,7 @@ class WeibullBijectorTest(test.TestCase): concentration = 0.3 bijector = Weibull( scale=scale, concentration=concentration, - event_ndims=1, validate_args=True) + validate_args=True) self.assertEqual("weibull", bijector.name) x = np.array([[[0.], [1.], [14.], [20.], [100.]]], dtype=np.float32) # Weibull distribution @@ -45,13 +45,11 @@ class WeibullBijectorTest(test.TestCase): self.assertAllClose(y, bijector.forward(x).eval()) self.assertAllClose(x, bijector.inverse(y).eval()) self.assertAllClose( - # We should lose a dimension from calculating the determinant of the - # jacobian. - np.squeeze(weibull_dist.logpdf(x), axis=2), - bijector.forward_log_det_jacobian(x).eval()) + weibull_dist.logpdf(x), + bijector.forward_log_det_jacobian(x, event_ndims=0).eval()) self.assertAllClose( - -bijector.inverse_log_det_jacobian(y).eval(), - bijector.forward_log_det_jacobian(x).eval(), + -bijector.inverse_log_det_jacobian(y, event_ndims=0).eval(), + bijector.forward_log_det_jacobian(x, event_ndims=0).eval(), rtol=1e-4, atol=0.) @@ -64,12 +62,12 @@ class WeibullBijectorTest(test.TestCase): def testBijectiveAndFinite(self): with self.test_session(): bijector = Weibull( - scale=20., concentration=2., event_ndims=0, validate_args=True) + scale=20., concentration=2., validate_args=True) x = np.linspace(1., 8., num=10).astype(np.float32) y = np.linspace( -np.expm1(-1 / 400.), -np.expm1(-16), num=10).astype(np.float32) - assert_bijective_and_finite(bijector, x, y, rtol=1e-3) + assert_bijective_and_finite(bijector, x, y, event_ndims=0, rtol=1e-3) if __name__ == "__main__": diff --git a/tensorflow/contrib/distributions/python/kernel_tests/conditional_transformed_distribution_test.py b/tensorflow/contrib/distributions/python/kernel_tests/conditional_transformed_distribution_test.py index 545471907f..4e8989b6c2 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/conditional_transformed_distribution_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/conditional_transformed_distribution_test.py @@ -44,6 +44,7 @@ class _ChooseLocation(ConditionalBijector): graph_parents=[self._loc], is_constant_jacobian=True, validate_args=False, + forward_min_event_ndims=0, name=name) def _forward(self, x, z): @@ -52,7 +53,7 @@ class _ChooseLocation(ConditionalBijector): def _inverse(self, x, z): return x - self._gather_loc(z) - def _inverse_log_det_jacobian(self, x, z=None): + def _inverse_log_det_jacobian(self, x, event_ndims, z=None): return 0. def _gather_loc(self, z): diff --git a/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_test.py b/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_test.py index 933756aa8e..9635134b08 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_test.py @@ -68,7 +68,7 @@ class MultivariateNormalDiagTest(test.TestCase): dist = ds.TransformedDistribution( base_dist, validate_args=True, - bijector=bijectors.Softplus(event_ndims=1)) + bijector=bijectors.Softplus()) samps = dist.sample(5) # Shape [5, 1, 3]. self.assertAllEqual([5, 1], dist.log_prob(samps).get_shape()) diff --git a/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py b/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py index f0ba1ec3eb..5fe1331d2c 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py @@ -28,6 +28,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import array_ops +from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops from tensorflow.python.platform import test @@ -36,6 +37,35 @@ ds = distributions la = linalg +class DummyMatrixTransform(bs.Bijector): + """Tractable matrix transformation. + + This is a non-sensical bijector that has forward/inverse_min_event_ndims=2. + The main use is to check that transformed distribution calculations are done + appropriately. + """ + + def __init__(self): + super(DummyMatrixTransform, self).__init__( + forward_min_event_ndims=2, + is_constant_jacobian=False, + validate_args=False, + name="dummy") + + def _forward(self, x): + return x + + def _inverse(self, y): + return y + + # Note: These jacobians don't make sense. + def _forward_log_det_jacobian(self, x): + return -linalg_ops.matrix_determinant(x) + + def _inverse_log_det_jacobian(self, x): + return linalg_ops.matrix_determinant(x) + + class TransformedDistributionTest(test.TestCase): def _cls(self): @@ -55,7 +85,7 @@ class TransformedDistributionTest(test.TestCase): # you may or may not need a reduce_sum. log_normal = self._cls()( distribution=ds.Normal(loc=mu, scale=sigma), - bijector=bs.Exp(event_ndims=0)) + bijector=bs.Exp()) sp_dist = stats.lognorm(s=sigma, scale=np.exp(mu)) # sample @@ -87,7 +117,7 @@ class TransformedDistributionTest(test.TestCase): sigma = 2.0 abs_normal = self._cls()( distribution=ds.Normal(loc=mu, scale=sigma), - bijector=bs.AbsoluteValue(event_ndims=0)) + bijector=bs.AbsoluteValue()) sp_normal = stats.norm(mu, sigma) # sample @@ -129,7 +159,7 @@ class TransformedDistributionTest(test.TestCase): self.assertAllClose(grid, cdf_, rtol=1e-6, atol=0.) def testCachedSamples(self): - exp_forward_only = bs.Exp(event_ndims=0) + exp_forward_only = bs.Exp() exp_forward_only._inverse = self._make_unimplemented( "inverse") exp_forward_only._inverse_event_shape_tensor = self._make_unimplemented( @@ -153,7 +183,7 @@ class TransformedDistributionTest(test.TestCase): self.assertAllClose(expected_log_pdf, log_pdf_val, rtol=1e-4, atol=0.) def testCachedSamplesInvert(self): - exp_inverse_only = bs.Exp(event_ndims=0) + exp_inverse_only = bs.Exp() exp_inverse_only._forward = self._make_unimplemented( "forward") exp_inverse_only._forward_event_shape_tensor = self._make_unimplemented( @@ -210,8 +240,11 @@ class TransformedDistributionTest(test.TestCase): int_identity = bs.Inline( forward_fn=array_ops.identity, inverse_fn=array_ops.identity, - inverse_log_det_jacobian_fn=lambda x: math_ops.cast(0, dtypes.int32), - forward_log_det_jacobian_fn=lambda x: math_ops.cast(0, dtypes.int32), + inverse_log_det_jacobian_fn=( + lambda y: math_ops.cast(0, dtypes.int32)), + forward_log_det_jacobian_fn=( + lambda x: math_ops.cast(0, dtypes.int32)), + forward_min_event_ndims=0, is_constant_jacobian=True) normal = self._cls()( distribution=ds.Normal(loc=0., scale=1.), @@ -435,6 +468,82 @@ class ScalarToMultiTest(test.TestCase): event_shape=[3], validate_args=True) + def testMatrixEvent(self): + with self.test_session() as sess: + batch_shape = [2] + event_shape = [2, 3, 3] + batch_shape_pl = array_ops.placeholder( + dtypes.int32, name="dynamic_batch_shape") + event_shape_pl = array_ops.placeholder( + dtypes.int32, name="dynamic_event_shape") + feed_dict = {batch_shape_pl: np.array(batch_shape, dtype=np.int32), + event_shape_pl: np.array(event_shape, dtype=np.int32)} + + scale = 2. + loc = 0. + fake_mvn_dynamic = self._cls()( + distribution=ds.Normal( + loc=loc, + scale=scale), + bijector=DummyMatrixTransform(), + batch_shape=batch_shape_pl, + event_shape=event_shape_pl, + validate_args=True) + + fake_mvn_static = self._cls()( + distribution=ds.Normal( + loc=loc, + scale=scale), + bijector=DummyMatrixTransform(), + batch_shape=batch_shape, + event_shape=event_shape, + validate_args=True) + + def actual_mvn_log_prob(x): + # This distribution is the normal PDF, reduced over the + # last 3 dimensions + a jacobian term which corresponds + # to the determinant of x. + return (np.sum( + stats.norm(loc, scale).logpdf(x), axis=(-1, -2, -3)) + + np.sum(np.linalg.det(x), axis=-1)) + + self.assertAllEqual([2, 3, 3], fake_mvn_static.event_shape) + self.assertAllEqual([2], fake_mvn_static.batch_shape) + + self.assertAllEqual(tensor_shape.TensorShape(None), + fake_mvn_dynamic.event_shape) + self.assertAllEqual(tensor_shape.TensorShape(None), + fake_mvn_dynamic.batch_shape) + + num_samples = 5e3 + for fake_mvn, feed_dict in ((fake_mvn_static, {}), + (fake_mvn_dynamic, feed_dict)): + # Ensure sample works by checking first, second moments. + y = fake_mvn.sample(int(num_samples), seed=0) + x = y[0:5, ...] + [ + x_, + fake_event_shape_, + fake_batch_shape_, + fake_log_prob_, + fake_prob_, + ] = sess.run([ + x, + fake_mvn.event_shape_tensor(), + fake_mvn.batch_shape_tensor(), + fake_mvn.log_prob(x), + fake_mvn.prob(x), + ], feed_dict=feed_dict) + + # Ensure all other functions work as intended. + self.assertAllEqual([5, 2, 2, 3, 3], x_.shape) + self.assertAllEqual([2, 3, 3], fake_event_shape_) + self.assertAllEqual([2], fake_batch_shape_) + self.assertAllClose(actual_mvn_log_prob(x_), fake_log_prob_, + atol=0., rtol=1e-6) + self.assertAllClose(np.exp(actual_mvn_log_prob(x_)), fake_prob_, + atol=0., rtol=1e-5) + if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/vector_laplace_diag_test.py b/tensorflow/contrib/distributions/python/kernel_tests/vector_laplace_diag_test.py index c355adeedb..1226c66113 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/vector_laplace_diag_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/vector_laplace_diag_test.py @@ -61,7 +61,7 @@ class VectorLaplaceDiagTest(test.TestCase): dist = ds.TransformedDistribution( base_dist, validate_args=True, - bijector=bijectors.Softplus(event_ndims=1)) + bijector=bijectors.Softplus()) samps = dist.sample(5) # Shape [5, 1, 3]. self.assertAllEqual([5, 1], dist.log_prob(samps).get_shape()) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/absolute_value.py b/tensorflow/contrib/distributions/python/ops/bijectors/absolute_value.py index 0fe9f6aa78..c9e31d7712 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/absolute_value.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/absolute_value.py @@ -18,9 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops +from tensorflow.python.framework import constant_op from tensorflow.python.ops import check_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops @@ -72,38 +70,22 @@ class AbsoluteValue(bijector.Bijector): """ - def __init__(self, event_ndims=0, validate_args=False, name="absolute_value"): + def __init__(self, validate_args=False, name="absolute_value"): """Instantiates the `AbsoluteValue` bijector. Args: - event_ndims: Python scalar indicating the number of dimensions associated - with a particular draw from the distribution. Currently only zero is - supported. validate_args: Python `bool` indicating whether arguments should be checked for correctness, in particular whether inputs to `inverse` and `inverse_log_det_jacobian` are non-negative. name: Python `str` name given to ops managed by this object. - - Raises: - ValueError: If `event_ndims` is not zero. """ self._graph_parents = [] self._name = name - event_ndims = ops.convert_to_tensor(event_ndims, name="event_ndims") - event_ndims_const = tensor_util.constant_value(event_ndims) - if event_ndims_const is not None and event_ndims_const not in (0,): - raise ValueError("event_ndims(%s) was not 0" % event_ndims_const) - else: - if validate_args: - event_ndims = control_flow_ops.with_dependencies( - [check_ops.assert_equal( - event_ndims, 0, message="event_ndims was not 0")], - event_ndims) - with self._name_scope("init"): super(AbsoluteValue, self).__init__( - event_ndims=event_ndims, + forward_min_event_ndims=0, + is_constant_jacobian=True, validate_args=validate_args, name=name) @@ -121,8 +103,7 @@ class AbsoluteValue(bijector.Bijector): # If event_ndims = 2, # F^{-1}(y) = (-y, y), so DF^{-1}(y) = (-1, 1), # so Log|DF^{-1}(y)| = Log[1, 1] = [0, 0]. - batch_shape = array_ops.shape(y)[:array_ops.rank(y) - self.event_ndims] - zeros = array_ops.zeros(batch_shape, dtype=y.dtype) + zeros = constant_op.constant(0., dtype=y.dtype) if self.validate_args: zeros = control_flow_ops.with_dependencies( [check_ops.assert_non_negative(y, message="Argument y was negative")], diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/affine.py b/tensorflow/contrib/distributions/python/ops/bijectors/affine.py index bef7bbb49b..b4c2939eb9 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/affine.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/affine.py @@ -184,6 +184,7 @@ class Affine(bijector.Bijector): with self._name_scope("init", values=[ shift, scale_identity_multiplier, scale_diag, scale_tril, scale_perturb_diag, scale_perturb_factor]): + # In the absence of `loc` and `scale`, we'll assume `dtype` is `float32`. dtype = dtypes.float32 @@ -234,7 +235,7 @@ class Affine(bijector.Bijector): event_ndims=1, validate_args=validate_args) super(Affine, self).__init__( - event_ndims=1, + forward_min_event_ndims=1, graph_parents=( [self._scale] if tensor_util.is_tensor(self._scale) else self._scale.graph_parents + @@ -360,16 +361,17 @@ class Affine(bijector.Bijector): x, sample_shape, expand_batch_dim=False) return x - def _inverse_log_det_jacobian(self, y): - return -self._forward_log_det_jacobian(y) - def _forward_log_det_jacobian(self, x): + # is_constant_jacobian = True for this bijector, hence the + # `log_det_jacobian` need only be specified for a single input, as this will + # be tiled to match `event_ndims`. if self._is_only_identity_multiplier: # We don't pad in this case and instead let the fldj be applied # via broadcast. event_size = array_ops.shape(x)[-1] event_size = math_ops.cast(event_size, dtype=self._scale.dtype) return math_ops.log(math_ops.abs(self._scale)) * event_size + return self.scale.log_abs_determinant() def _maybe_check_scale(self): diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/affine_linear_operator.py b/tensorflow/contrib/distributions/python/ops/bijectors/affine_linear_operator.py index 89043b1410..59f9742d57 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/affine_linear_operator.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/affine_linear_operator.py @@ -22,9 +22,6 @@ from tensorflow.contrib.distributions.python.ops.shape import _DistributionShape from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops.distributions import bijector from tensorflow.python.ops.linalg import linear_operator @@ -94,7 +91,6 @@ class AffineLinearOperator(bijector.Bijector): def __init__(self, shift=None, scale=None, - event_ndims=1, validate_args=False, name="affine_linear_operator"): """Instantiates the `AffineLinearOperator` bijector. @@ -103,14 +99,11 @@ class AffineLinearOperator(bijector.Bijector): shift: Floating-point `Tensor`. scale: Subclass of `LinearOperator`. Represents the (batch) positive definite matrix `M` in `R^{k x k}`. - event_ndims: Scalar `integer` `Tensor` indicating the number of dimensions - associated with a particular draw from the distribution. Must be 0 or 1. validate_args: Python `bool` indicating whether arguments should be checked for correctness. name: Python `str` name given to ops managed by this object. Raises: - ValueError: if `event_ndims` is not 0 or 1. TypeError: if `scale` is not a `LinearOperator`. TypeError: if `shift.dtype` does not match `scale.dtype`. ValueError: if not `scale.is_non_singular`. @@ -120,20 +113,6 @@ class AffineLinearOperator(bijector.Bijector): self._validate_args = validate_args graph_parents = [] with self._name_scope("init", values=[shift]): - event_ndims = ops.convert_to_tensor(event_ndims, name="event_ndims") - if tensor_util.constant_value(event_ndims) is not None: - event_ndims = tensor_util.constant_value(event_ndims) - if event_ndims not in (0, 1): - raise ValueError("event_ndims({}) was not 0 or 1".format(event_ndims)) - else: - if validate_args: - # Shape tool will catch if event_ndims is negative. - event_ndims = control_flow_ops.with_dependencies( - [check_ops.assert_less( - event_ndims, 2, message="event_ndims must be 0 or 1")], - event_ndims) - graph_parents += [event_ndims] - # In the absence of `loc` and `scale`, we'll assume `dtype` is `float32`. dtype = dtypes.float32 @@ -166,10 +145,10 @@ class AffineLinearOperator(bijector.Bijector): self._scale = scale self._shaper = _DistributionShape( batch_ndims=batch_ndims, - event_ndims=event_ndims, + event_ndims=1, validate_args=validate_args) super(AffineLinearOperator, self).__init__( - event_ndims=event_ndims, + forward_min_event_ndims=1, graph_parents=graph_parents, is_constant_jacobian=True, dtype=dtype, @@ -213,12 +192,13 @@ class AffineLinearOperator(bijector.Bijector): x, sample_shape, expand_batch_dim=False) return x - def _inverse_log_det_jacobian(self, y): - return -self._forward_log_det_jacobian(y) - - def _forward_log_det_jacobian(self, x): # pylint: disable=unused-argument + def _forward_log_det_jacobian(self, x): + # is_constant_jacobian = True for this bijector, hence the + # `log_det_jacobian` need only be specified for a single input, as this will + # be tiled to match `event_ndims`. if self.scale is None: - return constant_op.constant(0, dtype=x.dtype.base_dtype) + return constant_op.constant(0., dtype=x.dtype.base_dtype) + with ops.control_dependencies(self._maybe_collect_assertions() if self.validate_args else []): return self.scale.log_abs_determinant() diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/affine_scalar.py b/tensorflow/contrib/distributions/python/ops/bijectors/affine_scalar.py index 8adaa54c84..cd792e2c8c 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/affine_scalar.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/affine_scalar.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.framework import constant_op from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import check_ops @@ -99,7 +100,7 @@ class AffineScalar(bijector.Bijector): self._scale) super(AffineScalar, self).__init__( - event_ndims=0, + forward_min_event_ndims=0, is_constant_jacobian=True, validate_args=validate_args, name=name) @@ -131,8 +132,10 @@ class AffineScalar(bijector.Bijector): return x def _forward_log_det_jacobian(self, x): - log_det_jacobian = array_ops.zeros_like(x) + # is_constant_jacobian = True for this bijector, hence the + # `log_det_jacobian` need only be specified for a single input, as this will + # be tiled to match `event_ndims`. if self.scale is None: - return log_det_jacobian - log_det_jacobian += math_ops.log(math_ops.abs(self.scale)) - return log_det_jacobian + return constant_op.constant(0., dtype=x.dtype.base_dtype) + + return math_ops.log(math_ops.abs(self.scale)) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/batch_normalization.py b/tensorflow/contrib/distributions/python/ops/bijectors/batch_normalization.py index 33fdd32d7a..224cec8a63 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/batch_normalization.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/batch_normalization.py @@ -157,7 +157,12 @@ class BatchNormalization(bijector.Bijector): gamma_constraint=g_constraint) self._validate_bn_layer(self.batchnorm) self._training = training + if isinstance(self.batchnorm.axis, int): + forward_min_event_ndims = 1 + else: + forward_min_event_ndims = len(self.batchnorm.axis) super(BatchNormalization, self).__init__( + forward_min_event_ndims=forward_min_event_ndims, validate_args=validate_args, name=name) def _validate_bn_layer(self, layer): @@ -186,7 +191,6 @@ class BatchNormalization(bijector.Bijector): input_shape = np.int32(x.shape.as_list()) ndims = len(input_shape) - # event_dims = self._compute_event_dims(x) reduction_axes = [i for i in range(ndims) if i not in self.batchnorm.axis] # Broadcasting only necessary for single-axis batch norm where the axis is # not the last dimension diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/chain.py b/tensorflow/contrib/distributions/python/ops/bijectors/chain.py index 3ce7c26213..85ad23e413 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/chain.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/chain.py @@ -21,6 +21,9 @@ from __future__ import print_function import itertools from tensorflow.python.framework import constant_op +from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_util +from tensorflow.python.ops import array_ops from tensorflow.python.ops.distributions import bijector @@ -29,6 +32,91 @@ __all__ = [ ] +def _use_static_shape(input_tensor, ndims): + return input_tensor.shape.is_fully_defined() and isinstance(ndims, int) + + +def _maybe_get_event_ndims_statically(event_ndims): + static_event_ndims = (event_ndims if isinstance(event_ndims, int) + else tensor_util.constant_value(event_ndims)) + if static_event_ndims is not None: + return static_event_ndims + + return event_ndims + + +def _compute_min_event_ndims(bijector_list, compute_forward=True): + """Computes the min_event_ndims associated with the give list of bijectors. + + Given a list `bijector_list` of bijectors, compute the min_event_ndims that is + associated with the composition of bijectors in that list. + + min_event_ndims is the # of right most dimensions for which the bijector has + done necessary computation on (i.e. the non-broadcastable part of the + computation). + + We can derive the min_event_ndims for a chain of bijectors as follows: + + In the case where there are no rank changing bijectors, this will simply be + `max(b.forward_min_event_ndims for b in bijector_list)`. This is because the + bijector with the most forward_min_event_ndims requires the most dimensions, + and hence the chain also requires operating on those dimensions. + + However in the case of rank changing, more care is needed in determining the + exact amount of dimensions. Padding dimensions causes subsequent bijectors to + operate on the padded dimensions, and Removing dimensions causes bijectors to + operate more left. + + Args: + bijector_list: List of bijectors to be composed by chain. + compute_forward: Boolean. If True, computes the min_event_ndims associated + with a forward call to Chain, and otherwise computes the min_event_ndims + associated with an inverse call to Chain. The latter is the same as the + min_event_ndims associated with a forward call to Invert(Chain(....)). + + Returns: + min_event_ndims + """ + min_event_ndims = 0 + # This is a mouthful, but what this encapsulates is that if not for rank + # changing bijectors, we'd only need to compute the largest of the min + # required ndims. Hence "max_min". Due to rank changing bijectors, we need to + # account for synthetic rank growth / synthetic rank decrease from a rank + # changing bijector. + rank_changed_adjusted_max_min_event_ndims = 0 + + if compute_forward: + bijector_list = reversed(bijector_list) + + for b in bijector_list: + if compute_forward: + current_min_event_ndims = b.forward_min_event_ndims + current_inverse_min_event_ndims = b.inverse_min_event_ndims + else: + current_min_event_ndims = b.inverse_min_event_ndims + current_inverse_min_event_ndims = b.forward_min_event_ndims + + # New dimensions were touched. + if rank_changed_adjusted_max_min_event_ndims < current_min_event_ndims: + min_event_ndims += ( + current_min_event_ndims - rank_changed_adjusted_max_min_event_ndims) + rank_changed_adjusted_max_min_event_ndims = max( + current_min_event_ndims, rank_changed_adjusted_max_min_event_ndims) + + # If the number of dimensions has increased via forward, then + # inverse_min_event_ndims > forward_min_event_ndims, and hence the + # dimensions we computed on, have moved left (so we have operated + # on additional dimensions). + # Conversely, if the number of dimensions has decreased via forward, + # then we have inverse_min_event_ndims < forward_min_event_ndims, + # and so we will have operated on fewer right most dimensions. + + number_of_changed_dimensions = ( + current_min_event_ndims - current_inverse_min_event_ndims) + rank_changed_adjusted_max_min_event_ndims -= number_of_changed_dimensions + return min_event_ndims + + class Chain(bijector.Bijector): """Bijector which applies a sequence of bijectors. @@ -93,21 +181,24 @@ class Chain(bijector.Bijector): raise ValueError("incompatible dtypes: %s" % dtype) elif len(dtype) == 2: dtype = dtype[1] if dtype[0] is None else dtype[0] - event_ndims = bijectors[0].event_ndims elif len(dtype) == 1: dtype = dtype[0] - event_ndims = bijectors[0].event_ndims else: dtype = None - event_ndims = None + + inverse_min_event_ndims = _compute_min_event_ndims( + bijectors, compute_forward=False) + forward_min_event_ndims = _compute_min_event_ndims( + bijectors, compute_forward=True) super(Chain, self).__init__( graph_parents=list(itertools.chain.from_iterable( b.graph_parents for b in bijectors)), + forward_min_event_ndims=forward_min_event_ndims, + inverse_min_event_ndims=inverse_min_event_ndims, is_constant_jacobian=all(b.is_constant_jacobian for b in bijectors), validate_args=validate_args, dtype=dtype, - event_ndims=event_ndims, name=name or ("identity" if not bijectors else "_of_".join(["chain"] + [b.name for b in bijectors]))) @@ -147,10 +238,31 @@ class Chain(bijector.Bijector): return y def _inverse_log_det_jacobian(self, y, **kwargs): - ildj = constant_op.constant(0., dtype=y.dtype, - name="inverse_log_det_jacobian") + ildj = constant_op.constant( + 0., dtype=y.dtype.base_dtype, name="inverse_log_det_jacobian") + + if not self.bijectors: + return ildj + + event_ndims = _maybe_get_event_ndims_statically( + self.inverse_min_event_ndims) + + if _use_static_shape(y, event_ndims): + event_shape = y.shape[y.shape.ndims - event_ndims:] + else: + event_shape = array_ops.shape(y)[array_ops.rank(y) - event_ndims:] + for b in self.bijectors: - ildj += b.inverse_log_det_jacobian(y, **kwargs.get(b.name, {})) + ildj += b.inverse_log_det_jacobian( + y, event_ndims=event_ndims, **kwargs.get(b.name, {})) + + if _use_static_shape(y, event_ndims): + event_shape = b.inverse_event_shape(event_shape) + event_ndims = _maybe_get_event_ndims_statically(event_shape.ndims) + else: + event_shape = b.inverse_event_shape_tensor(event_shape) + event_ndims = _maybe_get_event_ndims_statically( + array_ops.rank(event_shape)) y = b.inverse(y, **kwargs.get(b.name, {})) return ildj @@ -160,9 +272,34 @@ class Chain(bijector.Bijector): return x def _forward_log_det_jacobian(self, x, **kwargs): - fldj = constant_op.constant(0., dtype=x.dtype, - name="forward_log_det_jacobian") + x = ops.convert_to_tensor(x, name="x") + + fldj = constant_op.constant( + 0., dtype=x.dtype, name="inverse_log_det_jacobian") + + if not self.bijectors: + return fldj + + event_ndims = _maybe_get_event_ndims_statically( + self.forward_min_event_ndims) + + if _use_static_shape(x, event_ndims): + event_shape = x.shape[x.shape.ndims - event_ndims:] + else: + event_shape = array_ops.shape(x)[array_ops.rank(x) - event_ndims:] + for b in reversed(self.bijectors): - fldj += b.forward_log_det_jacobian(x, **kwargs.get(b.name, {})) + fldj += b.forward_log_det_jacobian( + x, event_ndims=event_ndims, **kwargs.get(b.name, {})) + if _use_static_shape(x, event_ndims): + event_shape = b.forward_event_shape(event_shape) + event_ndims = _maybe_get_event_ndims_statically(event_shape.ndims) + else: + event_shape = b.forward_event_shape_tensor(event_shape) + event_ndims = _maybe_get_event_ndims_statically( + array_ops.rank(event_shape)) + x = b.forward(x, **kwargs.get(b.name, {})) + return fldj + diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/cholesky_outer_product.py b/tensorflow/contrib/distributions/python/ops/bijectors/cholesky_outer_product.py index 8f09e16058..caae2adcfa 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/cholesky_outer_product.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/cholesky_outer_product.py @@ -80,7 +80,7 @@ class CholeskyOuterProduct(bijector.Bijector): self._graph_parents = [] self._name = name super(CholeskyOuterProduct, self).__init__( - event_ndims=2, + forward_min_event_ndims=2, validate_args=validate_args, name=name) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/conditional_bijector.py b/tensorflow/contrib/distributions/python/ops/bijectors/conditional_bijector.py index ccb1f02927..e9e994f839 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/conditional_bijector.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/conditional_bijector.py @@ -44,12 +44,16 @@ class ConditionalBijector(bijector.Bijector): "**condition_kwargs": "Named arguments forwarded to subclass implementation."}) def inverse_log_det_jacobian( - self, y, name="inverse_log_det_jacobian", **condition_kwargs): - return self._call_inverse_log_det_jacobian(y, name, **condition_kwargs) + self, y, event_ndims, name="inverse_log_det_jacobian", + **condition_kwargs): + return self._call_inverse_log_det_jacobian( + y, event_ndims, name, **condition_kwargs) @distribution_util.AppendDocstring(kwargs_dict={ "**condition_kwargs": "Named arguments forwarded to subclass implementation."}) def forward_log_det_jacobian( - self, x, name="forward_log_det_jacobian", **condition_kwargs): - return self._call_forward_log_det_jacobian(x, name, **condition_kwargs) + self, x, event_ndims, name="forward_log_det_jacobian", + **condition_kwargs): + return self._call_forward_log_det_jacobian( + x, event_ndims, name, **condition_kwargs) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/exp.py b/tensorflow/contrib/distributions/python/ops/bijectors/exp.py index b1ff840d62..9fc1bbf052 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/exp.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/exp.py @@ -33,8 +33,8 @@ class Exp(power_transform.PowerTransform): ```python # Create the Y=g(X)=exp(X) transform which works only on Tensors with 1 - # batch ndim and 2 event ndims (i.e., vector of matrices). - exp = Exp(event_ndims=2) + # batch ndim 2. + exp = Exp() x = [[[1., 2], [3, 4]], [[5, 6], @@ -48,19 +48,17 @@ class Exp(power_transform.PowerTransform): """ def __init__(self, - event_ndims=0, validate_args=False, name="exp"): """Instantiates the `Exp` bijector. Args: - event_ndims: Scalar `int32` `Tensor` indicating the number of dimensions - associated with a particular draw from the distribution. validate_args: Python `bool` indicating whether arguments should be checked for correctness. name: Python `str` name given to ops managed by this object. """ + # forward_min_event_ndims = 0. + # No forward_min_event_ndims specified as this is done in PowerTransform. super(Exp, self).__init__( - event_ndims=event_ndims, validate_args=validate_args, name=name) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/gumbel.py b/tensorflow/contrib/distributions/python/ops/bijectors/gumbel.py index 67f3978556..e656a258e5 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/gumbel.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/gumbel.py @@ -48,7 +48,6 @@ class Gumbel(bijector.Bijector): def __init__(self, loc=0., scale=1., - event_ndims=0, validate_args=False, name="gumbel"): """Instantiates the `Gumbel` bijector. @@ -60,8 +59,6 @@ class Gumbel(bijector.Bijector): scale: Positive Float-like `Tensor` that is the same dtype and is broadcastable with `loc`. This is `scale` in `Y = g(X) = exp(-exp(-(X - loc) / scale))`. - event_ndims: Python scalar indicating the number of dimensions associated - with a particular draw from the distribution. validate_args: Python `bool` indicating whether arguments should be checked for correctness. name: Python `str` name given to ops managed by this object. @@ -80,7 +77,9 @@ class Gumbel(bijector.Bijector): ], self._scale) super(Gumbel, self).__init__( - event_ndims=event_ndims, validate_args=validate_args, name=name) + validate_args=validate_args, + forward_min_event_ndims=0, + name=name) @property def loc(self): @@ -102,15 +101,11 @@ class Gumbel(bijector.Bijector): def _inverse_log_det_jacobian(self, y): y = self._maybe_assert_valid_y(y) - event_dims = self._event_dims_tensor(y) - return math_ops.reduce_sum( - math_ops.log(self.scale / (-math_ops.log(y) * y)), axis=event_dims) + return math_ops.log(self.scale / (-math_ops.log(y) * y)) def _forward_log_det_jacobian(self, x): - event_dims = self._event_dims_tensor(x) z = (x - self.loc) / self.scale - return math_ops.reduce_sum( - -z - math_ops.exp(-z) - math_ops.log(self.scale), axis=event_dims) + return -z - math_ops.exp(-z) - math_ops.log(self.scale) def _maybe_assert_valid_y(self, y): if not self.validate_args: diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/inline.py b/tensorflow/contrib/distributions/python/ops/bijectors/inline.py index fab1b22fbf..2bde956d13 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/inline.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/inline.py @@ -40,7 +40,7 @@ class Inline(bijector.Bijector): name="exp") ``` - The above example is equivalent to the `Bijector` `Exp(event_ndims=1)`. + The above example is equivalent to the `Bijector` `Exp()`. """ def __init__(self, @@ -54,6 +54,8 @@ class Inline(bijector.Bijector): inverse_event_shape_tensor_fn=None, is_constant_jacobian=False, validate_args=False, + forward_min_event_ndims=None, + inverse_min_event_ndims=None, name="inline"): """Creates a `Bijector` from callables. @@ -76,10 +78,15 @@ class Inline(bijector.Bijector): constant for all input arguments. validate_args: Python `bool` indicating whether arguments should be checked for correctness. + forward_min_event_ndims: Python `int` indicating the minimal + dimensionality this bijector acts on. + inverse_min_event_ndims: Python `int` indicating the minimal + dimensionality this bijector acts on. name: Python `str`, name given to ops managed by this object. """ super(Inline, self).__init__( - event_ndims=0, + forward_min_event_ndims=forward_min_event_ndims, + inverse_min_event_ndims=inverse_min_event_ndims, is_constant_jacobian=is_constant_jacobian, validate_args=validate_args, name=name) @@ -134,8 +141,8 @@ class Inline(bijector.Bijector): "inverse_log_det_jacobian_fn is not a callable function.") return self._inverse_log_det_jacobian_fn(y, **kwargs) - def _forward_log_det_jacobian(self, y, **kwargs): + def _forward_log_det_jacobian(self, x, **kwargs): if not callable(self._forward_log_det_jacobian_fn): raise NotImplementedError( "forward_log_det_jacobian_fn is not a callable function.") - return self._forward_log_det_jacobian_fn(y, **kwargs) + return self._forward_log_det_jacobian_fn(x, **kwargs) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/invert.py b/tensorflow/contrib/distributions/python/ops/bijectors/invert.py index 2c603fe61f..1904239a0e 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/invert.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/invert.py @@ -66,8 +66,9 @@ class Invert(bijector_lib.Bijector): self._bijector = bijector super(Invert, self).__init__( - event_ndims=bijector.event_ndims, graph_parents=bijector.graph_parents, + forward_min_event_ndims=bijector.inverse_min_event_ndims, + inverse_min_event_ndims=bijector.forward_min_event_ndims, is_constant_jacobian=bijector.is_constant_jacobian, validate_args=validate_args, dtype=bijector.dtype, diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/kumaraswamy.py b/tensorflow/contrib/distributions/python/ops/bijectors/kumaraswamy.py index f5de052c9e..97000c1726 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/kumaraswamy.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/kumaraswamy.py @@ -19,7 +19,6 @@ from __future__ import division from __future__ import print_function from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import check_ops from tensorflow.python.ops import control_flow_ops @@ -48,7 +47,6 @@ class Kumaraswamy(bijector.Bijector): def __init__(self, concentration1=None, concentration0=None, - event_ndims=0, validate_args=False, name="kumaraswamy"): """Instantiates the `Kumaraswamy` bijector. @@ -60,31 +58,14 @@ class Kumaraswamy(bijector.Bijector): concentration0: Python `float` scalar indicating the transform power, i.e., `Y = g(X) = (1 - (1 - X)**(1 / b))**(1 / a)` where `b` is `concentration0`. - event_ndims: Python scalar indicating the number of dimensions associated - with a particular draw from the distribution. Currently only zero is - supported. validate_args: Python `bool` indicating whether arguments should be checked for correctness. name: Python `str` name given to ops managed by this object. - - Raises: - ValueError: If `event_ndims` is not zero. """ self._graph_parents = [] self._name = name self._validate_args = validate_args - event_ndims = ops.convert_to_tensor(event_ndims, name="event_ndims") - event_ndims_const = tensor_util.constant_value(event_ndims) - if event_ndims_const is not None and event_ndims_const not in (0,): - raise ValueError("event_ndims(%s) was not 0" % event_ndims_const) - else: - if validate_args: - event_ndims = control_flow_ops.with_dependencies( - [check_ops.assert_equal( - event_ndims, 0, message="event_ndims was not 0")], - event_ndims) - with self._name_scope("init", values=[concentration1, concentration0]): concentration1 = self._maybe_assert_valid_concentration( ops.convert_to_tensor(concentration1, name="concentration1"), @@ -96,7 +77,7 @@ class Kumaraswamy(bijector.Bijector): self._concentration1 = concentration1 self._concentration0 = concentration0 super(Kumaraswamy, self).__init__( - event_ndims=0, + forward_min_event_ndims=0, validate_args=validate_args, name=name) @@ -123,12 +104,10 @@ class Kumaraswamy(bijector.Bijector): def _inverse_log_det_jacobian(self, y): y = self._maybe_assert_valid(y) - event_dims = self._event_dims_tensor(y) - return math_ops.reduce_sum( + return ( math_ops.log(self.concentration1) + math_ops.log(self.concentration0) + (self.concentration1 - 1) * math_ops.log(y) + - (self.concentration0 - 1) * math_ops.log1p(-y**self.concentration1), - axis=event_dims) + (self.concentration0 - 1) * math_ops.log1p(-y**self.concentration1)) def _maybe_assert_valid_concentration(self, concentration, validate_args): """Checks the validity of a concentration parameter.""" diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py b/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py index 84b2340c75..ef56cf6ddd 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py @@ -61,7 +61,7 @@ class MaskedAutoregressiveFlow(bijector_lib.Bijector): this property by zeroing out weights in its `masked_dense` layers. In the `tf.distributions` framework, a "normalizing flow" is implemented as a - `tf.distributions.bijectors.Bijector`. The `forward` "autoregression" + `tf.contrib.distributions.bijectors.Bijector`. The `forward` "autoregression" is implemented using a `tf.while_loop` and a deep neural network (DNN) with masked weights such that the autoregressive property is automatically met in the `inverse`. @@ -220,6 +220,7 @@ class MaskedAutoregressiveFlow(bijector_lib.Bijector): self._shift_and_log_scale_fn = shift_and_log_scale_fn self._unroll_loop = unroll_loop super(MaskedAutoregressiveFlow, self).__init__( + forward_min_event_ndims=1, is_constant_jacobian=is_constant_jacobian, validate_args=validate_args, name=name) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/permute.py b/tensorflow/contrib/distributions/python/ops/bijectors/permute.py index 8654cc39d0..4978167803 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/permute.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/permute.py @@ -114,6 +114,7 @@ class Permute(bijector_lib.Bijector): ], permutation) self._permutation = permutation super(Permute, self).__init__( + forward_min_event_ndims=1, is_constant_jacobian=True, validate_args=validate_args, name=name or "permute") @@ -132,7 +133,10 @@ class Permute(bijector_lib.Bijector): axis=-1) def _inverse_log_det_jacobian(self, y): - return constant_op.constant(0., dtype=y.dtype) + # is_constant_jacobian = True for this bijector, hence the + # `log_det_jacobian` need only be specified for a single input, as this will + # be tiled to match `event_ndims`. + return constant_op.constant(0., dtype=y.dtype.base_dtype) def _forward_log_det_jacobian(self, x): - return constant_op.constant(0., dtype=x.dtype) + return constant_op.constant(0., dtype=x.dtype.base_dtype) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/power_transform.py b/tensorflow/contrib/distributions/python/ops/bijectors/power_transform.py index c37db61720..71f123f2a9 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/power_transform.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/power_transform.py @@ -43,7 +43,6 @@ class PowerTransform(bijector.Bijector): def __init__(self, power=0., - event_ndims=0, validate_args=False, name="power_transform"): """Instantiates the `PowerTransform` bijector. @@ -51,8 +50,6 @@ class PowerTransform(bijector.Bijector): Args: power: Python `float` scalar indicating the transform power, i.e., `Y = g(X) = (1 + X * c)**(1 / c)` where `c` is the `power`. - event_ndims: Python scalar indicating the number of dimensions associated - with a particular draw from the distribution. validate_args: Python `bool` indicating whether arguments should be checked for correctness. name: Python `str` name given to ops managed by this object. @@ -70,7 +67,7 @@ class PowerTransform(bijector.Bijector): raise ValueError("`power` must be a non-negative TF constant.") self._power = power super(PowerTransform, self).__init__( - event_ndims=event_ndims, + forward_min_event_ndims=0, validate_args=validate_args, name=name) @@ -97,18 +94,13 @@ class PowerTransform(bijector.Bijector): def _inverse_log_det_jacobian(self, y): y = self._maybe_assert_valid_y(y) - event_dims = self._event_dims_tensor(y) - return (self.power - 1.) * math_ops.reduce_sum( - math_ops.log(y), axis=event_dims) + return (self.power - 1.) * math_ops.log(y) def _forward_log_det_jacobian(self, x): x = self._maybe_assert_valid_x(x) - event_dims = self._event_dims_tensor(x) if self.power == 0.: - return math_ops.reduce_sum(x, axis=event_dims) - return (1. / self.power - 1.) * math_ops.reduce_sum( - math_ops.log1p(x * self.power), - axis=event_dims) + return x + return (1. / self.power - 1.) * math_ops.log1p(x * self.power) def _maybe_assert_valid_x(self, x): if not self.validate_args or self.power == 0.: diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/real_nvp.py b/tensorflow/contrib/distributions/python/ops/bijectors/real_nvp.py index 71ab369d01..f09ab21bce 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/real_nvp.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/real_nvp.py @@ -166,7 +166,7 @@ class RealNVP(bijector_lib.Bijector): self._input_depth = None self._shift_and_log_scale_fn = shift_and_log_scale_fn super(RealNVP, self).__init__( - event_ndims=1, + forward_min_event_ndims=1, is_constant_jacobian=is_constant_jacobian, validate_args=validate_args, name=name) @@ -224,7 +224,7 @@ class RealNVP(bijector_lib.Bijector): _, log_scale = self._shift_and_log_scale_fn( x0, self._input_depth - self._num_masked) if log_scale is None: - return constant_op.constant(0., dtype=x.dtype, name="ildj") + return constant_op.constant(0., dtype=x.dtype, name="fldj") return math_ops.reduce_sum(log_scale, axis=-1) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/reshape.py b/tensorflow/contrib/distributions/python/ops/bijectors/reshape.py index 55eca06312..82210cd6c9 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/reshape.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/reshape.py @@ -128,9 +128,11 @@ class Reshape(bijector_lib.Bijector): self._event_shape_in = event_shape_in self._event_shape_out = event_shape_out - super(Reshape, self).__init__(is_constant_jacobian=True, - validate_args=validate_args, - name=name or "reshape") + super(Reshape, self).__init__( + forward_min_event_ndims=0, + is_constant_jacobian=True, + validate_args=validate_args, + name=name or "reshape") def _maybe_check_valid_shape(self, shape, validate_args): """Check that a shape Tensor is int-type and otherwise sane.""" diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/sigmoid.py b/tensorflow/contrib/distributions/python/ops/bijectors/sigmoid.py index a640dfe7df..5df8c88631 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/sigmoid.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/sigmoid.py @@ -33,7 +33,9 @@ class Sigmoid(bijector.Bijector): def __init__(self, validate_args=False, name="sigmoid"): super(Sigmoid, self).__init__( - event_ndims=0, validate_args=validate_args, name=name) + forward_min_event_ndims=0, + validate_args=validate_args, + name=name) def _forward(self, x): return math_ops.sigmoid(x) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/sinh_arcsinh.py b/tensorflow/contrib/distributions/python/ops/bijectors/sinh_arcsinh.py index 3a75e4ae94..2a32e8abcd 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/sinh_arcsinh.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/sinh_arcsinh.py @@ -91,7 +91,6 @@ class SinhArcsinh(bijector.Bijector): def __init__(self, skewness=None, tailweight=None, - event_ndims=0, validate_args=False, name="SinhArcsinh"): """Instantiates the `SinhArcsinh` bijector. @@ -101,8 +100,6 @@ class SinhArcsinh(bijector.Bijector): of type `float32`. tailweight: Tailweight parameter. Positive `Tensor` of same `dtype` as `skewness` and broadcastable `shape`. Default is `1` of type `float32`. - event_ndims: Python scalar indicating the number of dimensions associated - with a particular draw from the distribution. validate_args: Python `bool` indicating whether arguments should be checked for correctness. name: Python `str` name given to ops managed by this object. @@ -125,7 +122,9 @@ class SinhArcsinh(bijector.Bijector): message="Argument tailweight was not positive") ], self._tailweight) super(SinhArcsinh, self).__init__( - event_ndims=event_ndims, validate_args=validate_args, name=name) + forward_min_event_ndims=0, + validate_args=validate_args, + name=name) @property def skewness(self): @@ -149,31 +148,29 @@ class SinhArcsinh(bijector.Bijector): # dx/dy # = cosh(arcsinh(y) / tailweight - skewness) # / (tailweight * sqrt(y**2 + 1)) - event_dims = self._event_dims_tensor(y) - return math_ops.reduce_sum( - # This is computed inside the log to avoid catastrophic cancellations - # from cosh((arcsinh(y) / tailweight) - skewness) and sqrt(x**2 + 1). + + # This is computed inside the log to avoid catastrophic cancellations + # from cosh((arcsinh(y) / tailweight) - skewness) and sqrt(x**2 + 1). + return ( math_ops.log(math_ops.cosh( math_ops.asinh(y) / self.tailweight - self.skewness) # TODO(srvasude): Consider using cosh(arcsinh(x)) in cases # where (arcsinh(x) / tailweight) - skewness ~= arcsinh(x). / _sqrtx2p1(y)) - - math_ops.log(self.tailweight), - axis=event_dims) + - math_ops.log(self.tailweight)) def _forward_log_det_jacobian(self, x): # y = sinh((arcsinh(x) + skewness) * tailweight) # Using sinh' = cosh, arcsinh'(x) = 1 / sqrt(x**2 + 1), # dy/dx # = cosh((arcsinh(x) + skewness) * tailweight) * tailweight / sqrt(x**2 + 1) - event_dims = self._event_dims_tensor(x) - return math_ops.reduce_sum( - # This is computed inside the log to avoid catastrophic cancellations - # from cosh((arcsinh(x) + skewness) * tailweight) and sqrt(x**2 + 1). + + # This is computed inside the log to avoid catastrophic cancellations + # from cosh((arcsinh(x) + skewness) * tailweight) and sqrt(x**2 + 1). + return ( math_ops.log(math_ops.cosh( (math_ops.asinh(x) + self.skewness) * self.tailweight) # TODO(srvasude): Consider using cosh(arcsinh(x)) in cases # where (arcsinh(x) + skewness) * tailweight ~= arcsinh(x). / _sqrtx2p1(x)) - + math_ops.log(self.tailweight), - axis=event_dims) + + math_ops.log(self.tailweight)) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/softmax_centered.py b/tensorflow/contrib/distributions/python/ops/bijectors/softmax_centered.py index dc94fd0a38..f52b91550e 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/softmax_centered.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/softmax_centered.py @@ -66,7 +66,7 @@ class SoftmaxCentered(bijector.Bijector): self._graph_parents = [] self._name = name super(SoftmaxCentered, self).__init__( - event_ndims=1, + forward_min_event_ndims=1, validate_args=validate_args, name=name) @@ -105,8 +105,6 @@ class SoftmaxCentered(bijector.Bijector): y.shape.assert_is_compatible_with(shape) y.set_shape(shape) - # Since we only support event_ndims in [0, 1] and we do padding, we always - # reduce over the last dimension, i.e., dim=-1 (which is the default). return nn_ops.softmax(y) def _inverse(self, y): @@ -162,8 +160,6 @@ class SoftmaxCentered(bijector.Bijector): # -log_normalization + reduce_sum(logits - log_normalization) log_normalization = nn_ops.softplus( math_ops.reduce_logsumexp(x, axis=-1, keep_dims=True)) - fldj = (-log_normalization + - math_ops.reduce_sum(x - log_normalization, - axis=-1, - keep_dims=True)) - return array_ops.squeeze(fldj, squeeze_dims=-1) + return array_ops.squeeze( + (-log_normalization + math_ops.reduce_sum( + x - log_normalization, axis=-1, keepdims=True)), axis=-1) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/softplus.py b/tensorflow/contrib/distributions/python/ops/bijectors/softplus.py index 81957fcf78..96a938c803 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/softplus.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/softplus.py @@ -62,7 +62,7 @@ class Softplus(bijector.Bijector): ```python # Create the Y=g(X)=softplus(X) transform which works only on Tensors with 1 # batch ndim and 2 event ndims (i.e., vector of matrices). - softplus = Softplus(event_ndims=2) + softplus = Softplus() x = [[[1., 2], [3, 4]], [[5, 6], @@ -81,7 +81,6 @@ class Softplus(bijector.Bijector): "Nonzero floating point `Tensor`. Controls the softness of what " "would otherwise be a kink at the origin. Default is 1.0")}) def __init__(self, - event_ndims=0, hinge_softness=None, validate_args=False, name="softplus"): @@ -101,7 +100,7 @@ class Softplus(bijector.Bijector): [nonzero_check], self.hinge_softness) super(Softplus, self).__init__( - event_ndims=event_ndims, + forward_min_event_ndims=0, validate_args=validate_args, name=name) @@ -130,14 +129,12 @@ class Softplus(bijector.Bijector): # 1 - exp{-Y} approx Y. if self.hinge_softness is not None: y /= math_ops.cast(self.hinge_softness, y.dtype) - return -math_ops.reduce_sum(math_ops.log(-math_ops.expm1(-y)), - axis=self._event_dims_tensor(y)) + return -math_ops.log(-math_ops.expm1(-y)) def _forward_log_det_jacobian(self, x): if self.hinge_softness is not None: x /= math_ops.cast(self.hinge_softness, x.dtype) - return -math_ops.reduce_sum(nn_ops.softplus(-x), - axis=self._event_dims_tensor(x)) + return -nn_ops.softplus(-x) @property def hinge_softness(self): diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/square.py b/tensorflow/contrib/distributions/python/ops/bijectors/square.py index 1e9dbf3509..2ccfdc9597 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/square.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/square.py @@ -59,7 +59,7 @@ class Square(bijector.Bijector): """ self._name = name super(Square, self).__init__( - event_ndims=0, + forward_min_event_ndims=0, validate_args=validate_args, name=name) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/weibull.py b/tensorflow/contrib/distributions/python/ops/bijectors/weibull.py index 00520bcda8..39129cd22c 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/weibull.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/weibull.py @@ -50,7 +50,6 @@ class Weibull(bijector.Bijector): def __init__(self, scale=1., concentration=1., - event_ndims=0, validate_args=False, name="weibull"): """Instantiates the `Weibull` bijector. @@ -62,8 +61,6 @@ class Weibull(bijector.Bijector): concentration: Positive Float-type `Tensor` that is the same dtype and is broadcastable with `scale`. This is `k` in `Y = g(X) = 1 - exp((-x / l) ** k)`. - event_ndims: Python scalar indicating the number of dimensions associated - with a particular draw from the distribution. validate_args: Python `bool` indicating whether arguments should be checked for correctness. name: Python `str` name given to ops managed by this object. @@ -89,7 +86,7 @@ class Weibull(bijector.Bijector): ], self._concentration) super(Weibull, self).__init__( - event_ndims=event_ndims, + forward_min_event_ndims=0, validate_args=validate_args, name=name) @@ -113,22 +110,18 @@ class Weibull(bijector.Bijector): def _inverse_log_det_jacobian(self, y): y = self._maybe_assert_valid_y(y) - event_dims = self._event_dims_tensor(y) - return math_ops.reduce_sum( + return ( -math_ops.log1p(-y) + (1 / self.concentration - 1) * math_ops.log(-math_ops.log1p(-y)) + - math_ops.log(self.scale / self.concentration), - axis=event_dims) + math_ops.log(self.scale / self.concentration)) def _forward_log_det_jacobian(self, x): x = self._maybe_assert_valid_x(x) - event_dims = self._event_dims_tensor(x) - return math_ops.reduce_sum( + return ( -(x / self.scale) ** self.concentration + (self.concentration - 1) * math_ops.log(x) + math_ops.log(self.concentration) + - -self.concentration * math_ops.log(self.scale), - axis=event_dims) + -self.concentration * math_ops.log(self.scale)) def _maybe_assert_valid_x(self, x): if not self.validate_args: diff --git a/tensorflow/contrib/distributions/python/ops/conditional_transformed_distribution.py b/tensorflow/contrib/distributions/python/ops/conditional_transformed_distribution.py index 1d4c5660d8..10b4536135 100644 --- a/tensorflow/contrib/distributions/python/ops/conditional_transformed_distribution.py +++ b/tensorflow/contrib/distributions/python/ops/conditional_transformed_distribution.py @@ -20,6 +20,7 @@ from __future__ import print_function from tensorflow.contrib.distributions.python.ops import conditional_distribution from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops.distributions import transformed_distribution @@ -105,7 +106,9 @@ class ConditionalTransformedDistribution( bijector_kwargs = bijector_kwargs or {} distribution_kwargs = distribution_kwargs or {} x = self.bijector.inverse(y, **bijector_kwargs) - ildj = self.bijector.inverse_log_det_jacobian(y, **bijector_kwargs) + event_ndims = self._maybe_get_event_ndims_statically() + ildj = self.bijector.inverse_log_det_jacobian( + y, event_ndims=event_ndims, **bijector_kwargs) if self.bijector._is_injective: # pylint: disable=protected-access return self._finish_log_prob_for_one_fiber(y, x, ildj, distribution_kwargs) @@ -128,7 +131,9 @@ class ConditionalTransformedDistribution( bijector_kwargs = bijector_kwargs or {} distribution_kwargs = distribution_kwargs or {} x = self.bijector.inverse(y, **bijector_kwargs) - ildj = self.bijector.inverse_log_det_jacobian(y, **bijector_kwargs) + event_ndims = self._maybe_get_event_ndims_statically() + ildj = self.bijector.inverse_log_det_jacobian( + y, event_ndims=event_ndims, **bijector_kwargs) if self.bijector._is_injective: # pylint: disable=protected-access return self._finish_prob_for_one_fiber(y, x, ildj, distribution_kwargs) @@ -214,3 +219,15 @@ class ConditionalTransformedDistribution( # implies the qth quantile of Y is g(x_q). inv_cdf = self.distribution.quantile(value, **distribution_kwargs) return self.bijector.forward(inv_cdf, **bijector_kwargs) + + def _maybe_get_event_ndims_statically(self): + if self.event_shape.ndims is not None: + return self.event_shape.ndims + + event_ndims = array_ops.size(self.event_shape_tensor()) + static_event_ndims = tensor_util.constant_value(event_ndims) + + if static_event_ndims is not None: + return static_event_ndims + + return event_ndims diff --git a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py index 92f2bba182..3314181898 100644 --- a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py +++ b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py @@ -114,7 +114,7 @@ def quadrature_scheme_lognormal_quantiles( # Create a LogNormal distribution. dist = transformed_lib.TransformedDistribution( distribution=normal_lib.Normal(loc=loc, scale=scale), - bijector=Exp(event_ndims=0), + bijector=Exp(), validate_args=validate_args) batch_ndims = dist.batch_shape.ndims if batch_ndims is None: diff --git a/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py b/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py index f56ba07816..02cf3c7992 100644 --- a/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py +++ b/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py @@ -409,5 +409,5 @@ class RelaxedOneHotCategorical( validate_args=validate_args, allow_nan_stats=allow_nan_stats) super(RelaxedOneHotCategorical, self).__init__(dist, - bijectors.Exp(event_ndims=1), + bijectors.Exp(), name=name) diff --git a/tensorflow/contrib/distributions/python/ops/sinh_arcsinh.py b/tensorflow/contrib/distributions/python/ops/sinh_arcsinh.py index 0d8a192691..cde6d85500 100644 --- a/tensorflow/contrib/distributions/python/ops/sinh_arcsinh.py +++ b/tensorflow/contrib/distributions/python/ops/sinh_arcsinh.py @@ -166,13 +166,13 @@ class SinhArcsinh(transformed_distribution.TransformedDistribution): # Make the SAS bijector, 'F'. f = bijectors.SinhArcsinh( - skewness=skewness, tailweight=tailweight, event_ndims=0) + skewness=skewness, tailweight=tailweight) if has_default_skewness: f_noskew = f else: f_noskew = bijectors.SinhArcsinh( skewness=skewness.dtype.as_numpy_dtype(0.), - tailweight=tailweight, event_ndims=0) + tailweight=tailweight) # Make the AffineScalar bijector, Z --> loc + scale * Z (2 / F_0(2)) c = 2 * scale / f_noskew.forward(ops.convert_to_tensor(2, dtype=dtype)) diff --git a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py index 971d65c4a6..da271a852d 100644 --- a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py +++ b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py @@ -427,7 +427,6 @@ class VectorDiffeomixture(distribution_lib.Distribution): self._endpoint_affine = [ AffineLinearOperator(shift=loc_, scale=scale_, - event_ndims=1, validate_args=validate_args, name="endpoint_affine_{}".format(k)) for k, (loc_, scale_) in enumerate(zip(loc, scale))] @@ -467,7 +466,6 @@ class VectorDiffeomixture(distribution_lib.Distribution): self._interpolated_affine = [ AffineLinearOperator(shift=loc_, scale=scale_, - event_ndims=1, validate_args=validate_args, name="interpolated_affine_{}".format(k)) for k, (loc_, scale_) in enumerate(zip( @@ -621,9 +619,11 @@ class VectorDiffeomixture(distribution_lib.Distribution): log_prob = math_ops.reduce_sum(self.distribution.log_prob(y), axis=-2) # Because the affine transformation has a constant Jacobian, it is the case # that `affine.fldj(x) = -affine.ildj(x)`. This is not true in general. - fldj = array_ops.stack( - [aff.forward_log_det_jacobian(x) for aff in self.interpolated_affine], - axis=-1) + fldj = array_ops.stack([ + aff.forward_log_det_jacobian( + x, + event_ndims=array_ops.rank(self.event_shape_tensor()) + ) for aff in self.interpolated_affine], axis=-1) return math_ops.reduce_logsumexp( self.mixture_distribution.logits - fldj + log_prob, axis=-1) diff --git a/tensorflow/contrib/distributions/python/ops/vector_sinh_arcsinh_diag.py b/tensorflow/contrib/distributions/python/ops/vector_sinh_arcsinh_diag.py index 003c66b941..05919be124 100644 --- a/tensorflow/contrib/distributions/python/ops/vector_sinh_arcsinh_diag.py +++ b/tensorflow/contrib/distributions/python/ops/vector_sinh_arcsinh_diag.py @@ -215,13 +215,13 @@ class VectorSinhArcsinhDiag(transformed_distribution.TransformedDistribution): tailweight = ops.convert_to_tensor( tailweight, dtype=dtype, name="tailweight") f = bijectors.SinhArcsinh( - skewness=skewness, tailweight=tailweight, event_ndims=1) + skewness=skewness, tailweight=tailweight) if has_default_skewness: f_noskew = f else: f_noskew = bijectors.SinhArcsinh( skewness=skewness.dtype.as_numpy_dtype(0.), - tailweight=tailweight, event_ndims=0) + tailweight=tailweight) # Make the Affine bijector, Z --> loc + C * Z. c = 2 * scale_diag_part / f_noskew.forward( diff --git a/tensorflow/python/kernel_tests/distributions/bijector_test.py b/tensorflow/python/kernel_tests/distributions/bijector_test.py index 9f9fb5c0bb..18582241e2 100644 --- a/tensorflow/python/kernel_tests/distributions/bijector_test.py +++ b/tensorflow/python/kernel_tests/distributions/bijector_test.py @@ -20,6 +20,7 @@ from __future__ import print_function import abc +import numpy as np import six from tensorflow.python.framework import constant_op @@ -43,11 +44,10 @@ class BaseBijectorTest(test.TestCase): """Minimal specification of a `Bijector`.""" def __init__(self): - super(_BareBonesBijector, self).__init__() + super(_BareBonesBijector, self).__init__(forward_min_event_ndims=0) with self.test_session() as sess: bij = _BareBonesBijector() - self.assertEqual(None, bij.event_ndims) self.assertEqual([], bij.graph_parents) self.assertEqual(False, bij.is_constant_jacobian) self.assertEqual(False, bij.validate_args) @@ -67,13 +67,21 @@ class BaseBijectorTest(test.TestCase): self.assertAllEqual(shape, inverse_event_shape_) self.assertAllEqual(shape, bij.inverse_event_shape(shape)) - for fn in ["forward", - "inverse", - "inverse_log_det_jacobian", - "forward_log_det_jacobian"]: - with self.assertRaisesRegexp( - NotImplementedError, fn + " not implemented"): - getattr(bij, fn)(0) + with self.assertRaisesRegexp( + NotImplementedError, "inverse not implemented"): + bij.inverse(0) + + with self.assertRaisesRegexp( + NotImplementedError, "forward not implemented"): + bij.forward(0) + + with self.assertRaisesRegexp( + NotImplementedError, "inverse_log_det_jacobian not implemented"): + bij.inverse_log_det_jacobian(0, event_ndims=0) + + with self.assertRaisesRegexp( + NotImplementedError, "forward_log_det_jacobian not implemented"): + bij.forward_log_det_jacobian(0, event_ndims=0) class IntentionallyMissingError(Exception): @@ -85,7 +93,7 @@ class BrokenBijector(bijector.Bijector): def __init__(self, forward_missing=False, inverse_missing=False): super(BrokenBijector, self).__init__( - event_ndims=0, validate_args=False, name="broken") + validate_args=False, forward_min_event_ndims=0, name="broken") self._forward_missing = forward_missing self._inverse_missing = inverse_missing @@ -120,35 +128,42 @@ class BijectorCachingTestBase(object): def testCachingOfForwardResults(self): broken_bijector = self.broken_bijector_cls(inverse_missing=True) - with self.test_session(): - x = constant_op.constant(1.1) + x = constant_op.constant(1.1) + + # Call forward and forward_log_det_jacobian one-by-one (not together). + y = broken_bijector.forward(x) + _ = broken_bijector.forward_log_det_jacobian(x, event_ndims=0) - # Call forward and forward_log_det_jacobian one-by-one (not together). - y = broken_bijector.forward(x) - _ = broken_bijector.forward_log_det_jacobian(x) + # Now, everything should be cached if the argument is y. + broken_bijector.inverse_log_det_jacobian(y, event_ndims=0) + try: + broken_bijector.inverse(y) + broken_bijector.inverse_log_det_jacobian(y, event_ndims=0) + except IntentionallyMissingError: + raise AssertionError("Tests failed! Cached values not used.") - # Now, everything should be cached if the argument is y. - try: - broken_bijector.inverse(y) - broken_bijector.inverse_log_det_jacobian(y) - except IntentionallyMissingError: - raise AssertionError("Tests failed! Cached values not used.") + # Different event_ndims should not be cached. + with self.assertRaises(IntentionallyMissingError): + broken_bijector.inverse_log_det_jacobian(y, event_ndims=1) def testCachingOfInverseResults(self): broken_bijector = self.broken_bijector_cls(forward_missing=True) - with self.test_session(): - y = constant_op.constant(1.1) + y = constant_op.constant(1.1) - # Call inverse and inverse_log_det_jacobian one-by-one (not together). - x = broken_bijector.inverse(y) - _ = broken_bijector.inverse_log_det_jacobian(y) + # Call inverse and inverse_log_det_jacobian one-by-one (not together). + x = broken_bijector.inverse(y) + _ = broken_bijector.inverse_log_det_jacobian(y, event_ndims=0) - # Now, everything should be cached if the argument is x. - try: - broken_bijector.forward(x) - broken_bijector.forward_log_det_jacobian(x) - except IntentionallyMissingError: - raise AssertionError("Tests failed! Cached values not used.") + # Now, everything should be cached if the argument is x. + try: + broken_bijector.forward(x) + broken_bijector.forward_log_det_jacobian(x, event_ndims=0) + except IntentionallyMissingError: + raise AssertionError("Tests failed! Cached values not used.") + + # Different event_ndims should not be cached. + with self.assertRaises(IntentionallyMissingError): + broken_bijector.forward_log_det_jacobian(x, event_ndims=1) class BijectorCachingTest(BijectorCachingTestBase, test.TestCase): @@ -159,5 +174,107 @@ class BijectorCachingTest(BijectorCachingTestBase, test.TestCase): return BrokenBijector +class ExpOnlyJacobian(bijector.Bijector): + """Only used for jacobian calculations.""" + + def __init__(self, forward_min_event_ndims=0): + super(ExpOnlyJacobian, self).__init__( + validate_args=False, + is_constant_jacobian=False, + forward_min_event_ndims=forward_min_event_ndims, + name="exp") + + def _inverse_log_det_jacobian(self, y): + return -math_ops.log(y) + + def _forward_log_det_jacobian(self, x): + return math_ops.log(x) + + +class ConstantJacobian(bijector.Bijector): + """Only used for jacobian calculations.""" + + def __init__(self, forward_min_event_ndims=0): + super(ConstantJacobian, self).__init__( + validate_args=False, + is_constant_jacobian=True, + forward_min_event_ndims=forward_min_event_ndims, + name="c") + + def _inverse_log_det_jacobian(self, y): + return constant_op.constant(2., y.dtype) + + def _forward_log_det_jacobian(self, x): + return constant_op.constant(-2., x.dtype) + + +class BijectorReduceEventDimsTest(test.TestCase): + """Test caching with BrokenBijector.""" + + def testReduceEventNdimsForward(self): + x = [[[1., 2.], [3., 4.]]] + bij = ExpOnlyJacobian() + self.assertAllClose( + np.log(x), + self.evaluate(bij.forward_log_det_jacobian(x, event_ndims=0))) + self.assertAllClose( + np.sum(np.log(x), axis=-1), + self.evaluate(bij.forward_log_det_jacobian(x, event_ndims=1))) + self.assertAllClose( + np.sum(np.log(x), axis=(-1, -2)), + self.evaluate(bij.forward_log_det_jacobian(x, event_ndims=2))) + + def testReduceEventNdimsForwardRaiseError(self): + x = [[[1., 2.], [3., 4.]]] + bij = ExpOnlyJacobian(forward_min_event_ndims=1) + with self.assertRaisesRegexp(ValueError, "must be larger than"): + bij.forward_log_det_jacobian(x, event_ndims=0) + + def testReduceEventNdimsInverse(self): + x = [[[1., 2.], [3., 4.]]] + bij = ExpOnlyJacobian() + self.assertAllClose( + -np.log(x), + self.evaluate(bij.inverse_log_det_jacobian(x, event_ndims=0))) + self.assertAllClose( + np.sum(-np.log(x), axis=-1), + self.evaluate(bij.inverse_log_det_jacobian(x, event_ndims=1))) + self.assertAllClose( + np.sum(-np.log(x), axis=(-1, -2)), + self.evaluate(bij.inverse_log_det_jacobian(x, event_ndims=2))) + + def testReduceEventNdimsInverseRaiseError(self): + x = [[[1., 2.], [3., 4.]]] + bij = ExpOnlyJacobian(forward_min_event_ndims=1) + with self.assertRaisesRegexp(ValueError, "must be larger than"): + bij.inverse_log_det_jacobian(x, event_ndims=0) + + def testReduceEventNdimsForwardConstJacobian(self): + x = [[[1., 2.], [3., 4.]]] + bij = ConstantJacobian() + self.assertAllClose( + -2., + self.evaluate(bij.forward_log_det_jacobian(x, event_ndims=0))) + self.assertAllClose( + -4., + self.evaluate(bij.forward_log_det_jacobian(x, event_ndims=1))) + self.assertAllClose( + -8., + self.evaluate(bij.forward_log_det_jacobian(x, event_ndims=2))) + + def testReduceEventNdimsInverseConstJacobian(self): + x = [[[1., 2.], [3., 4.]]] + bij = ConstantJacobian() + self.assertAllClose( + 2., + self.evaluate(bij.inverse_log_det_jacobian(x, event_ndims=0))) + self.assertAllClose( + 4., + self.evaluate(bij.inverse_log_det_jacobian(x, event_ndims=1))) + self.assertAllClose( + 8., + self.evaluate(bij.inverse_log_det_jacobian(x, event_ndims=2))) + + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/distributions/identity_bijector_test.py b/tensorflow/python/kernel_tests/distributions/identity_bijector_test.py index e8f9d0b728..b347c20db2 100644 --- a/tensorflow/python/kernel_tests/distributions/identity_bijector_test.py +++ b/tensorflow/python/kernel_tests/distributions/identity_bijector_test.py @@ -27,14 +27,19 @@ class IdentityBijectorTest(test.TestCase): """Tests correctness of the Y = g(X) = X transformation.""" def testBijector(self): - with self.test_session(): - bijector = identity_bijector.Identity() - self.assertEqual("identity", bijector.name) - x = [[[0.], [1.]]] - self.assertAllEqual(x, bijector.forward(x).eval()) - self.assertAllEqual(x, bijector.inverse(x).eval()) - self.assertAllEqual(0., bijector.inverse_log_det_jacobian(x).eval()) - self.assertAllEqual(0., bijector.forward_log_det_jacobian(x).eval()) + bijector = identity_bijector.Identity(validate_args=True) + self.assertEqual("identity", bijector.name) + x = [[[0.], [1.]]] + self.assertAllEqual(x, self.evaluate(bijector.forward(x))) + self.assertAllEqual(x, self.evaluate(bijector.inverse(x))) + self.assertAllEqual( + 0., + self.evaluate( + bijector.inverse_log_det_jacobian(x, event_ndims=3))) + self.assertAllEqual( + 0., + self.evaluate( + bijector.forward_log_det_jacobian(x, event_ndims=3))) def testScalarCongruency(self): with self.test_session(): diff --git a/tensorflow/python/ops/distributions/bijector_impl.py b/tensorflow/python/ops/distributions/bijector_impl.py index ed435557fd..4ebc600d03 100644 --- a/tensorflow/python/ops/distributions/bijector_impl.py +++ b/tensorflow/python/ops/distributions/bijector_impl.py @@ -23,7 +23,6 @@ import collections import contextlib import re -import numpy as np import six from tensorflow.python.framework import dtypes @@ -31,8 +30,8 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import tensor_util from tensorflow.python.ops import array_ops +from tensorflow.python.ops import check_ops from tensorflow.python.ops import math_ops -from tensorflow.python.util.tf_export import tf_export __all__ = [ @@ -41,23 +40,24 @@ __all__ = [ class _Mapping(collections.namedtuple( - "_Mapping", ["x", "y", "ildj", "kwargs"])): + "_Mapping", ["x", "y", "ildj_map", "kwargs"])): """Helper class to make it easier to manage caching in `Bijector`.""" - def __new__(cls, x=None, y=None, ildj=None, kwargs=None): + def __new__(cls, x=None, y=None, ildj_map=None, kwargs=None): """Custom __new__ so namedtuple items have defaults. Args: x: `Tensor`. Forward. y: `Tensor`. Inverse. - ildj: `Tensor`. Inverse log det Jacobian. + ildj_map: `Dictionary`. This is a mapping from event_ndims to a `Tensor` + representing the inverse log det jacobian. kwargs: Python dictionary. Extra args supplied to forward/inverse/etc functions. Returns: mapping: New instance of _Mapping. """ - return super(_Mapping, cls).__new__(cls, x, y, ildj, kwargs) + return super(_Mapping, cls).__new__(cls, x, y, ildj_map, kwargs) @property def x_key(self): @@ -69,13 +69,14 @@ class _Mapping(collections.namedtuple( """Returns key used for caching X=g^{-1}(Y).""" return (self.y,) + self._deep_tuple(tuple(sorted(self.kwargs.items()))) - def merge(self, x=None, y=None, ildj=None, kwargs=None, mapping=None): + def merge(self, x=None, y=None, ildj_map=None, kwargs=None, mapping=None): """Returns new _Mapping with args merged with self. Args: x: `Tensor`. Forward. y: `Tensor`. Inverse. - ildj: `Tensor`. Inverse log det Jacobian. + ildj_map: `Dictionary`. This is a mapping from event_ndims to a `Tensor` + representing the inverse log det jacobian. kwargs: Python dictionary. Extra args supplied to forward/inverse/etc functions. mapping: Instance of _Mapping to merge. Can only be specified if no other @@ -88,15 +89,30 @@ class _Mapping(collections.namedtuple( ValueError: if mapping and any other arg is not `None`. """ if mapping is None: - mapping = _Mapping(x=x, y=y, ildj=ildj, kwargs=kwargs) - elif not all(arg is None for arg in [x, y, ildj, kwargs]): - raise ValueError("Cannot specify mapping and individual args.") + mapping = _Mapping(x=x, y=y, ildj_map=ildj_map, kwargs=kwargs) + elif any(arg is not None for arg in [x, y, ildj_map, kwargs]): + raise ValueError("Cannot simultaneously specify mapping and individual " + "arguments.") + return _Mapping( x=self._merge(self.x, mapping.x), y=self._merge(self.y, mapping.y), - ildj=self._merge(self.ildj, mapping.ildj), + ildj_map=self._merge_dicts(self.ildj_map, mapping.ildj_map), kwargs=self._merge(self.kwargs, mapping.kwargs)) + def _merge_dicts(self, old=None, new=None): + """Helper to merge two dictionaries.""" + old = dict() if old is None else old + new = dict() if new is None else new + for k, v in six.iteritems(new): + val = old.get(k, None) + if val is not None and val != v: + raise ValueError("Found different value for existing key " + "(key:{} old_value:{} new_value:{}".format( + k, old[k], v)) + old[k] = v + return old + def _merge(self, old, new): """Helper to merge which handles merging one value.""" if old is None: @@ -112,7 +128,6 @@ class _Mapping(collections.namedtuple( @six.add_metaclass(abc.ABCMeta) -@tf_export("distributions.bijectors.Bijector") class Bijector(object): r"""Interface for transformations of a `Distribution` sample. @@ -137,11 +152,11 @@ class Bijector(object): 2. Inverse\ Useful for "reversing" a transformation to compute one probability in terms of another. - 3. `(log o det o Jacobian o inverse)(x)`\ + 3. `log_det_jacobian(x)`\ "The log of the determinant of the matrix of all first-order partial derivatives of the inverse function."\ Useful for inverting a transformation to compute one probability in terms - of another. Geometrically, the det(Jacobian) is the volume of the + of another. Geometrically, the Jacobian determinant is the volume of the transformation and is used to scale the probability. By convention, transformations of random variables are named in terms of the @@ -164,7 +179,7 @@ class Bijector(object): ```python def transformed_log_prob(bijector, log_prob, x): - return (bijector.inverse_log_det_jacobian(x) + + return (bijector.inverse_log_det_jacobian(x, event_ndims=0) + log_prob(bijector.inverse(x))) ``` @@ -199,9 +214,11 @@ class Bijector(object): ```python class Exp(Bijector): - def __init__(self, event_ndims=0, validate_args=False, name="exp"): + def __init__(self, validate_args=False, name="exp"): super(Exp, self).__init__( - event_ndims=event_ndims, validate_args=validate_args, name=name) + validate_args=validate_args, + forward_min_event_ndims=0, + name=name) def _forward(self, x): return math_ops.exp(x) @@ -213,10 +230,11 @@ class Bijector(object): return -self._forward_log_det_jacobian(self._inverse(y)) def _forward_log_det_jacobian(self, x): - if self.event_ndims is None: - raise ValueError("Jacobian requires known event_ndims.") - event_dims = array_ops.shape(x)[-self.event_ndims:] - return math_ops.reduce_sum(x, axis=event_dims) + # Notice that we needn't do any reducing, even when`event_ndims > 0`. + # The base Bijector class will handle reducing for us; it knows how + # to do so because we called `super` `__init__` with + # `forward_min_event_ndims = 0`. + return x ``` - "Affine" @@ -237,18 +255,50 @@ class Bijector(object): MultivariateNormal(inv(sqrtSigma) * (y - mu); 0, I_d) ``` - #### Jacobian + #### Min_event_ndims and Naming + + Bijectors are named for the dimensionality of data they act on (i.e. without + broadcasting). We can think of bijectors having an intrinsic `min_event_ndims` + , which is the minimum number of dimensions for the bijector act on. For + instance, a Cholesky decomposition requires a matrix, and hence + `min_event_ndims=2`. + + Some examples: + + `AffineScalar: min_event_ndims=0` + `Affine: min_event_ndims=1` + `Cholesky: min_event_ndims=2` + `Exp: min_event_ndims=0` + `Sigmoid: min_event_ndims=0` + `SoftmaxCentered: min_event_ndims=1` + + Note the difference between `Affine` and `AffineScalar`. `AffineScalar` + operates on scalar events, whereas `Affine` operates on vector-valued events. - The Jacobian is a reduction over event dims. To see this, consider the `Exp` - `Bijector` applied to a `Tensor` which has sample, batch, and event (S, B, E) - shape semantics. Suppose the `Tensor`'s partitioned-shape is `(S=[4], B=[2], - E=[3, 3])`. The shape of the `Tensor` returned by `forward` and `inverse` is - unchanged, i.e., `[4, 2, 3, 3]`. However the shape returned by - `inverse_log_det_jacobian` is `[4, 2]` because the Jacobian is a reduction - over the event dimensions. + More generally, there is a `forward_min_event_ndims` and an + `inverse_min_event_ndims`. In most cases, these will be the same. + However, for some shape changing bijectors, these will be different + (e.g. a bijector which pads an extra dimension at the end, might have + `forward_min_event_ndims=0` and `inverse_min_event_ndims=1`. - It is sometimes useful to implement the inverse Jacobian as the negative - forward Jacobian. For example, + + #### Jacobian Determinant + + The Jacobian determinant is a reduction over `event_ndims - min_event_ndims` + (`forward_min_event_ndims` for `forward_log_det_jacobian` and + `inverse_min_event_ndims` for `inverse_log_det_jacobian`). + To see this, consider the `Exp` `Bijector` applied to a `Tensor` which has + sample, batch, and event (S, B, E) shape semantics. Suppose the `Tensor`'s + partitioned-shape is `(S=[4], B=[2], E=[3, 3])`. The shape of the `Tensor` + returned by `forward` and `inverse` is unchanged, i.e., `[4, 2, 3, 3]`. + However the shape returned by `inverse_log_det_jacobian` is `[4, 2]` because + the Jacobian determinant is a reduction over the event dimensions. + + Another example is the `Affine` `Bijector`. Because `min_event_ndims = 1`, the + Jacobian determinant reduction is over `event_ndims - 1`. + + It is sometimes useful to implement the inverse Jacobian determinant as the + negative forward Jacobian determinant. For example, ```python def _inverse_log_det_jacobian(self, y): @@ -279,9 +329,54 @@ class Bijector(object): The claim follows from [properties of determinant]( https://en.wikipedia.org/wiki/Determinant#Multiplicativity_and_matrix_groups). - Generally its preferable to directly implement the inverse Jacobian. This - should have superior numerical stability and will often share subgraphs with - the `_inverse` implementation. + Generally its preferable to directly implement the inverse Jacobian + determinant. This should have superior numerical stability and will often + share subgraphs with the `_inverse` implementation. + + #### Is_constant_jacobian + + Certain bijectors will have constant jacobian matrices. For instance, the + `Affine` bijector encodes multiplication by a matrix plus a shift, with + jacobian matrix, the same aforementioned matrix. + + `is_constant_jacobian` encodes the fact that the jacobian matrix is constant. + The semantics of this argument are the following: + + * Repeated calls to "log_det_jacobian" functions with the same + `event_ndims` (but not necessarily same input), will return the first + computed jacobian (because the matrix is constant, and hence is input + independent). + * `log_det_jacobian` implementations are merely broadcastable to the true + `log_det_jacobian` (because, again, the jacobian matrix is input + independent). Specifically, `log_det_jacobian` is implemented as the + log jacobian determinant for a single input. + + ```python + class Identity(Bijector): + + def __init__(self, validate_args=False, name="identity"): + super(Identity, self).__init__( + is_constant_jacobian=True, + validate_args=validate_args, + forward_min_event_ndims=0, + name=name) + + def _forward(self, x): + return x + + def _inverse(self, y): + return y + + def _inverse_log_det_jacobian(self, y): + return -self._forward_log_det_jacobian(self._inverse(y)) + + def _forward_log_det_jacobian(self, x): + # The full log jacobian determinant would be array_ops.zero_like(x). + # However, we circumvent materializing that, since the jacobian + # calculation is input independent, and we specify it for one input. + return constant_op.constant(0., x.dtype.base_dtype) + + ``` #### Subclass Requirements @@ -364,14 +459,14 @@ class Bijector(object): ==> (-1., 1.) # The |dX/dY| is constant, == 1. So Log|dX/dY| == 0. - abs.inverse_log_det_jacobian(1.) + abs.inverse_log_det_jacobian(1., event_ndims=0) ==> (0., 0.) # Special case handling of 0. abs.inverse(0.) ==> (0., 0.) - abs.inverse_log_det_jacobian(0.) + abs.inverse_log_det_jacobian(0., event_ndims=0) ==> (0., 0.) ``` @@ -379,11 +474,12 @@ class Bijector(object): @abc.abstractmethod def __init__(self, - event_ndims=None, graph_parents=None, is_constant_jacobian=False, validate_args=False, dtype=None, + forward_min_event_ndims=None, + inverse_min_event_ndims=None, name=None): """Constructs Bijector. @@ -392,42 +488,61 @@ class Bijector(object): Examples: ```python - # Create the Y = g(X) = X transform which operates on vector events. - identity = Identity(event_ndims=1) + # Create the Y = g(X) = X transform. + identity = Identity() - # Create the Y = g(X) = exp(X) transform which operates on matrices. - exp = Exp(event_ndims=2) + # Create the Y = g(X) = exp(X) transform. + exp = Exp() ``` See `Bijector` subclass docstring for more details and specific examples. Args: - event_ndims: number of dimensions associated with event coordinates. graph_parents: Python list of graph prerequisites of this `Bijector`. - is_constant_jacobian: Python `bool` indicating that the Jacobian is not a - function of the input. + is_constant_jacobian: Python `bool` indicating that the Jacobian matrix is + not a function of the input. validate_args: Python `bool`, default `False`. Whether to validate input with asserts. If `validate_args` is `False`, and the inputs are invalid, correct behavior is not guaranteed. dtype: `tf.dtype` supported by this `Bijector`. `None` means dtype is not enforced. + forward_min_event_ndims: Python `integer` indicating the minimum number of + dimensions `forward` operates on. + inverse_min_event_ndims: Python `integer` indicating the minimum number of + dimensions `inverse` operates on. Will be set to + `forward_min_event_ndims` by default, if no value is provided. name: The name to give Ops created by the initializer. Raises: + ValueError: If neither `forward_min_event_ndims` and + `inverse_min_event_ndims` are specified, or if either of them is + negative. ValueError: If a member of `graph_parents` is not a `Tensor`. """ - self._event_ndims = ( - ops.convert_to_tensor(event_ndims, dtype=dtypes.int32) - if event_ndims is not None else None) self._graph_parents = graph_parents or [] + + if forward_min_event_ndims is None and inverse_min_event_ndims is None: + raise ValueError("Must specify at least one of `forward_min_event_ndims` " + "and `inverse_min_event_ndims`.") + elif inverse_min_event_ndims is None: + inverse_min_event_ndims = forward_min_event_ndims + elif forward_min_event_ndims is None: + forward_min_event_ndims = inverse_min_event_ndims + + if forward_min_event_ndims < 0: + raise ValueError("forward_min_event_ndims must be a non-negative " + "integer.") + if inverse_min_event_ndims < 0: + raise ValueError("inverse_min_event_ndims must be a non-negative " + "integer.") + self._forward_min_event_ndims = forward_min_event_ndims + self._inverse_min_event_ndims = inverse_min_event_ndims self._is_constant_jacobian = is_constant_jacobian + self._constant_ildj_map = {} self._validate_args = validate_args self._dtype = dtype self._from_y = {} self._from_x = {} - # Using abbreviation ildj for "inverse log det Jacobian." - # This variable is not `None` iff is_constant_jacobian is `True`. - self._constant_ildj = None if name: self._name = name else: @@ -442,21 +557,27 @@ class Bijector(object): if t is None or not tensor_util.is_tensor(t): raise ValueError("Graph parent item %d is not a Tensor; %s." % (i, t)) - @property - def event_ndims(self): - """Returns then number of event dimensions this bijector operates on.""" - return self._event_ndims - @property def graph_parents(self): """Returns this `Bijector`'s graph_parents as a Python list.""" return self._graph_parents + @property + def forward_min_event_ndims(self): + """Returns the minimal number of dimensions bijector.forward operates on.""" + return self._forward_min_event_ndims + + @property + def inverse_min_event_ndims(self): + """Returns the minimal number of dimensions bijector.inverse operates on.""" + return self._inverse_min_event_ndims + @property def is_constant_jacobian(self): - """Returns true iff the Jacobian is not a function of x. + """Returns true iff the Jacobian matrix is not a function of x. - Note: Jacobian is either constant for both forward and inverse or neither. + Note: Jacobian matrix is either constant for both forward and inverse or + neither. Returns: is_constant_jacobian: Python `bool`. @@ -653,36 +774,57 @@ class Bijector(object): return self._call_inverse(y, name) def _inverse_log_det_jacobian(self, y): - """Subclass implementation of `inverse_log_det_jacobian` public function.""" + """Subclass implementation of `inverse_log_det_jacobian` public function. + + In particular, this method differs from the public function, in that it + does not take `event_ndims`. Thus, this implements the minimal Jacobian + determinant calculation (i.e. over `inverse_min_event_ndims`). + + Args: + y: `Tensor`. The input to the "inverse_log_det_jacobian" evaluation. + Returns: + inverse_log_det_jacobian: `Tensor`, if this bijector is injective. + If not injective, returns the k-tuple containing jacobians for the + unique `k` points `(x1, ..., xk)` such that `g(xi) = y`. + """ raise NotImplementedError("inverse_log_det_jacobian not implemented.") - def _call_inverse_log_det_jacobian(self, y, name, **kwargs): + def _call_inverse_log_det_jacobian(self, y, event_ndims, name, **kwargs): with self._name_scope(name, [y]): - if self._constant_ildj is not None: - return self._constant_ildj + if event_ndims in self._constant_ildj_map: + return self._constant_ildj_map[event_ndims] y = ops.convert_to_tensor(y, name="y") self._maybe_assert_dtype(y) if not self._is_injective: # No caching for non-injective - return self._inverse_log_det_jacobian(y, **kwargs) + ildjs = self._inverse_log_det_jacobian(y, **kwargs) + return tuple(self._reduce_jacobian_det_over_event( + y, ildj, self.inverse_min_event_ndims, event_ndims) + for ildj in ildjs) mapping = self._lookup(y=y, kwargs=kwargs) - if mapping.ildj is not None: - return mapping.ildj + if mapping.ildj_map is not None and event_ndims in mapping.ildj_map: + return mapping.ildj_map[event_ndims] try: x = None # Not needed; leave cache as is. ildj = self._inverse_log_det_jacobian(y, **kwargs) + ildj = self._reduce_jacobian_det_over_event( + y, ildj, self.inverse_min_event_ndims, event_ndims) except NotImplementedError as original_exception: try: x = mapping.x if mapping.x is not None else self._inverse(y, **kwargs) ildj = -self._forward_log_det_jacobian(x, **kwargs) + ildj = self._reduce_jacobian_det_over_event( + x, ildj, self.forward_min_event_ndims, event_ndims) except NotImplementedError: raise original_exception - mapping = mapping.merge(x=x, ildj=ildj) + + mapping = mapping.merge(x=x, ildj_map={event_ndims: ildj}) self._cache(mapping) if self.is_constant_jacobian: - self._constant_ildj = mapping.ildj - return mapping.ildj + self._constant_ildj_map[event_ndims] = ildj + return ildj - def inverse_log_det_jacobian(self, y, name="inverse_log_det_jacobian"): + def inverse_log_det_jacobian( + self, y, event_ndims, name="inverse_log_det_jacobian"): """Returns the (log o det o Jacobian o inverse)(y). Mathematically, returns: `log(det(dX/dY))(Y)`. (Recall that: `X=g^{-1}(Y)`.) @@ -691,7 +833,12 @@ class Bijector(object): evaluated at `g^{-1}(y)`. Args: - y: `Tensor`. The input to the "inverse" Jacobian evaluation. + y: `Tensor`. The input to the "inverse" Jacobian determinant evaluation. + event_ndims: Number of dimensions in the probabilistic events being + transformed. Must be greater than or equal to + `self.inverse_min_event_ndims`. The result is summed over the final + dimensions to produce a scalar Jacobian determinant for each event, + i.e. it has shape `y.shape.ndims - event_ndims` dimensions. name: The name to give this op. Returns: @@ -705,45 +852,74 @@ class Bijector(object): `self.dtype`. NotImplementedError: if `_inverse_log_det_jacobian` is not implemented. """ - return self._call_inverse_log_det_jacobian(y, name) + with ops.control_dependencies(self._check_valid_event_ndims( + min_event_ndims=self.inverse_min_event_ndims, event_ndims=event_ndims)): + return self._call_inverse_log_det_jacobian(y, event_ndims, name) def _forward_log_det_jacobian(self, x): - """Subclass implementation of `forward_log_det_jacobian`.""" + """Subclass implementation of `forward_log_det_jacobian` public function. + + In particular, this method differs from the public function, in that it + does not take `event_ndims`. Thus, this implements the minimal Jacobian + determinant calculation (i.e. over `forward_min_event_ndims`). + + Args: + x: `Tensor`. The input to the "forward_log_det_jacobian" evaluation. + + Returns: + forward_log_det_jacobian: `Tensor`, if this bijector is injective. + If not injective, returns the k-tuple containing jacobians for the + unique `k` points `(x1, ..., xk)` such that `g(xi) = y`. + """ + raise NotImplementedError( "forward_log_det_jacobian not implemented.") - def _call_forward_log_det_jacobian(self, x, name, **kwargs): + def _call_forward_log_det_jacobian(self, x, event_ndims, name, **kwargs): with self._name_scope(name, [x]): - if self._constant_ildj is not None: + if event_ndims in self._constant_ildj_map: # Need "-1. *" to avoid invalid-unary-operand-type linter warning. - return -1. * self._constant_ildj + return -1. * self._constant_ildj_map[event_ndims] x = ops.convert_to_tensor(x, name="x") self._maybe_assert_dtype(x) if not self._is_injective: - return self._forward_log_det_jacobian(x, **kwargs) # No caching. + fldjs = self._forward_log_det_jacobian(x, **kwargs) # No caching. + return tuple(self._reduce_jacobian_det_over_event( + x, fldj, self.forward_min_event_ndims, event_ndims) + for fldj in fldjs) mapping = self._lookup(x=x, kwargs=kwargs) - if mapping.ildj is not None: - return -mapping.ildj + if mapping.ildj_map is not None and event_ndims in mapping.ildj_map: + return -mapping.ildj_map[event_ndims] try: y = None # Not needed; leave cache as is. ildj = -self._forward_log_det_jacobian(x, **kwargs) + ildj = self._reduce_jacobian_det_over_event( + x, ildj, self.forward_min_event_ndims, event_ndims) except NotImplementedError as original_exception: try: y = mapping.y if mapping.y is not None else self._forward(x, **kwargs) ildj = self._inverse_log_det_jacobian(y, **kwargs) + ildj = self._reduce_jacobian_det_over_event( + y, ildj, self.inverse_min_event_ndims, event_ndims) except NotImplementedError: raise original_exception - mapping = mapping.merge(y=y, ildj=ildj) + mapping = mapping.merge(y=y, ildj_map={event_ndims: ildj}) self._cache(mapping) if self.is_constant_jacobian: - self._constant_ildj = mapping.ildj - return -mapping.ildj + self._constant_ildj_map[event_ndims] = ildj + return -ildj - def forward_log_det_jacobian(self, x, name="forward_log_det_jacobian"): + def forward_log_det_jacobian( + self, x, event_ndims, name="forward_log_det_jacobian"): """Returns both the forward_log_det_jacobian. Args: - x: `Tensor`. The input to the "forward" Jacobian evaluation. + x: `Tensor`. The input to the "forward" Jacobian determinant evaluation. + event_ndims: Number of dimensions in the probabilistic events being + transformed. Must be greater than or equal to + `self.forward_min_event_ndims`. The result is summed over the final + dimensions to produce a scalar Jacobian determinant for each event, + i.e. it has shape `x.shape.ndims - event_ndims` dimensions. name: The name to give this op. Returns: @@ -761,7 +937,9 @@ class Bijector(object): raise NotImplementedError( "forward_log_det_jacobian cannot be implemented for non-injective " "transforms.") - return self._call_forward_log_det_jacobian(x, name) + with ops.control_dependencies(self._check_valid_event_ndims( + min_event_ndims=self.forward_min_event_ndims, event_ndims=event_ndims)): + return self._call_forward_log_det_jacobian(x, event_ndims, name) @contextlib.contextmanager def _name_scope(self, name=None, values=None): @@ -779,9 +957,6 @@ class Bijector(object): def _cache(self, mapping): """Helper which stores mapping info in forward/inverse dicts.""" - if self._constant_ildj is not None: - # Fold in ildj if known constant Jacobian. - mapping = mapping.merge(ildj=self._constant_ildj) # Merging from lookup is an added check that we're not overwriting anything # which is not None. mapping = mapping.merge(mapping=self._lookup( @@ -803,22 +978,66 @@ class Bijector(object): return self._from_y.get(mapping.y_key, mapping) return mapping - def _event_dims_tensor(self, sample): - """Return a 1D `int32` tensor: `range(rank(sample))[-event_ndims:]`.""" - if self.event_ndims is None: - raise ValueError("Jacobian cannot be computed with unknown event_ndims") - static_event_ndims = tensor_util.constant_value(self.event_ndims) - static_rank = sample.get_shape().ndims - if static_event_ndims is not None and static_rank is not None: - return ops.convert_to_tensor( - static_rank + np.arange(-static_event_ndims, 0).astype(np.int32)) - - if static_event_ndims is not None: - event_range = np.arange(-static_event_ndims, 0).astype(np.int32) - else: - event_range = math_ops.range(-self.event_ndims, 0, dtype=dtypes.int32) - - if static_rank is not None: - return event_range + static_rank + def _reduce_jacobian_det_over_event( + self, y, ildj, min_event_ndims, event_ndims): + """Reduce jacobian over event_ndims - min_event_ndims.""" + if not self.is_constant_jacobian: + return math_ops.reduce_sum( + ildj, + self._get_event_reduce_dims(min_event_ndims, event_ndims)) + + # In this case, we need to tile the jacobian over the event and reduce. + y_rank = array_ops.rank(y) + y_shape = array_ops.shape(y)[ + y_rank - event_ndims : y_rank - min_event_ndims] + + ones = array_ops.ones(y_shape, ildj.dtype) + reduced_ildj = math_ops.reduce_sum( + ones * ildj, + axis=self._get_event_reduce_dims(min_event_ndims, event_ndims)) + # The multiplication by ones can change the inferred static shape so we try + # to recover as much as possible. + if (isinstance(event_ndims, int) and + y.get_shape().ndims and ildj.get_shape().ndims): + y_shape = y.get_shape() + y_shape = y_shape[y_shape.ndims - event_ndims : + y_shape.ndims - min_event_ndims] + ildj_shape = ildj.get_shape() + broadcast_shape = array_ops.broadcast_static_shape( + ildj_shape, y_shape) + reduced_ildj.set_shape( + broadcast_shape[: broadcast_shape.ndims - ( + event_ndims - min_event_ndims)]) + + return reduced_ildj + + def _get_event_reduce_dims(self, min_event_ndims, event_ndims): + """Compute the reduction dimensions given event_ndims.""" + min_event_ndims_ = (min_event_ndims if isinstance(min_event_ndims, int) + else tensor_util.constant_value(min_event_ndims)) + event_ndims_ = (event_ndims if isinstance(event_ndims, int) + else tensor_util.constant_value(event_ndims)) + + if min_event_ndims_ is not None and event_ndims_ is not None: + return [-index for index in range(1, event_ndims_ - min_event_ndims_ + 1)] else: - return event_range + array_ops.rank(sample) + reduce_ndims = event_ndims - min_event_ndims + return math_ops.range(-reduce_ndims, 0) + + def _check_valid_event_ndims(self, min_event_ndims, event_ndims): + """Check whether event_ndims is atleast min_event_ndims.""" + min_event_ndims_ = (min_event_ndims if isinstance(min_event_ndims, int) + else tensor_util.constant_value(min_event_ndims)) + event_ndims_ = (event_ndims if isinstance(event_ndims, int) + else tensor_util.constant_value(event_ndims)) + + if min_event_ndims_ is not None and event_ndims_ is not None: + if min_event_ndims_ > event_ndims_: + raise ValueError("event_ndims ({}) must be larger than " + "min_event_ndims ({})".format( + event_ndims_, min_event_ndims_)) + return [] + + if self.validate_args: + return [check_ops.assert_greater_equal(event_ndims, min_event_ndims)] + return [] diff --git a/tensorflow/python/ops/distributions/bijector_test_util.py b/tensorflow/python/ops/distributions/bijector_test_util.py index ff3535c626..784bfd5835 100644 --- a/tensorflow/python/ops/distributions/bijector_test_util.py +++ b/tensorflow/python/ops/distributions/bijector_test_util.py @@ -79,9 +79,7 @@ def assert_scalar_congruency(bijector, Raises: AssertionError: If tests fail. """ - # Checks and defaults. - assert bijector.event_ndims.eval() == 0 if sess is None: sess = ops.get_default_session() @@ -111,7 +109,10 @@ def assert_scalar_congruency(bijector, # (b - a) = \int_a^b dx = \int_{y(a)}^{y(b)} |dx/dy| dy # "change_measure_dy_dx" below is a Monte Carlo approximation to the right # hand side, which should then be close to the left, which is (b - a). - dy_dx = math_ops.exp(bijector.inverse_log_det_jacobian(uniform_y_samps)) + # We assume event_ndims=0 because we assume scalar -> scalar. The log_det + # methods will handle whether they expect event_ndims > 0. + dy_dx = math_ops.exp(bijector.inverse_log_det_jacobian( + uniform_y_samps, event_ndims=0)) # E[|dx/dy|] under Uniform[lower_y, upper_y] # = \int_{y(a)}^{y(b)} |dx/dy| dP(u), where dP(u) is the uniform measure expectation_of_dy_dx_under_uniform = math_ops.reduce_mean(dy_dx) @@ -121,7 +122,8 @@ def assert_scalar_congruency(bijector, # We'll also check that dy_dx = 1 / dx_dy. dx_dy = math_ops.exp( - bijector.forward_log_det_jacobian(bijector.inverse(uniform_y_samps))) + bijector.forward_log_det_jacobian( + bijector.inverse(uniform_y_samps), event_ndims=0)) [ forward_on_10_pts_v, @@ -158,7 +160,8 @@ def assert_scalar_congruency(bijector, dy_dx_v, np.divide(1., dx_dy_v), atol=1e-5, rtol=1e-3) -def assert_bijective_and_finite(bijector, x, y, atol=0, rtol=1e-5, sess=None): +def assert_bijective_and_finite( + bijector, x, y, event_ndims, atol=0, rtol=1e-5, sess=None): """Assert that forward/inverse (along with jacobians) are inverses and finite. It is recommended to use x and y values that are very very close to the edge @@ -168,6 +171,8 @@ def assert_bijective_and_finite(bijector, x, y, atol=0, rtol=1e-5, sess=None): bijector: A Bijector instance. x: np.array of values in the domain of bijector.forward. y: np.array of values in the domain of bijector.inverse. + event_ndims: Integer describing the number of event dimensions this bijector + operates on. atol: Absolute tolerance. rtol: Relative tolerance. sess: TensorFlow session. Defaults to the default session. @@ -197,10 +202,10 @@ def assert_bijective_and_finite(bijector, x, y, atol=0, rtol=1e-5, sess=None): ] = sess.run([ bijector.inverse(f_x), bijector.forward(g_y), - bijector.inverse_log_det_jacobian(f_x), - bijector.forward_log_det_jacobian(x), - bijector.inverse_log_det_jacobian(y), - bijector.forward_log_det_jacobian(g_y), + bijector.inverse_log_det_jacobian(f_x, event_ndims=event_ndims), + bijector.forward_log_det_jacobian(x, event_ndims=event_ndims), + bijector.inverse_log_det_jacobian(y, event_ndims=event_ndims), + bijector.forward_log_det_jacobian(g_y, event_ndims=event_ndims), f_x, g_y, ]) diff --git a/tensorflow/python/ops/distributions/bijectors.py b/tensorflow/python/ops/distributions/bijectors.py deleted file mode 100644 index 69c3a5d4c0..0000000000 --- a/tensorflow/python/ops/distributions/bijectors.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Core module for TensorFlow distribution bijectors.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - - -# go/tf-wildcard-import -# pylint: disable=wildcard-import,unused-import -from tensorflow.python.ops.distributions.bijector import Bijector -from tensorflow.python.ops.distributions.identity_bijector import Identity - -# pylint: enable=wildcard-import,unused-import -from tensorflow.python.util.all_util import remove_undocumented - -_allowed_symbols = ["Bijector", "Identity"] - -remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/python/ops/distributions/distributions.py b/tensorflow/python/ops/distributions/distributions.py index 9df7d148a5..7c4b8697d8 100644 --- a/tensorflow/python/ops/distributions/distributions.py +++ b/tensorflow/python/ops/distributions/distributions.py @@ -19,7 +19,6 @@ from __future__ import print_function # pylint: disable=wildcard-import,unused-import -from tensorflow.python.ops.distributions import bijectors from tensorflow.python.ops.distributions.bernoulli import Bernoulli from tensorflow.python.ops.distributions.beta import Beta from tensorflow.python.ops.distributions.categorical import Categorical @@ -40,7 +39,6 @@ from tensorflow.python.util.all_util import remove_undocumented _allowed_symbols = [ - "bijectors", "Bernoulli", "Beta", "Categorical", diff --git a/tensorflow/python/ops/distributions/identity_bijector.py b/tensorflow/python/ops/distributions/identity_bijector.py index 2972c3554b..8628e68f96 100644 --- a/tensorflow/python/ops/distributions/identity_bijector.py +++ b/tensorflow/python/ops/distributions/identity_bijector.py @@ -20,7 +20,6 @@ from __future__ import print_function from tensorflow.python.framework import constant_op from tensorflow.python.ops.distributions import bijector -from tensorflow.python.util.tf_export import tf_export __all__ = [ @@ -28,7 +27,6 @@ __all__ = [ ] -@tf_export("distributions.bijectors.Identity") class Identity(bijector.Bijector): """Compute Y = g(X) = X. @@ -37,7 +35,7 @@ class Identity(bijector.Bijector): ```python # Create the Y=g(X)=X transform which is intended for Tensors with 1 batch # ndim and 1 event ndim (i.e., vector of vectors). - identity = Identity(event_ndims=1) + identity = Identity() x = [[1., 2], [3, 4]] x == identity.forward(x) == identity.inverse(x) @@ -45,10 +43,10 @@ class Identity(bijector.Bijector): """ - def __init__(self, validate_args=False, event_ndims=0, name="identity"): + def __init__(self, validate_args=False, name="identity"): super(Identity, self).__init__( + forward_min_event_ndims=0, is_constant_jacobian=True, - event_ndims=event_ndims, validate_args=validate_args, name=name) diff --git a/tensorflow/python/ops/distributions/transformed_distribution.py b/tensorflow/python/ops/distributions/transformed_distribution.py index 1efcf9d32e..1ad63a8cf6 100644 --- a/tensorflow/python/ops/distributions/transformed_distribution.py +++ b/tensorflow/python/ops/distributions/transformed_distribution.py @@ -197,8 +197,7 @@ class TransformedDistribution(distribution_lib.Distribution): distribution=ds.Normal(loc=0., scale=1.), bijector=ds.bijectors.Affine( shift=-1., - scale_identity_multiplier=2., - event_ndims=0), + scale_identity_multiplier=2.) name="NormalTransformedDistribution") ``` @@ -419,48 +418,51 @@ class TransformedDistribution(distribution_lib.Distribution): # For caching to work, it is imperative that the bijector is the first to # modify the input. x = self.bijector.inverse(y) - ildj = self.bijector.inverse_log_det_jacobian(y) + event_ndims = self._maybe_get_event_ndims_statically() + + ildj = self.bijector.inverse_log_det_jacobian(y, event_ndims=event_ndims) if self.bijector._is_injective: # pylint: disable=protected-access - return self._finish_log_prob_for_one_fiber(y, x, ildj) + return self._finish_log_prob_for_one_fiber(y, x, ildj, event_ndims) lp_on_fibers = [ - self._finish_log_prob_for_one_fiber(y, x_i, ildj_i) + self._finish_log_prob_for_one_fiber(y, x_i, ildj_i, event_ndims) for x_i, ildj_i in zip(x, ildj)] return math_ops.reduce_logsumexp(array_ops.stack(lp_on_fibers), axis=0) - def _finish_log_prob_for_one_fiber(self, y, x, ildj): + def _finish_log_prob_for_one_fiber(self, y, x, ildj, event_ndims): """Finish computation of log_prob on one element of the inverse image.""" x = self._maybe_rotate_dims(x, rotate_right=True) log_prob = self.distribution.log_prob(x) if self._is_maybe_event_override: log_prob = math_ops.reduce_sum(log_prob, self._reduce_event_indices) log_prob += math_ops.cast(ildj, log_prob.dtype) - if self._is_maybe_event_override: + if self._is_maybe_event_override and isinstance(event_ndims, int): log_prob.set_shape(array_ops.broadcast_static_shape( - y.get_shape().with_rank_at_least(1)[:-1], self.batch_shape)) + x.get_shape().with_rank_at_least(1)[:-event_ndims], self.batch_shape)) return log_prob def _prob(self, y): x = self.bijector.inverse(y) - ildj = self.bijector.inverse_log_det_jacobian(y) + event_ndims = self._maybe_get_event_ndims_statically() + ildj = self.bijector.inverse_log_det_jacobian(y, event_ndims=event_ndims) if self.bijector._is_injective: # pylint: disable=protected-access - return self._finish_prob_for_one_fiber(y, x, ildj) + return self._finish_prob_for_one_fiber(y, x, ildj, event_ndims) prob_on_fibers = [ - self._finish_prob_for_one_fiber(y, x_i, ildj_i) + self._finish_prob_for_one_fiber(y, x_i, ildj_i, event_ndims) for x_i, ildj_i in zip(x, ildj)] return sum(prob_on_fibers) - def _finish_prob_for_one_fiber(self, y, x, ildj): + def _finish_prob_for_one_fiber(self, y, x, ildj, event_ndims): """Finish computation of prob on one element of the inverse image.""" x = self._maybe_rotate_dims(x, rotate_right=True) prob = self.distribution.prob(x) if self._is_maybe_event_override: prob = math_ops.reduce_prod(prob, self._reduce_event_indices) prob *= math_ops.exp(math_ops.cast(ildj, prob.dtype)) - if self._is_maybe_event_override: + if self._is_maybe_event_override and isinstance(event_ndims, int): prob.set_shape(array_ops.broadcast_static_shape( - y.get_shape().with_rank_at_least(1)[:-1], self.batch_shape)) + y.get_shape().with_rank_at_least(1)[:-event_ndims], self.batch_shape)) return prob def _log_cdf(self, y): @@ -545,10 +547,17 @@ class TransformedDistribution(distribution_lib.Distribution): _ones_like(self.distribution.batch_shape_tensor()) ], 0) entropy = array_ops.tile(entropy, multiples) - dummy = array_ops.zeros([], self.dtype) - entropy -= math_ops.cast( - self.bijector.inverse_log_det_jacobian(dummy), - entropy.dtype) + dummy = array_ops.zeros( + shape=array_ops.concat( + [self.batch_shape_tensor(), self.event_shape_tensor()], + 0), + dtype=self.dtype) + event_ndims = (self.event_shape.ndims if self.event_shape.ndims is not None + else array_ops.size(self.event_shape_tensor())) + ildj = self.bijector.inverse_log_det_jacobian( + dummy, event_ndims=event_ndims) + + entropy -= math_ops.cast(ildj, entropy.dtype) entropy.set_shape(self.batch_shape) return entropy @@ -610,3 +619,16 @@ class TransformedDistribution(distribution_lib.Distribution): n = (ndims - self._rotate_ndims) if rotate_right else self._rotate_ndims return array_ops.transpose( x, _concat_vectors(math_ops.range(n, ndims), math_ops.range(0, n))) + + def _maybe_get_event_ndims_statically(self): + if self.event_shape.ndims is not None: + return self.event_shape.ndims + + event_ndims = array_ops.size(self.event_shape_tensor()) + + static_event_ndims = tensor_util.constant_value(event_ndims) + + if static_event_ndims is not None: + return static_event_ndims + + return event_ndims diff --git a/tensorflow/tools/api/golden/tensorflow.distributions.bijectors.-bijector.pbtxt b/tensorflow/tools/api/golden/tensorflow.distributions.bijectors.-bijector.pbtxt deleted file mode 100644 index 11565bd3e4..0000000000 --- a/tensorflow/tools/api/golden/tensorflow.distributions.bijectors.-bijector.pbtxt +++ /dev/null @@ -1,65 +0,0 @@ -path: "tensorflow.distributions.bijectors.Bijector" -tf_class { - is_instance: "" - is_instance: "" - member { - name: "dtype" - mtype: "" - } - member { - name: "event_ndims" - mtype: "" - } - member { - name: "graph_parents" - mtype: "" - } - member { - name: "is_constant_jacobian" - mtype: "" - } - member { - name: "name" - mtype: "" - } - member { - name: "validate_args" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'event_ndims\', \'graph_parents\', \'is_constant_jacobian\', \'validate_args\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'False\', \'None\', \'None\'], " - } - member_method { - name: "forward" - argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'forward\'], " - } - member_method { - name: "forward_event_shape" - argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "forward_event_shape_tensor" - argspec: "args=[\'self\', \'input_shape\', \'name\'], varargs=None, keywords=None, defaults=[\'forward_event_shape_tensor\'], " - } - member_method { - name: "forward_log_det_jacobian" - argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'forward_log_det_jacobian\'], " - } - member_method { - name: "inverse" - argspec: "args=[\'self\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], " - } - member_method { - name: "inverse_event_shape" - argspec: "args=[\'self\', \'output_shape\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "inverse_event_shape_tensor" - argspec: "args=[\'self\', \'output_shape\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse_event_shape_tensor\'], " - } - member_method { - name: "inverse_log_det_jacobian" - argspec: "args=[\'self\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse_log_det_jacobian\'], " - } -} diff --git a/tensorflow/tools/api/golden/tensorflow.distributions.bijectors.-identity.pbtxt b/tensorflow/tools/api/golden/tensorflow.distributions.bijectors.-identity.pbtxt deleted file mode 100644 index 1e5fe624eb..0000000000 --- a/tensorflow/tools/api/golden/tensorflow.distributions.bijectors.-identity.pbtxt +++ /dev/null @@ -1,66 +0,0 @@ -path: "tensorflow.distributions.bijectors.Identity" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - member { - name: "dtype" - mtype: "" - } - member { - name: "event_ndims" - mtype: "" - } - member { - name: "graph_parents" - mtype: "" - } - member { - name: "is_constant_jacobian" - mtype: "" - } - member { - name: "name" - mtype: "" - } - member { - name: "validate_args" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'validate_args\', \'event_ndims\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'0\', \'identity\'], " - } - member_method { - name: "forward" - argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'forward\'], " - } - member_method { - name: "forward_event_shape" - argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "forward_event_shape_tensor" - argspec: "args=[\'self\', \'input_shape\', \'name\'], varargs=None, keywords=None, defaults=[\'forward_event_shape_tensor\'], " - } - member_method { - name: "forward_log_det_jacobian" - argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'forward_log_det_jacobian\'], " - } - member_method { - name: "inverse" - argspec: "args=[\'self\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], " - } - member_method { - name: "inverse_event_shape" - argspec: "args=[\'self\', \'output_shape\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "inverse_event_shape_tensor" - argspec: "args=[\'self\', \'output_shape\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse_event_shape_tensor\'], " - } - member_method { - name: "inverse_log_det_jacobian" - argspec: "args=[\'self\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse_log_det_jacobian\'], " - } -} diff --git a/tensorflow/tools/api/golden/tensorflow.distributions.bijectors.pbtxt b/tensorflow/tools/api/golden/tensorflow.distributions.bijectors.pbtxt deleted file mode 100644 index 1d0144f36e..0000000000 --- a/tensorflow/tools/api/golden/tensorflow.distributions.bijectors.pbtxt +++ /dev/null @@ -1,11 +0,0 @@ -path: "tensorflow.distributions.bijectors" -tf_module { - member { - name: "Bijector" - mtype: "" - } - member { - name: "Identity" - mtype: "" - } -} diff --git a/tensorflow/tools/api/golden/tensorflow.distributions.pbtxt b/tensorflow/tools/api/golden/tensorflow.distributions.pbtxt index 2fba7c506e..90b60ef074 100644 --- a/tensorflow/tools/api/golden/tensorflow.distributions.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.distributions.pbtxt @@ -68,10 +68,6 @@ tf_module { name: "Uniform" mtype: "" } - member { - name: "bijectors" - mtype: "" - } member_method { name: "kl_divergence" argspec: "args=[\'distribution_a\', \'distribution_b\', \'allow_nan_stats\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'None\'], " -- GitLab From e5201672aa664cf39725f4a52b9774d2bae43ba3 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 14:04:09 -0700 Subject: [PATCH 185/791] Adds a nodedef_fn parameter to copy_op_handler, allowing customization by mutating NodeDef before creating the copied operation. PiperOrigin-RevId: 192505209 --- .../graph_editor/tests/transform_test.py | 29 +++++++++++++++++++ tensorflow/contrib/graph_editor/transform.py | 11 ++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/graph_editor/tests/transform_test.py b/tensorflow/contrib/graph_editor/tests/transform_test.py index 2603de6407..97f38c923f 100644 --- a/tensorflow/contrib/graph_editor/tests/transform_test.py +++ b/tensorflow/contrib/graph_editor/tests/transform_test.py @@ -18,9 +18,11 @@ from __future__ import division from __future__ import print_function import collections +import functools import numpy as np from tensorflow.contrib import graph_editor as ge from tensorflow.contrib.graph_editor.tests import match +from tensorflow.core.framework import attr_value_pb2 from tensorflow.python.client import session from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes @@ -42,6 +44,7 @@ class TransformTest(test.TestCase): self.graph = ops.Graph() with self.graph.as_default(): c0 = constant_op.constant(1.0, shape=[10], name="Const") + c0.op._set_attr("_foo", attr_value_pb2.AttrValue(s=b"foo")) c1 = constant_op.constant(1.0, shape=[10], name="Const") c2 = constant_op.constant(1.0, shape=[10], name="Const") i = constant_op.constant(1.0, shape=[10], name="Input") @@ -112,6 +115,32 @@ class TransformTest(test.TestCase): top = ge.select_ops("^AddNoise_2$", graph=graph)[0] self.assertTrue(matcher2(top)) + def test_transform_nodedef_fn(self): + transformer = ge.Transformer() + + def nodedef_fn(node_def): + if "_foo" in node_def.attr: + del node_def.attr["_foo"] + node_def.attr["_bar"].s = b"bar" + return node_def + + my_copy_op_handler = functools.partial( + ge.transform.copy_op_handler, nodedef_fn=nodedef_fn) + transformer.transform_op_handler = my_copy_op_handler + + graph = ops.Graph() + transformer(self.graph, graph, "", "") + + c0_before = self.graph.get_operation_by_name("Const") + c0_after = graph.get_operation_by_name("Const") + self.assertEquals(c0_before.get_attr("_foo"), b"foo") + with self.assertRaises(ValueError): + c0_after.get_attr("_foo") + + all_ops = graph.get_operations() + for op in all_ops: + self.assertEquals(op.get_attr("_bar"), b"bar") + def test_copy_with_input_replacements(self): with self.graph.as_default(): ten = constant_op.constant(10.0, shape=[10], name="Input") diff --git a/tensorflow/contrib/graph_editor/transform.py b/tensorflow/contrib/graph_editor/transform.py index d8a48387a7..a320a3f232 100644 --- a/tensorflow/contrib/graph_editor/transform.py +++ b/tensorflow/contrib/graph_editor/transform.py @@ -129,7 +129,7 @@ def transform_op_if_inside_handler(info, op, keep_if_possible=True): return None -def copy_op_handler(info, op, new_inputs, copy_shape=True): +def copy_op_handler(info, op, new_inputs, copy_shape=True, nodedef_fn=None): """Copy a `tf.Operation`. Args: @@ -137,6 +137,11 @@ def copy_op_handler(info, op, new_inputs, copy_shape=True): op: the `tf.Operation` to be copied. new_inputs: The new inputs for this op. copy_shape: also copy the shape of the tensor + nodedef_fn: If provided, a function that will be run on the NodeDef + and should return a mutated NodeDef before a new Operation is created. + This is useful as certain features cannot be set on the Operation and + must be modified in NodeDef. + Returns: A `(op, op_outputs)` tuple containing the transformed op and its outputs. """ @@ -155,6 +160,10 @@ def copy_op_handler(info, op, new_inputs, copy_shape=True): name_ = info.graph_.unique_name(name_) node_def_.name = name_ + # Mutate NodeDef if requested: + if nodedef_fn is not None: + node_def_ = nodedef_fn(node_def_) + # Copy the other inputs needed for initialization output_types_ = op._output_types[:] input_types_ = op._input_types[:] -- GitLab From 21fb4eeb3e09fb0dea1dd12b0fff7a7bf0a33643 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 14:39:11 -0700 Subject: [PATCH 186/791] Adding support for batch_to_space_nd op with crops. PiperOrigin-RevId: 192511036 --- .../contrib/lite/kernels/batch_to_space_nd.cc | 2 + .../contrib/lite/kernels/internal/BUILD | 9 ++ .../internal/batch_to_space_nd_test.cc | 98 +++++++++++++++++++ .../internal/optimized/optimized_ops.h | 66 +++++++++++-- .../internal/reference/reference_ops.h | 27 +++-- .../contrib/lite/testing/generate_examples.py | 5 +- .../propagate_fixed_sizes.cc | 16 ++- tensorflow/contrib/lite/toco/model.h | 3 +- 8 files changed, 195 insertions(+), 31 deletions(-) create mode 100644 tensorflow/contrib/lite/kernels/internal/batch_to_space_nd_test.cc diff --git a/tensorflow/contrib/lite/kernels/batch_to_space_nd.cc b/tensorflow/contrib/lite/kernels/batch_to_space_nd.cc index bc438f99c6..90edf4f9e3 100644 --- a/tensorflow/contrib/lite/kernels/batch_to_space_nd.cc +++ b/tensorflow/contrib/lite/kernels/batch_to_space_nd.cc @@ -123,6 +123,8 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { GetTensorDims(op_context.input), \ GetTensorData(op_context.block_shape), \ GetTensorDims(op_context.block_shape), \ + GetTensorData(op_context.crops), \ + GetTensorDims(op_context.crops), \ GetTensorData(op_context.output), \ GetTensorDims(op_context.output)) switch (op_context.input->type) { // Already know in/out types are same. diff --git a/tensorflow/contrib/lite/kernels/internal/BUILD b/tensorflow/contrib/lite/kernels/internal/BUILD index 32a0acf888..67dd188496 100644 --- a/tensorflow/contrib/lite/kernels/internal/BUILD +++ b/tensorflow/contrib/lite/kernels/internal/BUILD @@ -432,4 +432,13 @@ cc_library( ), ) +cc_test( + name = "batch_to_space_nd_test", + srcs = ["batch_to_space_nd_test.cc"], + deps = [ + ":optimized_base", + "@com_google_googletest//:gtest_main", + ], +) + exports_files(["optimized/eigen_tensor_reduced_instantiations_oss.h"]) diff --git a/tensorflow/contrib/lite/kernels/internal/batch_to_space_nd_test.cc b/tensorflow/contrib/lite/kernels/internal/batch_to_space_nd_test.cc new file mode 100644 index 0000000000..5a2901ac8c --- /dev/null +++ b/tensorflow/contrib/lite/kernels/internal/batch_to_space_nd_test.cc @@ -0,0 +1,98 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h" + +#include + +namespace tflite { +namespace { + +// A light wrapper of GetIndexRange which returns a pair of start / end +// indices. +std::pair GetIndexRange(int spatial_index_dim, int block_shape_dim, + int input_dim, int output_dim) { + int index_start = 0; + int index_end = 0; + optimized_ops::GetIndexRange(spatial_index_dim, block_shape_dim, input_dim, + output_dim, &index_start, &index_end); + return {index_start, index_end}; +} + +TEST(BatchToSpaceNDTest, TestIndexRange) { + // Simple test case, no cropping. + EXPECT_EQ(GetIndexRange(/*spatial_index_dim=*/3, /*block_shape_dim=*/6, + /*input_dim=*/1, /*output_dim=*/6), + std::make_pair(0, 1)); + + // No cropping and input_dim > 1. + EXPECT_EQ(GetIndexRange(/*spatial_index_dim=*/2, /*block_shape_dim=*/6, + /*input_dim=*/5, /*output_dim=*/30), + std::make_pair(0, 5)); + + // With small cropping values (can be either at the beginning or at the end). + EXPECT_EQ(GetIndexRange(/*spatial_index_dim=*/0, /*block_shape_dim=*/2, + /*input_dim=*/3, /*output_dim=*/4), + std::make_pair(0, 2)); + + // With positive cropping values at the beginning. + EXPECT_EQ(GetIndexRange(/*spatial_index_dim=*/-2, /*block_shape_dim=*/2, + /*input_dim=*/3, /*output_dim=*/4), + std::make_pair(1, 3)); + + // Large crop at the beginning. + EXPECT_EQ(GetIndexRange(/*spatial_index_dim=*/-30, /*block_shape_dim=*/5, + /*input_dim=*/7, /*output_dim=*/5), + std::make_pair(6, 7)); + + EXPECT_EQ(GetIndexRange(/*spatial_index_dim=*/-26, /*block_shape_dim=*/5, + /*input_dim=*/7, /*output_dim=*/5), + std::make_pair(6, 7)); + + // Large crop at the end. + EXPECT_EQ(GetIndexRange(/*spatial_index_dim=*/0, /*block_shape_dim=*/5, + /*input_dim=*/7, /*output_dim=*/5), + std::make_pair(0, 1)); + + EXPECT_EQ(GetIndexRange(/*spatial_index_dim=*/4, /*block_shape_dim=*/5, + /*input_dim=*/7, /*output_dim=*/5), + std::make_pair(0, 1)); + + // Rounding up incorrectly will fail this test. + EXPECT_EQ(GetIndexRange(/*spatial_index_dim=*/3, /*block_shape_dim=*/5, + /*input_dim=*/7, /*output_dim=*/5), + std::make_pair(0, 1)); + + // Extreme cropping with output of a single spatial location. + // Valid position 1, when large crop at the end. + EXPECT_EQ(GetIndexRange(/*spatial_index_dim=*/0, /*block_shape_dim=*/5, + /*input_dim=*/7, /*output_dim=*/1), + std::make_pair(0, 1)); + + // Valid position 2, when large crop at the beginning. + EXPECT_EQ(GetIndexRange(/*spatial_index_dim=*/-30, /*block_shape_dim=*/5, + /*input_dim=*/7, /*output_dim=*/1), + std::make_pair(6, 7)); + + // Invalid positions. + EXPECT_EQ(GetIndexRange(/*spatial_index_dim=*/1, /*block_shape_dim=*/5, + /*input_dim=*/7, /*output_dim=*/1), + std::make_pair(0, 0)); + EXPECT_EQ(GetIndexRange(/*spatial_index_dim=*/-29, /*block_shape_dim=*/5, + /*input_dim=*/7, /*output_dim=*/1), + std::make_pair(6, 6)); +} + +} // namespace +} // namespace tflite diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h index 5f60b2d6a0..fa91db7fe1 100644 --- a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h +++ b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h @@ -5212,6 +5212,7 @@ inline void SpaceToBatchND(const T* input_data, const Dims<4>& input_dims, const int32* paddings_data, const Dims<4>& paddings_dims, T* output_data, const Dims<4>& output_dims) { + // Unoptimized - Straight copy from reference ops. gemmlowp::ScopedProfilingLabel label("SpaceToBatchND"); const int output_batch_size = ArraySize(output_dims, 3); @@ -5253,29 +5254,76 @@ inline void SpaceToBatchND(const T* input_data, const Dims<4>& input_dims, } } +// Helper methods for BatchToSpaceND. +// `spatial_index_dim` specifies post-crop offset index in this spatial +// dimension, i.e. spatial offset introduced by flattening batch to spatial +// dimension minus the crop size at beginning. `block_shape_dim` is the block +// size in current dimension. `input_dim` and `output_dim` are input and output +// size of BatchToSpaceND operation in current dimension. +// Output start index is inclusive and end index is exclusive. +inline void GetIndexRange(int spatial_index_dim, int block_shape_dim, + int input_dim, int output_dim, int* start_index, + int* end_index) { + // (*start_index) * block_shape_dim is effectively rounded up to the next + // multiple of block_shape_dim by the integer division. + *start_index = + std::max(0, (-spatial_index_dim + block_shape_dim - 1) / block_shape_dim); + // Similarly, (*end_index) * block_shape_dim is rounded up too (note that + // end_index is exclusive). + *end_index = std::min( + input_dim, + (output_dim - spatial_index_dim + block_shape_dim - 1) / block_shape_dim); +} + template inline void BatchToSpaceND(const T* input_data, const Dims<4>& input_dims, const int32* block_shape_data, - const Dims<4>& block_shape_dims, T* output_data, - const Dims<4>& output_dims) { + const Dims<4>& block_shape_dims, + const int32* crops_data, const Dims<4>& crops_dims, + T* output_data, const Dims<4>& output_dims) { gemmlowp::ScopedProfilingLabel label("BatchToSpaceND"); const int output_batch_size = ArraySize(output_dims, 3); + const int output_height = ArraySize(output_dims, 2); + const int output_width = ArraySize(output_dims, 1); const int input_batch_size = ArraySize(input_dims, 3); const int input_height = ArraySize(input_dims, 2); const int input_width = ArraySize(input_dims, 1); const int depth = ArraySize(input_dims, 0); const int block_shape_width = block_shape_data[1]; const int block_shape_height = block_shape_data[0]; + const int crops_top = crops_data[0]; + const int crops_left = crops_data[2]; for (int in_batch = 0; in_batch < input_batch_size; ++in_batch) { - for (int in_h = 0; in_h < input_height; ++in_h) { - for (int in_w = 0; in_w < input_width; ++in_w) { - int out_batch = in_batch % output_batch_size; - int out_w = in_w * block_shape_width + - (in_batch / output_batch_size) % block_shape_width; - int out_h = in_h * block_shape_height + - (in_batch / output_batch_size) / block_shape_width; + const int out_batch = in_batch % output_batch_size; + const int spatial_offset = in_batch / output_batch_size; + + int in_h_start = 0; + int in_h_end = 0; + // GetIndexRange ensures start and end indices are in [0, output_height). + GetIndexRange(spatial_offset / block_shape_width - crops_top, + block_shape_height, input_height, output_height, &in_h_start, + &in_h_end); + + for (int in_h = in_h_start; in_h < in_h_end; ++in_h) { + const int out_h = in_h * block_shape_height + + spatial_offset / block_shape_width - crops_top; + TFLITE_DCHECK_GE(out_h, 0); + TFLITE_DCHECK_LT(out_h, output_height); + + int in_w_start = 0; + int in_w_end = 0; + // GetIndexRange ensures start and end indices are in [0, output_width). + GetIndexRange(spatial_offset % block_shape_width - crops_left, + block_shape_width, input_width, output_width, &in_w_start, + &in_w_end); + + for (int in_w = in_w_start; in_w < in_w_end; ++in_w) { + const int out_w = in_w * block_shape_width + + spatial_offset % block_shape_width - crops_left; + TFLITE_DCHECK_GE(out_w, 0); + TFLITE_DCHECK_LT(out_w, output_width); T* out = output_data + Offset(output_dims, 0, out_w, out_h, out_batch); const T* in = input_data + Offset(input_dims, 0, in_w, in_h, in_batch); memcpy(out, in, depth * sizeof(T)); diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h index 0912f5928c..c6019390f2 100644 --- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h +++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h @@ -2873,24 +2873,37 @@ inline void SpaceToBatchND(const T* input_data, const Dims<4>& input_dims, template inline void BatchToSpaceND(const T* input_data, const Dims<4>& input_dims, const int32* block_shape_data, - const Dims<4>& block_shape_dims, T* output_data, - const Dims<4>& output_dims) { + const Dims<4>& block_shape_dims, + const int32* crops_data, const Dims<4>& crops_dims, + T* output_data, const Dims<4>& output_dims) { const int output_batch_size = ArraySize(output_dims, 3); + const int output_height = ArraySize(output_dims, 2); + const int output_width = ArraySize(output_dims, 1); const int input_batch_size = ArraySize(input_dims, 3); const int input_height = ArraySize(input_dims, 2); const int input_width = ArraySize(input_dims, 1); const int depth = ArraySize(input_dims, 0); const int block_shape_width = block_shape_data[1]; const int block_shape_height = block_shape_data[0]; + const int crops_top = crops_data[0]; + const int crops_left = crops_data[2]; for (int in_batch = 0; in_batch < input_batch_size; ++in_batch) { + const int out_batch = in_batch % output_batch_size; + const int spatial_offset = in_batch / output_batch_size; for (int in_h = 0; in_h < input_height; ++in_h) { + const int out_h = in_h * block_shape_height + + spatial_offset / block_shape_width - crops_top; + if (out_h < 0 || out_h >= output_height) { + continue; + } for (int in_w = 0; in_w < input_width; ++in_w) { - int out_batch = in_batch % output_batch_size; - int out_w = in_w * block_shape_width + - (in_batch / output_batch_size) % block_shape_width; - int out_h = in_h * block_shape_height + - (in_batch / output_batch_size) / block_shape_width; + const int out_w = in_w * block_shape_width + + spatial_offset % block_shape_width - crops_left; + + if (out_w < 0 || out_w >= output_width) { + continue; + } T* out = output_data + Offset(output_dims, 0, out_w, out_h, out_batch); const T* in = input_data + Offset(input_dims, 0, in_w, in_h, in_batch); memcpy(out, in, depth * sizeof(T)); diff --git a/tensorflow/contrib/lite/testing/generate_examples.py b/tensorflow/contrib/lite/testing/generate_examples.py index 0e6aceeb86..4b4ccc0c37 100644 --- a/tensorflow/contrib/lite/testing/generate_examples.py +++ b/tensorflow/contrib/lite/testing/generate_examples.py @@ -93,9 +93,6 @@ KNOWN_BUGS = { r"softmax.*input_shape=\[1,3,4,3\]": "67749831", # SpaceToDepth only supports float32. r"space_to_depth.*(float16|int32|uint8|int64)": "68018134", - # BatchToSpaceND doesn't support cropping. This catches test cases with - # const tensors as crops. - r"batch_to_space_nd.*crops=\[\[1,1\],\[1,1\]\]": "70594634", # BatchToSpaceND only supports 4D tensors. r"batch_to_space_nd.*input_shape=\[8,2,2,2,1,1\]": "70594733", # Div will use floordiv. @@ -1595,7 +1592,7 @@ def make_batch_to_space_nd_tests(zip_path): test_parameters = [ { "dtype": [tf.float32, tf.int64, tf.int32], - "input_shape": [[12, 2, 2, 1]], + "input_shape": [[12, 3, 3, 1]], "block_shape": [[1, 4], [2, 2], [3, 4]], "crops": [[[0, 0], [0, 0]], [[1, 1], [1, 1]]], "constant_block_shape": [True, False], diff --git a/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc index a648b770f8..9191e69662 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc +++ b/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc @@ -1060,17 +1060,15 @@ void ProcessBatchToSpaceNDOperator(Model* model, BatchToSpaceNDOperator* op) { } QCHECK(crops_array.data_type == ArrayDataType::kInt32); const auto& crops_data = crops_array.GetBuffer().data; - // We don't support crops now. - QCHECK_EQ(crops_data[0], 0); - QCHECK_EQ(crops_data[1], 0); - QCHECK_EQ(crops_data[2], 0); - QCHECK_EQ(crops_data[3], 0); - + const int crops_top = crops_data[0]; + const int crops_bottom = crops_data[1]; + const int crops_left = crops_data[2]; + const int crops_right = crops_data[3]; + const int output_height = + input_height * block_height - crops_top - crops_bottom; + const int output_width = input_width * block_width - crops_left - crops_right; QCHECK_EQ(input_shape.dims(0) % (block_height * block_width), 0); - int output_height = input_height * block_height; - int output_width = input_width * block_width; - model->GetArray(op->outputs[0]) .copy_shape(Shape({input_shape.dims(0) / (block_height * block_width), output_height, output_width, input_shape.dims(3)})); diff --git a/tensorflow/contrib/lite/toco/model.h b/tensorflow/contrib/lite/toco/model.h index 616d53ae3e..716a579d22 100644 --- a/tensorflow/contrib/lite/toco/model.h +++ b/tensorflow/contrib/lite/toco/model.h @@ -1420,8 +1420,7 @@ struct SpaceToBatchNDOperator : Operator { }; // BatchToSpaceND operator. Rearranges data from batch into blocks of -// spatial data. Currently, only 2-d blocks are supported. Cropping is not -// supported, either, and the crops array should be all zero. +// spatial data. Currently, only 2-d blocks are supported. // // Inputs: // inputs[0]: required: the input array -- GitLab From 64c3e9f9636c73a5aec11572475f2cd26dbbc87b Mon Sep 17 00:00:00 2001 From: bhavani-subramanian Date: Wed, 11 Apr 2018 15:10:38 -0700 Subject: [PATCH 187/791] [INTEL MKL] Skip special nodes inserted by TF and MKL (#18077) * Skip special nodes inserted by TF. This fixes TFDO-178 JIIRA issue. * Added a comment about skipping nodes with an /_ in them. * Stripped trailing whitespace. * Wrapped code such that it is executed only when INTEL_MKL is defined. --- tensorflow/core/grappler/clusters/single_machine_test.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tensorflow/core/grappler/clusters/single_machine_test.cc b/tensorflow/core/grappler/clusters/single_machine_test.cc index c6352c1448..352f08fede 100644 --- a/tensorflow/core/grappler/clusters/single_machine_test.cc +++ b/tensorflow/core/grappler/clusters/single_machine_test.cc @@ -196,10 +196,19 @@ TEST_F(SingleMachineTest, GraphOptimizations) { TF_CHECK_OK(cluster_->Run(item.graph, item.feed, item.fetch, &metadata)); std::set cost_nodes; for (const auto& node : metadata.cost_graph().node()) { +#ifdef INTEL_MKL + // Skip the special nodes inserted by TF (and MKL): these are either + // prefixed with an underscore or contain "/_". + if (node.name()[0] == '_' || node.name().find("/_") != string::npos) { + continue; + } + cost_nodes.insert(node.name()); +#else // Skip nodes added by TF internally. if (node.name()[0] != '_') { cost_nodes.insert(node.name()); } +#endif } const std::set expected_cost_nodes = { "zero", "one", "add", "square", -- GitLab From d2690cf5893cb117ab52f0169fe730736dc22ab7 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 15:09:07 -0700 Subject: [PATCH 188/791] Extend support to remove transpose/reverse on dimensions of size 1. PiperOrigin-RevId: 192516190 --- tensorflow/core/grappler/op_types.cc | 8 +- tensorflow/core/grappler/op_types.h | 1 + .../grappler/optimizers/constant_folding.cc | 95 +++++++++++++++++-- .../optimizers/constant_folding_test.cc | 80 +++++++++++++++- 4 files changed, 168 insertions(+), 16 deletions(-) diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc index 9c45aed62f..cfe1329dbf 100644 --- a/tensorflow/core/grappler/op_types.cc +++ b/tensorflow/core/grappler/op_types.cc @@ -249,6 +249,10 @@ bool IsPrint(const NodeDef& node) { return node.op() == "Print"; } bool IsProd(const NodeDef& node) { return node.op() == "Prod"; } +bool IsRandomShuffle(const NodeDef& node) { + return node.op() == "RandomShuffle"; +} + bool IsReal(const NodeDef& node) { return node.op() == "Real"; } bool IsRealDiv(const NodeDef& node) { return node.op() == "RealDiv"; } @@ -298,9 +302,7 @@ bool IsShape(const NodeDef& node) { return node.op() == "Shape"; } bool IsShapeN(const NodeDef& node) { return node.op() == "ShapeN"; } -bool IsShuffle(const NodeDef& node) { - return node.op() == "Shuffle" || node.op() == "RandomShuffle"; -} +bool IsShuffle(const NodeDef& node) { return node.op() == "Shuffle"; } bool IsSigmoidGrad(const NodeDef& node) { return node.op() == "SigmoidGrad"; } diff --git a/tensorflow/core/grappler/op_types.h b/tensorflow/core/grappler/op_types.h index 79fd05e187..0573b02604 100644 --- a/tensorflow/core/grappler/op_types.h +++ b/tensorflow/core/grappler/op_types.h @@ -98,6 +98,7 @@ bool IsPolygamma(const NodeDef& node); bool IsPrint(const NodeDef& node); bool IsProd(const NodeDef& node); bool IsPow(const NodeDef& node); +bool IsRandomShuffle(const NodeDef& node); bool IsReal(const NodeDef& node); bool IsRealDiv(const NodeDef& node); bool IsRelu6Grad(const NodeDef& node); diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc index b2a1ce6ab6..17d8b7421c 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding.cc +++ b/tensorflow/core/grappler/optimizers/constant_folding.cc @@ -1574,24 +1574,99 @@ Status ConstantFolding::SimplifyGraph(GraphDef* optimized_graph, continue; } - // Remove Shuffle or Reverse op over scalar values. - if (use_shape_info && - !properties->GetInputProperties(node->name()).empty() && - (IsShuffle(*node) || IsReverse(*node) || IsTranspose(*node))) { + // Remove Shuffle or Transpose op over dimensions of size 1. + if (use_shape_info && (IsShuffle(*node) || IsTranspose(*node)) && + !properties->GetInputProperties(node->name()).empty()) { const auto& shape = properties->GetInputProperties(node->name())[0].shape(); - // The node is replaceable iff - // unknown_rank == false && (dim_size == 0 || all dims have size 1) - bool replaceable = !shape.unknown_rank(); - for (int j = 0; replaceable && j < shape.dim_size(); ++j) { - replaceable &= shape.dim(j).size() == 1; + if (shape.unknown_rank()) { + // Not optimizable. + continue; } - if (replaceable) { + const auto& p = properties->GetInputProperties(node->name())[1]; + if (TensorShape::IsValid(p.shape()) && p.has_value()) { + Tensor perm(p.dtype(), p.shape()); + if (!perm.FromProto(p.value())) { + return errors::InvalidArgument("Cannot parse tensor from proto: ", + p.value().DebugString()); + } + std::vector permutation; + for (int j = 0; j < perm.NumElements(); ++j) { + if (perm.dtype() == DT_INT64) { + permutation.push_back(perm.vec()(j)); + } else { + permutation.push_back(perm.vec()(j)); + } + } + if (permutation.size() != shape.dim_size()) { + // Number of elements in perm should be same as dim_size. Skip if not. + continue; + } + // The node is replaceable iff + // dim_size == 0 || all dims have size 1 || + // all dims with > 1 size are not permuted. + bool replaceable = true; + for (int j = 0; replaceable && j < shape.dim_size(); ++j) { + replaceable &= shape.dim(j).size() == 1 || j == permutation[j]; + } + if (replaceable) { + ReplaceOperationWithIdentity(0, node, optimized_graph); + continue; + } + } + } + + // Remove RandomShuffle op if it is scalar or first dimension is of size 1. + if (use_shape_info && IsRandomShuffle(*node) && + !properties->GetInputProperties(node->name()).empty()) { + const auto& shape = + properties->GetInputProperties(node->name())[0].shape(); + // The node is replaceable iff + // unknown_rank == false && (dim_size == 0 || first dim is of size 1) + if (!shape.unknown_rank() && + (shape.dim_size() == 0 || shape.dim(0).size() == 1)) { ReplaceOperationWithIdentity(0, node, optimized_graph); continue; } } + // Remove Reverse op over dimensions with size 1. + if (use_shape_info && IsReverse(*node) && + !properties->GetInputProperties(node->name()).empty()) { + const auto& shape = + properties->GetInputProperties(node->name())[0].shape(); + const auto& a = properties->GetInputProperties(node->name())[1]; + if (TensorShape::IsValid(a.shape()) && a.has_value()) { + Tensor axis(a.dtype(), a.shape()); + if (!axis.FromProto(a.value())) { + return errors::InvalidArgument("Cannot parse tensor from proto: ", + a.value().DebugString()); + } + std::set target_axes; + for (int j = 0; j < axis.NumElements(); ++j) { + if (axis.dtype() == DT_INT64) { + target_axes.insert(axis.vec()(j)); + } else { + target_axes.insert(axis.vec()(j)); + } + } + + // The node is replaceable iff + // unknown_rank == false && + // (dim_size == 0 || all dims have size 1 || + // all dims with > 1 size are not in target_axes) + bool replaceable = !shape.unknown_rank(); + for (int j = 0; replaceable && j < shape.dim_size(); ++j) { + replaceable &= shape.dim(j).size() == 1 || + target_axes.find(j) == target_axes.end(); + } + if (replaceable) { + ReplaceOperationWithIdentity(0, node, optimized_graph); + continue; + } + } + } + if (use_shape_info && IsSlice(*node) && properties->GetInputProperties(node->name()).size() == 3) { const auto& input = properties->GetInputProperties(node->name())[0]; diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc index 31abe43846..7453fb6731 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc +++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc @@ -1389,8 +1389,6 @@ TEST_F(ConstantFoldingTest, SplitVRemoval) { ops::SplitV s1(scope.WithOpName("s1"), in1, size_splits1, split_dim, 1); ops::SplitV s2(scope.WithOpName("s2"), in2, size_splits2, split_dim, 2); - LOG(INFO) << s1.output.size(); - LOG(INFO) << s2.output.size(); ops::Add out(scope.WithOpName("out"), s1[0], s2[0]); GrapplerItem item; @@ -1418,7 +1416,45 @@ TEST_F(ConstantFoldingTest, SplitVRemoval) { CompareGraphs(want, got); } -TEST_F(ConstantFoldingTest, ShuffleReverseOnScalarRemoval) { +TEST_F(ConstantFoldingTest, TransposeOnSize1DimsRemoval) { + tensorflow::Scope scope = tensorflow::Scope::NewRootScope(); + + Output in1 = ops::Variable(scope.WithOpName("in1"), TensorShape({1, 2, 4, 1}), + DT_FLOAT); + Output p1 = ops::Const(scope.WithOpName("p1"), {3, 2, 1, 0}, {4}); + Output in2 = ops::Variable(scope.WithOpName("in2"), TensorShape({1, 4, 2, 1}), + DT_FLOAT); + Output p2 = ops::Const(scope.WithOpName("p2"), {3, 1, 2, 0}, {4}); + ops::Transpose t1(scope.WithOpName("t1"), in1, p1); + ops::Transpose t2(scope.WithOpName("t2").WithControlDependencies({in1}), in2, + p2); + + ops::Add out1(scope.WithOpName("out1"), t1, t2); + + GrapplerItem item; + item.fetch = {"out1"}; + TF_CHECK_OK(scope.ToGraphDef(&item.graph)); + + ConstantFolding optimizer(nullptr /* cpu_device */); + GraphDef got; + Status status = optimizer.Optimize(nullptr, item, &got); + TF_EXPECT_OK(status); + + GraphDef want; + AddNode("in1", "VariableV2", {}, {}, &want); + AddNode("in2", "VariableV2", {}, {}, &want); + AddNode("p1", "Const", {}, {}, &want); + AddNode("p2", "Const", {}, {}, &want); + AddNode("t1", "Transpose", {"in1", "p1"}, {}, &want); + AddNode("t2", "Identity", + {"in2", AsControlDependency("in1"), AsControlDependency("p2")}, {}, + &want); + AddNode("out1", "Add", {"t1", "t2"}, {}, &want); + + CompareGraphs(want, got); +} + +TEST_F(ConstantFoldingTest, RandomShuffleOnScalarRemoval) { tensorflow::Scope scope = tensorflow::Scope::NewRootScope(); Output in1 = @@ -1452,6 +1488,44 @@ TEST_F(ConstantFoldingTest, ShuffleReverseOnScalarRemoval) { CompareGraphs(want, got); } +TEST_F(ConstantFoldingTest, ReverseOnSize1DimsRemoval) { + tensorflow::Scope scope = tensorflow::Scope::NewRootScope(); + + Output in1 = ops::Variable(scope.WithOpName("in1"), TensorShape({1, 2, 4, 1}), + DT_FLOAT); + Output a1 = ops::Const(scope.WithOpName("a1"), {3, 2, 1, 0}, {4}); + Output in2 = ops::Variable(scope.WithOpName("in2"), TensorShape({1, 2, 4, 1}), + DT_FLOAT); + Output a2 = ops::Const(scope.WithOpName("a2"), {0, 3}, {2}); + ops::Reverse r1(scope.WithOpName("r1"), in1, a1); + ops::Reverse r2(scope.WithOpName("r2").WithControlDependencies({in1}), in2, + a2); + + ops::Add out1(scope.WithOpName("out1"), r1, r2); + + GrapplerItem item; + item.fetch = {"out1"}; + TF_CHECK_OK(scope.ToGraphDef(&item.graph)); + + ConstantFolding optimizer(nullptr /* cpu_device */); + GraphDef got; + Status status = optimizer.Optimize(nullptr, item, &got); + TF_EXPECT_OK(status); + + GraphDef want; + AddNode("in1", "VariableV2", {}, {}, &want); + AddNode("in2", "VariableV2", {}, {}, &want); + AddNode("a1", "Const", {}, {}, &want); + AddNode("a2", "Const", {}, {}, &want); + AddNode("r1", "ReverseV2", {"in1", "a1"}, {}, &want); + AddNode("r2", "Identity", + {"in2", AsControlDependency("in1"), AsControlDependency("a2")}, {}, + &want); + AddNode("out1", "Add", {"r1", "r2"}, {}, &want); + + CompareGraphs(want, got); +} + TEST_F(ConstantFoldingTest, SliceWithSameDimensionRemoval) { { // size = {3, 5} tensorflow::Scope scope = tensorflow::Scope::NewRootScope(); -- GitLab From 079d63d59b75bdfd25f7371efda25ec5f6739b78 Mon Sep 17 00:00:00 2001 From: Sourabh Bajaj Date: Wed, 11 Apr 2018 15:20:11 -0700 Subject: [PATCH 189/791] GCS Filesystem should not cache checkpoint file as we need to read the updated checkpoints from the contents. PiperOrigin-RevId: 192517819 --- .../core/platform/cloud/gcs_file_system.cc | 8 ++++ .../platform/cloud/gcs_file_system_test.cc | 48 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/tensorflow/core/platform/cloud/gcs_file_system.cc b/tensorflow/core/platform/cloud/gcs_file_system.cc index 3c0dc13d75..6ed1d5dad2 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system.cc +++ b/tensorflow/core/platform/cloud/gcs_file_system.cc @@ -301,6 +301,14 @@ class GcsRandomAccessFile : public RandomAccessFile { TF_RETURN_IF_ERROR(file_block_cache_->Read(filename_, offset, n, scratch, &bytes_transferred)); *result = StringPiece(scratch, bytes_transferred); + string checkpoint_ending = "/checkpoint"; + // Check if the file is the checkpoint file as we should not be caching + // that. As it's contents are updated and used for iterating checkpoints. + if (std::equal(checkpoint_ending.rbegin(), checkpoint_ending.rend(), + filename_.rbegin())) { + // Remove the checkpoint file from the cache + file_block_cache_->RemoveFile(filename_); + } if (bytes_transferred < n) { // This is not an error per se. The RandomAccessFile interface expects // that Read returns OutOfRange if fewer bytes were read than requested. diff --git a/tensorflow/core/platform/cloud/gcs_file_system_test.cc b/tensorflow/core/platform/cloud/gcs_file_system_test.cc index 2fbde9b6a7..e9eca04fef 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system_test.cc +++ b/tensorflow/core/platform/cloud/gcs_file_system_test.cc @@ -198,6 +198,54 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache) { EXPECT_EQ("0123", result); } +TEST(GcsFileSystemTest, NewRandomAccessFile_CheckpointFile_WithBlockCache) { + // Our underlying file in this test changes as new data comes in + std::vector requests( + {new FakeHttpRequest( + "Uri: https://storage.googleapis.com/bucket/checkpoint\n" + "Auth Token: fake_token\n" + "Range: 0-8\n" + "Timeouts: 5 1 20\n", + "012345678"), + new FakeHttpRequest( + "Uri: https://storage.googleapis.com/bucket/checkpoint\n" + "Auth Token: fake_token\n" + "Range: 0-8\n" + "Timeouts: 5 1 20\n", + "abcdefghi")}); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + 9 /* block size */, 18 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + kTestTimeoutConfig, nullptr /* gcs additional header */); + + char scratch[100]; + StringPiece result; + { + // We are instantiating this in an enclosed scope to make sure after the + // unique ptr goes out of scope, we can still access result. + std::unique_ptr file; + TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/checkpoint", &file)); + + // Read the first chunk. The cache will be populated with the first block of + // 9 bytes. + scratch[5] = 'x'; + TF_EXPECT_OK(file->Read(0, 4, &result, scratch)); + EXPECT_EQ("0123", result); + EXPECT_EQ(scratch[5], 'x'); // Make sure we only copied 4 bytes. + + // The second chunk should not be in cache so we make a new request + // As the checkpoint file should not be cached + TF_EXPECT_OK(file->Read(0, 4, &result, scratch)); + EXPECT_EQ("abcd", result); + EXPECT_EQ(scratch[5], 'x'); // Make sure we only copied 4 bytes. + } +} + TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_Flush) { // Our underlying file in this test is a 15 byte file with contents // "0123456789abcde". -- GitLab From 4b08b66ab504e5356f1bf2ecf2f0c9e61f1157e7 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Wed, 11 Apr 2018 15:23:17 -0700 Subject: [PATCH 190/791] Fixes issue where name scope collisions could lead to an invalid variable in the metagraph. PiperOrigin-RevId: 192518307 --- .../resource_variable_ops_test.py | 47 +++++++++++++++---- .../python/ops/resource_variable_ops.py | 14 ++++-- 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py index 6d33086936..984192258c 100644 --- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py +++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py @@ -36,6 +36,9 @@ from tensorflow.python.ops import state_ops from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables from tensorflow.python.platform import test +from tensorflow.python.training import momentum +from tensorflow.python.training import saver +from tensorflow.python.training import training_util from tensorflow.python.util import compat @@ -228,16 +231,40 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): @test_util.run_in_graph_and_eager_modes() def testScatterMin(self): - handle = resource_variable_ops.var_handle_op( - dtype=dtypes.int32, shape=[1, 1]) - self.evaluate( - resource_variable_ops.assign_variable_op( - handle, constant_op.constant([[6]], dtype=dtypes.int32))) - self.evaluate( - resource_variable_ops.resource_scatter_min( - handle, [0], constant_op.constant([[3]], dtype=dtypes.int32))) - read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertEqual(self.evaluate(read), [[3]]) + with ops.device("cpu:0"): + handle = resource_variable_ops.var_handle_op( + dtype=dtypes.int32, shape=[1, 1]) + self.evaluate( + resource_variable_ops.assign_variable_op(handle, + constant_op.constant( + [[6]], + dtype=dtypes.int32))) + self.evaluate( + resource_variable_ops.resource_scatter_min(handle, [0], + constant_op.constant( + [[3]], + dtype=dtypes.int32))) + read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) + self.assertEqual(self.evaluate(read), [[3]]) + + def testMetagraph(self): + with ops.Graph().as_default(): + with variable_scope.variable_scope("foo", use_resource=True): + a = variable_scope.get_variable("a", initializer=10.0) + + momentum.MomentumOptimizer( + learning_rate=0.001, momentum=0.1).minimize( + a, + colocate_gradients_with_ops=True, + global_step=training_util.get_or_create_global_step()) + + graph = ops.get_default_graph() + meta_graph_def = saver.export_meta_graph(graph=graph) + + with ops.Graph().as_default(): + saver.import_meta_graph(meta_graph_def, import_scope="") + meta_graph_two = saver.export_meta_graph(graph=graph) + self.assertEqual(meta_graph_def, meta_graph_two) @test_util.run_in_graph_and_eager_modes() def testScatterMax(self): diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py index 508ba9bfee..c51d1e467d 100644 --- a/tensorflow/python/ops/resource_variable_ops.py +++ b/tensorflow/python/ops/resource_variable_ops.py @@ -525,8 +525,15 @@ class ResourceVariable(variables.Variable): self._cached_value = g.as_graph_element( ops.prepend_name_scope( variable_def.snapshot_name, import_scope=import_scope)) + self._graph_element = g.as_graph_element( + ops.prepend_name_scope(variable_def.snapshot_name, + import_scope=import_scope)) else: self._cached_value = None + # Legacy case for protos without the snapshot name; assume it's the + # following. + self._graph_element = g.get_tensor_by_name( + self._handle.op.name + "/Read/ReadVariableOp:0") if variable_def.HasField("save_slice_info_def"): self._save_slice_info = variables.Variable.SaveSliceInfo( save_slice_info_def=variable_def.save_slice_info_def, @@ -535,8 +542,6 @@ class ResourceVariable(variables.Variable): self._save_slice_info = None self._caching_device = None self._dtype = dtypes.as_dtype(self._handle.op.get_attr("dtype")) - self._graph_element = g.get_tensor_by_name( - self._handle.op.name + "/Read/ReadVariableOp:0") self._constraint = None self._cached_shape_as_list = None @@ -745,6 +750,10 @@ class ResourceVariable(variables.Variable): if self._cached_value is not None: var_def.snapshot_name = ops.strip_name_scope(self._cached_value.name, export_scope) + else: + # Store the graph_element here + var_def.snapshot_name = ops.strip_name_scope(self._graph_element.name, + export_scope) var_def.is_resource = True if self._save_slice_info: var_def.save_slice_info_def.MergeFrom( @@ -910,7 +919,6 @@ class ResourceVariable(variables.Variable): def _dense_var_to_tensor(self, dtype=None, name=None, as_ref=False): del name if dtype is not None and dtype != self.dtype: - print("trying to switch the dtype to ", dtype, " from ", self.dtype) return NotImplemented if as_ref: return self.read_value().op.inputs[0] -- GitLab From f029631d65a2209aa3f089cbb980d61ee9d0e7f5 Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Wed, 11 Apr 2018 15:33:06 -0700 Subject: [PATCH 191/791] Increase size of //tensorflow/python/kernel_tests:sets_test to "medium". PiperOrigin-RevId: 192519639 --- tensorflow/python/kernel_tests/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index 1827a26902..5738e79b27 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -2822,7 +2822,7 @@ sycl_py_test( tf_py_test( name = "sets_test", - size = "small", + size = "medium", srcs = ["sets_test.py"], additional_deps = [ "//third_party/py/numpy", -- GitLab From 9ce7791be6980932c249832dc23d464c1b736cc4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 15:37:49 -0700 Subject: [PATCH 192/791] Revealing the range of node ids in the latest layer via resource' state PiperOrigin-RevId: 192520351 --- ...tedTreesCalculateBestGainsPerFeature.pbtxt | 4 +- ...pi_def_BoostedTreesGetEnsembleStates.pbtxt | 12 +++++- .../kernels/boosted_trees/boosted_trees.proto | 4 ++ .../kernels/boosted_trees/resource_ops.cc | 12 ++++++ .../core/kernels/boosted_trees/resources.h | 20 ++++++++++ .../core/kernels/boosted_trees/stats_ops.cc | 6 +-- .../kernels/boosted_trees/training_ops.cc | 8 ++++ tensorflow/core/ops/boosted_trees_ops.cc | 2 + .../core/ops/compat/ops_history.v1.pbtxt | 4 ++ .../python/estimator/canned/boosted_trees.py | 9 ++--- .../estimator/canned/boosted_trees_test.py | 12 ++++++ .../boosted_trees/resource_ops_test.py | 31 +++++++++----- .../boosted_trees/stats_ops_test.py | 8 ++-- .../boosted_trees/training_ops_test.py | 40 +++++++++++++++++-- tensorflow/python/ops/boosted_trees_ops.py | 15 ++++--- 15 files changed, 150 insertions(+), 37 deletions(-) diff --git a/tensorflow/core/api_def/base_api/api_def_BoostedTreesCalculateBestGainsPerFeature.pbtxt b/tensorflow/core/api_def/base_api/api_def_BoostedTreesCalculateBestGainsPerFeature.pbtxt index b1921e3507..62876a293c 100644 --- a/tensorflow/core/api_def/base_api/api_def_BoostedTreesCalculateBestGainsPerFeature.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_BoostedTreesCalculateBestGainsPerFeature.pbtxt @@ -4,7 +4,7 @@ op { in_arg { name: "node_id_range" description: <allocate_output(0, TensorShape(), &output_stamp_token_t)); @@ -110,11 +111,22 @@ class BoostedTreesGetEnsembleStatesOp : public OpKernel { OP_REQUIRES_OK(context, context->allocate_output(3, TensorShape(), &output_num_attempted_layers_t)); + OP_REQUIRES_OK(context, context->allocate_output( + 4, {2}, &output_last_layer_nodes_range_t)); output_stamp_token_t->scalar()() = tree_ensemble_resource->stamp(); output_num_trees_t->scalar()() = num_trees; output_num_finalized_trees_t->scalar()() = num_finalized_trees; output_num_attempted_layers_t->scalar()() = num_attempted_layers; + + int32 range_start; + int32 range_end; + tree_ensemble_resource->GetLastLayerNodesRange(&range_start, &range_end); + + output_last_layer_nodes_range_t->vec()(0) = range_start; + // For a completely empty ensemble, this will be 0. To make it a valid range + // we add this max cond. + output_last_layer_nodes_range_t->vec()(1) = std::max(1, range_end); } }; diff --git a/tensorflow/core/kernels/boosted_trees/resources.h b/tensorflow/core/kernels/boosted_trees/resources.h index c82588b950..561ca3a18a 100644 --- a/tensorflow/core/kernels/boosted_trees/resources.h +++ b/tensorflow/core/kernels/boosted_trees/resources.h @@ -93,6 +93,26 @@ class BoostedTreesEnsembleResource : public StampedResource { new_num_layers); } + void UpdateLastLayerNodesRange(const int32 node_range_start, + int32 node_range_end) const { + tree_ensemble_->mutable_growing_metadata()->set_last_layer_node_start( + node_range_start); + tree_ensemble_->mutable_growing_metadata()->set_last_layer_node_end( + node_range_end); + } + + void GetLastLayerNodesRange(int32* node_range_start, + int32* node_range_end) const { + *node_range_start = + tree_ensemble_->growing_metadata().last_layer_node_start(); + *node_range_end = tree_ensemble_->growing_metadata().last_layer_node_end(); + } + + int64 GetNumNodes(const int32 tree_id) { + DCHECK_LT(tree_id, tree_ensemble_->trees_size()); + return tree_ensemble_->trees(tree_id).nodes_size(); + } + void UpdateGrowingMetadata() const; int32 GetNumLayersAttempted() { diff --git a/tensorflow/core/kernels/boosted_trees/stats_ops.cc b/tensorflow/core/kernels/boosted_trees/stats_ops.cc index 33fdab6a86..16e65cf284 100644 --- a/tensorflow/core/kernels/boosted_trees/stats_ops.cc +++ b/tensorflow/core/kernels/boosted_trees/stats_ops.cc @@ -42,8 +42,8 @@ class BoostedTreesCalculateBestGainsPerFeatureOp : public OpKernel { const Tensor* node_id_range_t; OP_REQUIRES_OK(context, context->input("node_id_range", &node_id_range_t)); const auto node_id_range = node_id_range_t->vec(); - int32 node_id_first = node_id_range(0); - int32 node_id_last = node_id_range(1); // inclusive. + const int32 node_id_first = node_id_range(0); // inclusive + const int32 node_id_last = node_id_range(1); // exclusive // stats_summary_list OpInputList stats_summary_list; OP_REQUIRES_OK(context, context->input_list("stats_summary_list", @@ -86,7 +86,7 @@ class BoostedTreesCalculateBestGainsPerFeatureOp : public OpKernel { std::vector output_thresholds; std::vector output_left_node_contribs; std::vector output_right_node_contribs; - for (int node_id = node_id_first; node_id <= node_id_last; ++node_id) { + for (int node_id = node_id_first; node_id < node_id_last; ++node_id) { // Calculate gains. cum_grad.clear(); cum_hess.clear(); diff --git a/tensorflow/core/kernels/boosted_trees/training_ops.cc b/tensorflow/core/kernels/boosted_trees/training_ops.cc index b9ded4054a..67cac14c52 100644 --- a/tensorflow/core/kernels/boosted_trees/training_ops.cc +++ b/tensorflow/core/kernels/boosted_trees/training_ops.cc @@ -101,6 +101,7 @@ class BoostedTreesUpdateEnsembleOp : public OpKernel { << current_tree << " of ensemble of " << current_tree + 1 << " trees."; bool split_happened = false; + int32 node_id_start = ensemble_resource->GetNumNodes(current_tree); // Add the splits to the tree. for (auto& split_entry : best_splits) { const int32 node_id = split_entry.first; @@ -139,11 +140,15 @@ class BoostedTreesUpdateEnsembleOp : public OpKernel { right_contrib, &left_node_id, &right_node_id); split_happened = true; } + int32 node_id_end = ensemble_resource->GetNumNodes(current_tree); if (split_happened) { // Update growable tree metadata. ensemble_resource->SetNumLayersGrown(current_tree, new_num_layers); // Finalize the tree if needed. if (ensemble_resource->GetNumLayersGrown(current_tree) >= max_depth_) { + // If the tree is finalized, next growing will start from node 0; + node_id_start = 0; + node_id_end = 1; ensemble_resource->SetIsFinalized(current_tree, true); if (pruning_mode_ == kPostPruning) { ensemble_resource->PostPruneTree(current_tree); @@ -153,6 +158,9 @@ class BoostedTreesUpdateEnsembleOp : public OpKernel { ensemble_resource->AddNewTree(kLayerByLayerTreeWeight); } } + // If we managed to split, update the node range. If we didn't, don't + // update as we will try to split the same nodes with new instances. + ensemble_resource->UpdateLastLayerNodesRange(node_id_start, node_id_end); } } diff --git a/tensorflow/core/ops/boosted_trees_ops.cc b/tensorflow/core/ops/boosted_trees_ops.cc index 297e94655f..8af4903418 100644 --- a/tensorflow/core/ops/boosted_trees_ops.cc +++ b/tensorflow/core/ops/boosted_trees_ops.cc @@ -128,6 +128,7 @@ REGISTER_OP("BoostedTreesGetEnsembleStates") .Output("num_trees: int32") .Output("num_finalized_trees: int32") .Output("num_attempted_layers: int32") + .Output("last_layer_nodes_range: int32") .SetShapeFn([](shape_inference::InferenceContext* c) { shape_inference::ShapeHandle unused_input; TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input)); @@ -135,6 +136,7 @@ REGISTER_OP("BoostedTreesGetEnsembleStates") c->set_output(1, c->Scalar()); c->set_output(2, c->Scalar()); c->set_output(3, c->Scalar()); + c->set_output(4, c->Vector(2)); return Status::OK(); }); diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt index 12df60a2ae..ba442a0582 100644 --- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt +++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt @@ -10981,6 +10981,10 @@ op { name: "num_attempted_layers" type: DT_INT32 } + output_arg { + name: "last_layer_nodes_range" + type: DT_INT32 + } is_stateful: true } op { diff --git a/tensorflow/python/estimator/canned/boosted_trees.py b/tensorflow/python/estimator/canned/boosted_trees.py index c5d5455b1a..58af59dbb1 100644 --- a/tensorflow/python/estimator/canned/boosted_trees.py +++ b/tensorflow/python/estimator/canned/boosted_trees.py @@ -349,8 +349,8 @@ def _bt_model_fn( array_ops.zeros( [batch_size, head.logits_dimension], dtype=dtypes.float32)) with ops.control_dependencies([ensemble_reload]): - (stamp_token, num_trees, num_finalized_trees, - num_attempted_layers) = local_tree_ensemble.get_states() + (stamp_token, num_trees, num_finalized_trees, num_attempted_layers, + last_layer_nodes_range) = local_tree_ensemble.get_states() summary.scalar('ensemble/num_trees', num_trees) summary.scalar('ensemble/num_finalized_trees', num_finalized_trees) summary.scalar('ensemble/num_attempted_layers', num_attempted_layers) @@ -393,10 +393,7 @@ def _bt_model_fn( (node_ids_per_feature, gains_list, thresholds_list, left_node_contribs_list, right_node_contribs_list) = ( boosted_trees_ops.calculate_best_gains_per_feature( - node_id_range=array_ops.stack([ - math_ops.reduce_min(node_ids), - math_ops.reduce_max(node_ids) - ]), + node_id_range=last_layer_nodes_range, stats_summary_list=stats_summary_list, l1=tree_hparams.l1, l2=tree_hparams.l2, diff --git a/tensorflow/python/estimator/canned/boosted_trees_test.py b/tensorflow/python/estimator/canned/boosted_trees_test.py index 625745a3f9..7823ef8410 100644 --- a/tensorflow/python/estimator/canned/boosted_trees_test.py +++ b/tensorflow/python/estimator/canned/boosted_trees_test.py @@ -223,6 +223,8 @@ class ModelFnTests(test_util.TensorFlowTestCase): growing_metadata { num_trees_attempted: 1 num_layers_attempted: 1 + last_layer_node_start: 1 + last_layer_node_end: 3 } """ second_round = """ @@ -307,6 +309,8 @@ class ModelFnTests(test_util.TensorFlowTestCase): growing_metadata { num_trees_attempted: 1 num_layers_attempted: 2 + last_layer_node_start: 0 + last_layer_node_end: 1 } """ third_round = """ @@ -407,6 +411,8 @@ class ModelFnTests(test_util.TensorFlowTestCase): growing_metadata { num_trees_attempted: 2 num_layers_attempted: 3 + last_layer_node_start: 1 + last_layer_node_end: 3 } """ return (first_round, second_round, third_round) @@ -444,6 +450,8 @@ class ModelFnTests(test_util.TensorFlowTestCase): growing_metadata { num_trees_attempted: 1 num_layers_attempted: 1 + last_layer_node_start: 1 + last_layer_node_end: 3 } """ second_round = """ @@ -528,6 +536,8 @@ class ModelFnTests(test_util.TensorFlowTestCase): growing_metadata { num_trees_attempted: 1 num_layers_attempted: 2 + last_layer_node_start: 0 + last_layer_node_end: 1 } """ third_round = """ @@ -628,6 +638,8 @@ class ModelFnTests(test_util.TensorFlowTestCase): growing_metadata { num_trees_attempted: 2 num_layers_attempted: 3 + last_layer_node_start: 1 + last_layer_node_end: 3 } """ return (first_round, second_round, third_round) diff --git a/tensorflow/python/kernel_tests/boosted_trees/resource_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/resource_ops_test.py index a223241e89..d5f0c22d6e 100644 --- a/tensorflow/python/kernel_tests/boosted_trees/resource_ops_test.py +++ b/tensorflow/python/kernel_tests/boosted_trees/resource_ops_test.py @@ -36,16 +36,18 @@ class ResourceOpsTest(test_util.TensorFlowTestCase): resources.initialize_resources(resources.shared_resources()).run() stamp_token = ensemble.get_stamp_token() self.assertEqual(0, stamp_token.eval()) - (_, num_trees, num_finalized_trees, - num_attempted_layers) = ensemble.get_states() + (_, num_trees, num_finalized_trees, num_attempted_layers, + nodes_range) = ensemble.get_states() self.assertEqual(0, num_trees.eval()) self.assertEqual(0, num_finalized_trees.eval()) self.assertEqual(0, num_attempted_layers.eval()) + self.assertAllEqual([0, 1], nodes_range.eval()) def testCreateWithProto(self): with self.test_session(): ensemble_proto = boosted_trees_pb2.TreeEnsemble() - text_format.Merge(""" + text_format.Merge( + """ trees { nodes { bucketized_split { @@ -141,6 +143,8 @@ class ResourceOpsTest(test_util.TensorFlowTestCase): growing_metadata { num_trees_attempted: 2 num_layers_attempted: 6 + last_layer_node_start: 16 + last_layer_node_end: 19 } """, ensemble_proto) ensemble = boosted_trees_ops.TreeEnsemble( @@ -148,28 +152,31 @@ class ResourceOpsTest(test_util.TensorFlowTestCase): stamp_token=7, serialized_proto=ensemble_proto.SerializeToString()) resources.initialize_resources(resources.shared_resources()).run() - (stamp_token, num_trees, num_finalized_trees, - num_attempted_layers) = ensemble.get_states() + (stamp_token, num_trees, num_finalized_trees, num_attempted_layers, + nodes_range) = ensemble.get_states() self.assertEqual(7, stamp_token.eval()) self.assertEqual(2, num_trees.eval()) self.assertEqual(1, num_finalized_trees.eval()) self.assertEqual(6, num_attempted_layers.eval()) + self.assertAllEqual([16, 19], nodes_range.eval()) def testSerializeDeserialize(self): with self.test_session(): # Initialize. ensemble = boosted_trees_ops.TreeEnsemble('ensemble', stamp_token=5) resources.initialize_resources(resources.shared_resources()).run() - (stamp_token, num_trees, num_finalized_trees, - num_attempted_layers) = ensemble.get_states() + (stamp_token, num_trees, num_finalized_trees, num_attempted_layers, + nodes_range) = ensemble.get_states() self.assertEqual(5, stamp_token.eval()) self.assertEqual(0, num_trees.eval()) self.assertEqual(0, num_finalized_trees.eval()) self.assertEqual(0, num_attempted_layers.eval()) + self.assertAllEqual([0, 1], nodes_range.eval()) # Deserialize. ensemble_proto = boosted_trees_pb2.TreeEnsemble() - text_format.Merge(""" + text_format.Merge( + """ trees { nodes { bucketized_split { @@ -201,6 +208,8 @@ class ResourceOpsTest(test_util.TensorFlowTestCase): growing_metadata { num_trees_attempted: 1 num_layers_attempted: 5 + last_layer_node_start: 3 + last_layer_node_end: 7 } """, ensemble_proto) with ops.control_dependencies([ @@ -208,13 +217,15 @@ class ResourceOpsTest(test_util.TensorFlowTestCase): stamp_token=3, serialized_proto=ensemble_proto.SerializeToString()) ]): - (stamp_token, num_trees, num_finalized_trees, - num_attempted_layers) = ensemble.get_states() + (stamp_token, num_trees, num_finalized_trees, num_attempted_layers, + nodes_range) = ensemble.get_states() self.assertEqual(3, stamp_token.eval()) self.assertEqual(1, num_trees.eval()) # This reads from metadata, not really counting the layers. self.assertEqual(5, num_attempted_layers.eval()) self.assertEqual(0, num_finalized_trees.eval()) + self.assertAllEqual([3, 7], nodes_range.eval()) + # Serialize. new_ensemble_proto = boosted_trees_pb2.TreeEnsemble() diff --git a/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py index a54cc43517..4d09cf94d4 100644 --- a/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py +++ b/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py @@ -29,7 +29,7 @@ class StatsOpsTest(test_util.TensorFlowTestCase): """Testing Gain calculation without any regularization.""" with self.test_session() as sess: max_splits = 7 - node_id_range = [1, 2] # node 1 through 2 will be processed. + node_id_range = [1, 3] # node 1 through 2 will be processed. stats_summary_list = [ [ [[0., 0.], [.08, .09], [0., 0.], [0., 0.]], # node 0; ignored @@ -76,7 +76,7 @@ class StatsOpsTest(test_util.TensorFlowTestCase): """Testing Gain calculation with L2.""" with self.test_session() as sess: max_splits = 7 - node_id_range = [1, 2] # node 1 through 2 will be processed. + node_id_range = [1, 3] # node 1 through 2 will be processed. stats_summary_list = [ [ [[0., 0.], [.08, .09], [0., 0.], [0., 0.]], # node 0; ignored @@ -123,7 +123,7 @@ class StatsOpsTest(test_util.TensorFlowTestCase): """Testing Gain calculation with L1.""" with self.test_session() as sess: max_splits = 7 - node_id_range = [1, 2] # node 1 through 2 will be processed. + node_id_range = [1, 3] # node 1 through 2 will be processed. stats_summary_list = [ [ [[0., 0.], [.08, .09], [0., 0.], [0., 0.]], # node 0; ignored @@ -173,7 +173,7 @@ class StatsOpsTest(test_util.TensorFlowTestCase): """Testing Gain calculation with L2.""" with self.test_session() as sess: max_splits = 7 - node_id_range = [1, 2] # node 1 through 2 will be processed. + node_id_range = [1, 3] # node 1 through 2 will be processed. stats_summary_list = [ [ [[0., 0.], [.08, .09], [0., 0.], [0., 0.]], # node 0; ignored diff --git a/tensorflow/python/kernel_tests/boosted_trees/training_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/training_ops_test.py index 4226ff75c2..d6c0047747 100644 --- a/tensorflow/python/kernel_tests/boosted_trees/training_ops_test.py +++ b/tensorflow/python/kernel_tests/boosted_trees/training_ops_test.py @@ -132,6 +132,8 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): growing_metadata { num_trees_attempted: 1 num_layers_attempted: 1 + last_layer_node_start: 0 + last_layer_node_end: 1 } """ self.assertEqual(new_stamp, 1) @@ -314,6 +316,8 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): growing_metadata { num_trees_attempted: 1 num_layers_attempted: 2 + last_layer_node_start: 0 + last_layer_node_end: 1 } """ self.assertEqual(new_stamp, 1) @@ -461,6 +465,8 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): growing_metadata { num_trees_attempted: 2 num_layers_attempted: 2 + last_layer_node_start: 1 + last_layer_node_end: 3 } """ self.assertEqual(new_stamp, 1) @@ -615,6 +621,8 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): growing_metadata { num_trees_attempted: 1 num_layers_attempted: 2 + last_layer_node_start: 3 + last_layer_node_end: 5 } """ self.assertEqual(new_stamp, 1) @@ -624,7 +632,8 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): """Test that the metadata is updated even though we can't split.""" with self.test_session() as session: tree_ensemble_config = boosted_trees_pb2.TreeEnsemble() - text_format.Merge(""" + text_format.Merge( + """ trees { nodes { bucketized_split { @@ -655,6 +664,9 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): growing_metadata { num_trees_attempted: 1 num_layers_attempted: 1 + last_layer_node_start: 1 + last_layer_node_end: 3 + } """, tree_ensemble_config) @@ -685,7 +697,7 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): # Expect no new splits created, but attempted (global) stats updated. Meta # data for this tree should not be updated (we didn't succeed building a - # layer. + # layer. Node ranges don't change. new_stamp, serialized = session.run(tree_ensemble.serialize()) tree_ensemble = boosted_trees_pb2.TreeEnsemble() tree_ensemble.ParseFromString(serialized) @@ -721,6 +733,8 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): growing_metadata { num_trees_attempted: 1 num_layers_attempted: 2 + last_layer_node_start: 1 + last_layer_node_end: 3 } """ self.assertEqual(new_stamp, 1) @@ -730,7 +744,8 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): """Test metadata is updated correctly when no split due to prepruning.""" with self.test_session() as session: tree_ensemble_config = boosted_trees_pb2.TreeEnsemble() - text_format.Merge(""" + text_format.Merge( + """ trees { nodes { bucketized_split { @@ -761,6 +776,8 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): growing_metadata { num_trees_attempted: 1 num_layers_attempted: 1 + last_layer_node_start: 1 + last_layer_node_end: 3 } """, tree_ensemble_config) @@ -851,6 +868,8 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): growing_metadata { num_trees_attempted: 1 num_layers_attempted: 2 + last_layer_node_start: 1 + last_layer_node_end: 3 } """ self.assertEqual(new_stamp, 1) @@ -941,6 +960,8 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): growing_metadata { num_trees_attempted: 1 num_layers_attempted: 1 + last_layer_node_start: 1 + last_layer_node_end: 3 } """ self.assertEqual(new_stamp, 1) @@ -1046,6 +1067,8 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): growing_metadata { num_trees_attempted: 1 num_layers_attempted: 2 + last_layer_node_start: 3 + last_layer_node_end: 7 } """ self.assertEqual(new_stamp, 2) @@ -1179,6 +1202,8 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): growing_metadata { num_trees_attempted: 1 num_layers_attempted: 3 + last_layer_node_start: 0 + last_layer_node_end: 1 } """ self.assertEqual(new_stamp, 3) @@ -1268,6 +1293,8 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): growing_metadata { num_trees_attempted: 1 num_layers_attempted: 1 + last_layer_node_start: 1 + last_layer_node_end: 3 } """ self.assertEqual(new_stamp, 1) @@ -1307,7 +1334,8 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): # Expect the ensemble to be empty as post-pruning will prune # the entire finalized tree. self.assertEqual(new_stamp, 2) - self.assertProtoEquals(""" + self.assertProtoEquals( + """ trees { nodes { leaf { @@ -1359,6 +1387,8 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): growing_metadata { num_trees_attempted: 1 num_layers_attempted: 2 + last_layer_node_start: 0 + last_layer_node_end: 1 } """, res_ensemble) @@ -1455,6 +1485,8 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): growing_metadata { num_trees_attempted: 1 num_layers_attempted: 1 + last_layer_node_start: 0 + last_layer_node_end: 1 } """ self.assertEqual(new_stamp, 1) diff --git a/tensorflow/python/ops/boosted_trees_ops.py b/tensorflow/python/ops/boosted_trees_ops.py index 174d00987f..2a2bcdd9d6 100644 --- a/tensorflow/python/ops/boosted_trees_ops.py +++ b/tensorflow/python/ops/boosted_trees_ops.py @@ -115,7 +115,7 @@ class TreeEnsemble(object): def get_stamp_token(self): """Returns the current stamp token of the resource.""" - stamp_token, _, _, _ = ( + stamp_token, _, _, _, _ = ( gen_boosted_trees_ops.boosted_trees_get_ensemble_states( self.resource_handle)) return stamp_token @@ -124,17 +124,20 @@ class TreeEnsemble(object): """Returns states of the tree ensemble. Returns: - stamp_token, num_trees, num_finalized_trees, num_attempted_layers. + stamp_token, num_trees, num_finalized_trees, num_attempted_layers and + range of the nodes in the latest layer. """ - stamp_token, num_trees, num_finalized_trees, num_attempted_layers = ( - gen_boosted_trees_ops.boosted_trees_get_ensemble_states( - self.resource_handle)) + (stamp_token, num_trees, num_finalized_trees, num_attempted_layers, + nodes_range) = ( + gen_boosted_trees_ops.boosted_trees_get_ensemble_states( + self.resource_handle)) # Use identity to give names. return (array_ops.identity(stamp_token, name='stamp_token'), array_ops.identity(num_trees, name='num_trees'), array_ops.identity(num_finalized_trees, name='num_finalized_trees'), array_ops.identity( - num_attempted_layers, name='num_attempted_layers')) + num_attempted_layers, name='num_attempted_layers'), + array_ops.identity(nodes_range, name='last_layer_nodes_range')) def serialize(self): """Serializes the ensemble into proto and returns the serialized proto. -- GitLab From acad7022b09b090da0684f209ac8d0feb1c986a2 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 15:44:55 -0700 Subject: [PATCH 193/791] Adding support of core feature columns and losses to gradient boosted trees estimators. PiperOrigin-RevId: 192521398 --- .../boosted_trees/estimator_batch/BUILD | 33 +++++ .../estimator_batch/custom_export_strategy.py | 5 +- .../dnn_tree_combined_estimator.py | 96 ++---------- .../estimator_batch/estimator.py | 19 ++- .../estimator_batch/estimator_test.py | 138 ++++++++++++++++++ .../estimator_batch/estimator_utils.py | 71 +++++++++ .../boosted_trees/estimator_batch/model.py | 27 +++- .../python/training/functions/gbdt_batch.py | 17 ++- .../training/functions/gbdt_batch_test.py | 45 +++++- 9 files changed, 346 insertions(+), 105 deletions(-) create mode 100644 tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py create mode 100644 tensorflow/contrib/boosted_trees/estimator_batch/estimator_utils.py diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD index 17e20c4b31..0f65881aee 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD +++ b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD @@ -51,6 +51,18 @@ py_library( ], ) +py_library( + name = "estimator_utils", + srcs = ["estimator_utils.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/contrib/learn", + "//tensorflow/core:protos_all_py", + "//tensorflow/python:control_flow_ops", + "//tensorflow/python:framework_ops", + ], +) + py_test( name = "trainer_hooks_test", size = "small", @@ -118,6 +130,7 @@ py_library( srcs = ["estimator.py"], srcs_version = "PY2AND3", deps = [ + ":estimator_utils", ":model", "//tensorflow/contrib/boosted_trees:losses", "//tensorflow/contrib/learn", @@ -130,6 +143,7 @@ py_library( srcs = ["dnn_tree_combined_estimator.py"], srcs_version = "PY2AND3", deps = [ + ":estimator_utils", ":trainer_hooks", "//tensorflow/contrib/boosted_trees:gbdt_batch", "//tensorflow/contrib/boosted_trees:model_ops_py", @@ -159,3 +173,22 @@ py_test( "//tensorflow/python:framework_for_generated_wrappers", ], ) + +py_test( + name = "estimator_test", + size = "medium", + srcs = ["estimator_test.py"], + srcs_version = "PY2AND3", + tags = [ + "no_gpu", + "no_pip_gpu", + "notsan", + ], + deps = [ + ":estimator", + "//tensorflow/contrib/boosted_trees:gbdt_batch", + "//tensorflow/contrib/layers:layers_py", + "//tensorflow/python:array_ops", + "//tensorflow/python:framework_for_generated_wrappers", + ], +) diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py index d9b0d89a03..62f1f4122b 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py +++ b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py @@ -39,7 +39,8 @@ _SPARSE_FLOAT_FEATURE_NAME_TEMPLATE = "%s_%d" def make_custom_export_strategy(name, convert_fn, feature_columns, - export_input_fn): + export_input_fn, + use_core_columns=False): """Makes custom exporter of GTFlow tree format. Args: @@ -58,7 +59,7 @@ def make_custom_export_strategy(name, input_fn = export_input_fn() (sorted_feature_names, dense_floats, sparse_float_indices, _, _, sparse_int_indices, _, _) = gbdt_batch.extract_features( - input_fn.features, feature_columns) + input_fn.features, feature_columns, use_core_columns) def export_fn(estimator, export_dir, checkpoint_path=None, eval_result=None): """A wrapper to export to SavedModel, and convert it to other formats.""" diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator.py b/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator.py index 2e7b8cba05..449c130b2d 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator.py +++ b/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator.py @@ -19,25 +19,19 @@ logits of the DNN. The input layer of the DNN (including the embeddings learned over sparse features) can optionally be provided to the boosted trees as an additional input feature. """ - from __future__ import absolute_import from __future__ import division from __future__ import print_function import six from tensorflow.contrib import layers +from tensorflow.contrib.boosted_trees.estimator_batch import estimator_utils from tensorflow.contrib.boosted_trees.estimator_batch import trainer_hooks from tensorflow.contrib.boosted_trees.python.ops import model_ops from tensorflow.contrib.boosted_trees.python.training.functions import gbdt_batch from tensorflow.contrib.layers.python.layers import optimizers -from tensorflow.contrib.learn.python.learn.estimators import constants from tensorflow.contrib.learn.python.learn.estimators import estimator from tensorflow.contrib.learn.python.learn.estimators import head as head_lib -from tensorflow.contrib.learn.python.learn.estimators import model_fn -from tensorflow.contrib.learn.python.learn.estimators import model_fn as contrib_model_fn_lib -from tensorflow.contrib.learn.python.learn.estimators import prediction_key -from tensorflow.python.estimator import model_fn as model_fn_lib -from tensorflow.python.estimator.export import export_output from tensorflow.python.feature_column import feature_column as feature_column_lib from tensorflow.python.framework import ops from tensorflow.python.ops import control_flow_ops @@ -48,56 +42,8 @@ from tensorflow.python.ops import variable_scope from tensorflow.python.summary import summary from tensorflow.python.training import training_util - _DNN_LEARNING_RATE = 0.001 -_CORE_MODE_TO_CONTRIB_MODE_ = { - model_fn_lib.ModeKeys.TRAIN: contrib_model_fn_lib.ModeKeys.TRAIN, - model_fn_lib.ModeKeys.EVAL: contrib_model_fn_lib.ModeKeys.EVAL, - model_fn_lib.ModeKeys.PREDICT: contrib_model_fn_lib.ModeKeys.INFER -} - - -def _core_mode_to_contrib_mode(mode): - return _CORE_MODE_TO_CONTRIB_MODE_[mode] - - -def _export_outputs_to_output_alternatives(export_outputs): - """Converts EstimatorSpec.export_outputs to output_alternatives. - - Args: - export_outputs: export_outputs created by create_estimator_spec. - Returns: - converted output_alternatives. - """ - output = dict() - if export_outputs is not None: - for key, value in export_outputs.items(): - if isinstance(value, export_output.ClassificationOutput): - exported_predictions = { - prediction_key.PredictionKey.SCORES: value.scores, - prediction_key.PredictionKey.CLASSES: value.classes - } - output[key] = (constants.ProblemType.CLASSIFICATION, - exported_predictions) - return output - return None - - -def _estimator_spec_to_model_fn_ops(estimator_spec, is_regression): - alternatives = [] - if not is_regression: - _export_outputs_to_output_alternatives(estimator_spec.export_outputs) - - return model_fn.ModelFnOps( - mode=_core_mode_to_contrib_mode(estimator_spec.mode), - predictions=estimator_spec.predictions, - loss=estimator_spec.loss, - train_op=estimator_spec.train_op, - eval_metric_ops=estimator_spec.eval_metric_ops, - output_alternatives=alternatives) - - def _get_optimizer(optimizer): if callable(optimizer): return optimizer() @@ -128,8 +74,7 @@ def _dnn_tree_combined_model_fn(features, dnn_steps_to_train=10000, tree_feature_columns=None, tree_center_bias=False, - use_core_versions=False, - is_regression=False): + use_core_versions=False): """DNN and GBDT combined model_fn. Args: @@ -169,7 +114,6 @@ def _dnn_tree_combined_model_fn(features, first fitting the bias. use_core_versions: Whether feature columns and loss are from the core (as opposed to contrib) version of tensorflow. - is_regression: Whether the problem is regression or not. Returns: A `ModelFnOps` object. @@ -305,8 +249,8 @@ def _dnn_tree_combined_model_fn(features, labels=labels, train_op_fn=_dnn_train_op_fn, logits=dnn_logits) - dnn_train_op = _estimator_spec_to_model_fn_ops(dnn_train_op, - is_regression).train_op + dnn_train_op = estimator_utils.estimator_spec_to_model_fn_ops( + dnn_train_op).train_op tree_train_op = head.create_estimator_spec( features=tree_features, @@ -314,10 +258,10 @@ def _dnn_tree_combined_model_fn(features, labels=labels, train_op_fn=_tree_train_op_fn, logits=tree_train_logits) - tree_train_op = _estimator_spec_to_model_fn_ops(tree_train_op, - is_regression).train_op + tree_train_op = estimator_utils.estimator_spec_to_model_fn_ops( + tree_train_op).train_op - model_fn_ops = _estimator_spec_to_model_fn_ops(model_fn_ops, is_regression) + model_fn_ops = estimator_utils.estimator_spec_to_model_fn_ops(model_fn_ops) else: model_fn_ops = head.create_model_fn_ops( features=features, @@ -529,26 +473,12 @@ class DNNBoostedTreeCombinedRegressor(estimator.Estimator): def _model_fn(features, labels, mode, config): return _dnn_tree_combined_model_fn( - features, - labels, - mode, - head, - dnn_hidden_units, - dnn_feature_columns, - tree_learner_config, - num_trees, - tree_examples_per_layer, - config, - dnn_optimizer, - dnn_activation_fn, - dnn_dropout, - dnn_input_layer_partitioner, - dnn_input_layer_to_tree, - dnn_steps_to_train, - tree_feature_columns, - tree_center_bias, - use_core_versions, - is_regression=True) + features, labels, mode, head, dnn_hidden_units, dnn_feature_columns, + tree_learner_config, num_trees, tree_examples_per_layer, config, + dnn_optimizer, dnn_activation_fn, dnn_dropout, + dnn_input_layer_partitioner, dnn_input_layer_to_tree, + dnn_steps_to_train, tree_feature_columns, tree_center_bias, + use_core_versions) super(DNNBoostedTreeCombinedRegressor, self).__init__( model_fn=_model_fn, model_dir=model_dir, diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py index 70454aa6db..89d0d611d2 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py +++ b/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py @@ -40,7 +40,8 @@ class GradientBoostedDecisionTreeClassifier(estimator.Estimator): label_keys=None, feature_engineering_fn=None, logits_modifier_function=None, - center_bias=True): + center_bias=True, + use_core_libs=False): """Initializes a GradientBoostedDecisionTreeClassifier estimator instance. Args: @@ -63,7 +64,8 @@ class GradientBoostedDecisionTreeClassifier(estimator.Estimator): logits_modifier_function: A modifier function for the logits. center_bias: Whether a separate tree should be created for first fitting the bias. - + use_core_libs: Whether feature columns and loss are from the core (as + opposed to contrib) version of tensorflow. Raises: ValueError: If learner_config is not valid. """ @@ -99,6 +101,7 @@ class GradientBoostedDecisionTreeClassifier(estimator.Estimator): 'examples_per_layer': examples_per_layer, 'center_bias': center_bias, 'logits_modifier_function': logits_modifier_function, + 'use_core_libs': use_core_libs, }, model_dir=model_dir, config=config, @@ -120,7 +123,8 @@ class GradientBoostedDecisionTreeRegressor(estimator.Estimator): config=None, feature_engineering_fn=None, logits_modifier_function=None, - center_bias=True): + center_bias=True, + use_core_libs=False): """Initializes a GradientBoostedDecisionTreeRegressor estimator instance. Args: @@ -145,6 +149,8 @@ class GradientBoostedDecisionTreeRegressor(estimator.Estimator): logits_modifier_function: A modifier function for the logits. center_bias: Whether a separate tree should be created for first fitting the bias. + use_core_libs: Whether feature columns and loss are from the core (as + opposed to contrib) version of tensorflow. """ head = head_lib.regression_head( label_name=label_name, @@ -166,6 +172,7 @@ class GradientBoostedDecisionTreeRegressor(estimator.Estimator): 'examples_per_layer': examples_per_layer, 'logits_modifier_function': logits_modifier_function, 'center_bias': center_bias, + 'use_core_libs': use_core_libs, }, model_dir=model_dir, config=config, @@ -189,7 +196,8 @@ class GradientBoostedDecisionTreeEstimator(estimator.Estimator): config=None, feature_engineering_fn=None, logits_modifier_function=None, - center_bias=True): + center_bias=True, + use_core_libs=False): """Initializes a GradientBoostedDecisionTreeEstimator estimator instance. Args: @@ -210,6 +218,8 @@ class GradientBoostedDecisionTreeEstimator(estimator.Estimator): logits_modifier_function: A modifier function for the logits. center_bias: Whether a separate tree should be created for first fitting the bias. + use_core_libs: Whether feature columns and loss are from the core (as + opposed to contrib) version of tensorflow. """ super(GradientBoostedDecisionTreeEstimator, self).__init__( model_fn=model.model_builder, @@ -222,6 +232,7 @@ class GradientBoostedDecisionTreeEstimator(estimator.Estimator): 'examples_per_layer': examples_per_layer, 'logits_modifier_function': logits_modifier_function, 'center_bias': center_bias, + 'use_core_libs': use_core_libs, }, model_dir=model_dir, config=config, diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py new file mode 100644 index 0000000000..0d58317bd5 --- /dev/null +++ b/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py @@ -0,0 +1,138 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for GBDT estimator.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import tempfile +from tensorflow.contrib.boosted_trees.estimator_batch import estimator +from tensorflow.contrib.boosted_trees.proto import learner_pb2 +from tensorflow.contrib.layers.python.layers import feature_column as contrib_feature_column +from tensorflow.contrib.learn.python.learn.estimators import run_config +from tensorflow.python.estimator.canned import head as head_lib +from tensorflow.python.feature_column import feature_column_lib as core_feature_column +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util +from tensorflow.python.ops.losses import losses +from tensorflow.python.platform import gfile +from tensorflow.python.platform import googletest + + +def _train_input_fn(): + features = {"x": constant_op.constant([[2.], [1.], [1.]])} + label = constant_op.constant([[1], [0], [0]], dtype=dtypes.int32) + return features, label + + +def _eval_input_fn(): + features = {"x": constant_op.constant([[1.], [2.], [2.]])} + label = constant_op.constant([[0], [1], [1]], dtype=dtypes.int32) + return features, label + + +class BoostedTreeEstimatorTest(test_util.TensorFlowTestCase): + + def setUp(self): + self._export_dir_base = tempfile.mkdtemp() + "export/" + gfile.MkDir(self._export_dir_base) + + def testFitAndEvaluateDontThrowException(self): + learner_config = learner_pb2.LearnerConfig() + learner_config.num_classes = 2 + learner_config.constraints.max_tree_depth = 1 + model_dir = tempfile.mkdtemp() + config = run_config.RunConfig() + + classifier = estimator.GradientBoostedDecisionTreeClassifier( + learner_config=learner_config, + num_trees=1, + examples_per_layer=3, + model_dir=model_dir, + config=config, + feature_columns=[contrib_feature_column.real_valued_column("x")]) + + classifier.fit(input_fn=_train_input_fn, steps=15) + classifier.evaluate(input_fn=_eval_input_fn, steps=1) + classifier.export(self._export_dir_base) + + def testFitAndEvaluateDontThrowExceptionWithCoreForEstimator(self): + learner_config = learner_pb2.LearnerConfig() + learner_config.num_classes = 2 + learner_config.constraints.max_tree_depth = 1 + model_dir = tempfile.mkdtemp() + config = run_config.RunConfig() + + # Use core head + head_fn = head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss( + loss_reduction=losses.Reduction.SUM_OVER_BATCH_SIZE) + + model = estimator.GradientBoostedDecisionTreeEstimator( + head=head_fn, + learner_config=learner_config, + num_trees=1, + examples_per_layer=3, + model_dir=model_dir, + config=config, + feature_columns=[core_feature_column.numeric_column("x")], + use_core_libs=True) + + model.fit(input_fn=_train_input_fn, steps=15) + model.evaluate(input_fn=_eval_input_fn, steps=1) + model.export(self._export_dir_base) + + def testFitAndEvaluateDontThrowExceptionWithCoreForClassifier(self): + learner_config = learner_pb2.LearnerConfig() + learner_config.num_classes = 2 + learner_config.constraints.max_tree_depth = 1 + model_dir = tempfile.mkdtemp() + config = run_config.RunConfig() + + classifier = estimator.GradientBoostedDecisionTreeClassifier( + learner_config=learner_config, + num_trees=1, + examples_per_layer=3, + model_dir=model_dir, + config=config, + feature_columns=[core_feature_column.numeric_column("x")], + use_core_libs=True) + + classifier.fit(input_fn=_train_input_fn, steps=15) + classifier.evaluate(input_fn=_eval_input_fn, steps=1) + classifier.export(self._export_dir_base) + + def testFitAndEvaluateDontThrowExceptionWithCoreForRegressor(self): + learner_config = learner_pb2.LearnerConfig() + learner_config.num_classes = 2 + learner_config.constraints.max_tree_depth = 1 + model_dir = tempfile.mkdtemp() + config = run_config.RunConfig() + + regressor = estimator.GradientBoostedDecisionTreeRegressor( + learner_config=learner_config, + num_trees=1, + examples_per_layer=3, + model_dir=model_dir, + config=config, + feature_columns=[core_feature_column.numeric_column("x")], + use_core_libs=True) + + regressor.fit(input_fn=_train_input_fn, steps=15) + regressor.evaluate(input_fn=_eval_input_fn, steps=1) + regressor.export(self._export_dir_base) + + +if __name__ == "__main__": + googletest.main() diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator_utils.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator_utils.py new file mode 100644 index 0000000000..c9cf4ae25a --- /dev/null +++ b/tensorflow/contrib/boosted_trees/estimator_batch/estimator_utils.py @@ -0,0 +1,71 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities for converting between core and contrib feature columns.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.learn.python.learn.estimators import constants +from tensorflow.contrib.learn.python.learn.estimators import model_fn +from tensorflow.contrib.learn.python.learn.estimators import model_fn as contrib_model_fn_lib +from tensorflow.contrib.learn.python.learn.estimators import prediction_key +from tensorflow.python.estimator import model_fn as model_fn_lib +from tensorflow.python.estimator.export import export_output + +_CORE_MODE_TO_CONTRIB_MODE_ = { + model_fn_lib.ModeKeys.TRAIN: contrib_model_fn_lib.ModeKeys.TRAIN, + model_fn_lib.ModeKeys.EVAL: contrib_model_fn_lib.ModeKeys.EVAL, + model_fn_lib.ModeKeys.PREDICT: contrib_model_fn_lib.ModeKeys.INFER +} + + +def _core_mode_to_contrib_mode(mode): + return _CORE_MODE_TO_CONTRIB_MODE_[mode] + + +def _export_outputs_to_output_alternatives(export_outputs): + """Converts EstimatorSpec.export_outputs to output_alternatives. + + Args: + export_outputs: export_outputs created by create_estimator_spec. + Returns: + converted output_alternatives. + """ + output = dict() + if export_outputs is not None: + for key, value in export_outputs.items(): + if isinstance(value, export_output.ClassificationOutput): + exported_predictions = { + prediction_key.PredictionKey.SCORES: value.scores, + prediction_key.PredictionKey.CLASSES: value.classes + } + output[key] = (constants.ProblemType.CLASSIFICATION, + exported_predictions) + return output + return None + + +def estimator_spec_to_model_fn_ops(estimator_spec): + alternatives = _export_outputs_to_output_alternatives( + estimator_spec.export_outputs) + + return model_fn.ModelFnOps( + mode=_core_mode_to_contrib_mode(estimator_spec.mode), + predictions=estimator_spec.predictions, + loss=estimator_spec.loss, + train_op=estimator_spec.train_op, + eval_metric_ops=estimator_spec.eval_metric_ops, + output_alternatives=alternatives) diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/model.py b/tensorflow/contrib/boosted_trees/estimator_batch/model.py index c6455a7ea3..15ab6d8145 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/model.py +++ b/tensorflow/contrib/boosted_trees/estimator_batch/model.py @@ -20,6 +20,7 @@ from __future__ import print_function import copy +from tensorflow.contrib.boosted_trees.estimator_batch import estimator_utils from tensorflow.contrib.boosted_trees.estimator_batch import trainer_hooks from tensorflow.contrib.boosted_trees.python.ops import model_ops from tensorflow.contrib.boosted_trees.python.training.functions import gbdt_batch @@ -60,6 +61,7 @@ def model_builder(features, labels, mode, params, config): feature_columns = params["feature_columns"] weight_column_name = params["weight_column_name"] num_trees = params["num_trees"] + use_core_libs = params["use_core_libs"] logits_modifier_function = params["logits_modifier_function"] if features is None: raise ValueError("At least one feature must be specified.") @@ -93,7 +95,8 @@ def model_builder(features, labels, mode, params, config): learner_config=learner_config, feature_columns=feature_columns, logits_dimension=head.logits_dimension, - features=training_features) + features=training_features, + use_core_columns=use_core_libs) with ops.name_scope("gbdt", "gbdt_optimizer"): predictions_dict = gbdt_model.predict(mode) logits = predictions_dict["predictions"] @@ -108,12 +111,22 @@ def model_builder(features, labels, mode, params, config): update_op = state_ops.assign_add(global_step, 1).op return update_op - model_fn_ops = head.create_model_fn_ops( - features=features, - mode=mode, - labels=labels, - train_op_fn=_train_op_fn, - logits=logits) + create_estimator_spec_op = getattr(head, "create_estimator_spec", None) + if use_core_libs and callable(create_estimator_spec_op): + model_fn_ops = head.create_estimator_spec( + features=features, + mode=mode, + labels=labels, + train_op_fn=_train_op_fn, + logits=logits) + model_fn_ops = estimator_utils.estimator_spec_to_model_fn_ops(model_fn_ops) + else: + model_fn_ops = head.create_model_fn_ops( + features=features, + mode=mode, + labels=labels, + train_op_fn=_train_op_fn, + logits=logits) if num_trees: if center_bias: num_trees += 1 diff --git a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py index 85b909e4f2..4bde7f3e33 100644 --- a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py +++ b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py @@ -23,7 +23,6 @@ import copy from tensorflow.contrib import learn from tensorflow.contrib import stateless - from tensorflow.contrib.boosted_trees.lib.learner.batch import categorical_split_handler from tensorflow.contrib.boosted_trees.lib.learner.batch import ordinal_split_handler from tensorflow.contrib.boosted_trees.proto import learner_pb2 @@ -141,7 +140,7 @@ class _OpRoundRobinStrategy(object): return task -def extract_features(features, feature_columns): +def extract_features(features, feature_columns, use_core_columns): """Extracts columns from a dictionary of features. Args: @@ -174,7 +173,11 @@ def extract_features(features, feature_columns): transformed_features = collections.OrderedDict() for fc in feature_columns: # pylint: disable=protected-access - if isinstance(fc, feature_column_lib._EmbeddingColumn): + if use_core_columns: + # pylint: disable=protected-access + tensor = fc_core._transform_features(features, [fc])[fc] + transformed_features[fc.name] = tensor + elif isinstance(fc, feature_column_lib._EmbeddingColumn): # pylint: enable=protected-access transformed_features[fc.name] = fc_core.input_layer( features, [fc], @@ -265,7 +268,8 @@ class GradientBoostedDecisionTreeModel(object): learner_config, features, logits_dimension, - feature_columns=None): + feature_columns=None, + use_core_columns=False): """Construct a new GradientBoostedDecisionTreeModel function. Args: @@ -338,8 +342,9 @@ class GradientBoostedDecisionTreeModel(object): if not features: raise ValueError("Features dictionary must be specified.") (fc_names, dense_floats, sparse_float_indices, sparse_float_values, - sparse_float_shapes, sparse_int_indices, sparse_int_values, - sparse_int_shapes) = extract_features(features, self._feature_columns) + sparse_float_shapes, sparse_int_indices, + sparse_int_values, sparse_int_shapes) = extract_features( + features, self._feature_columns, use_core_columns) logging.info("Active Feature Columns: " + str(fc_names)) self._fc_names = fc_names self._dense_floats = dense_floats diff --git a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch_test.py b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch_test.py index 6411f57a54..17dcb49f47 100644 --- a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch_test.py +++ b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch_test.py @@ -27,9 +27,11 @@ from tensorflow.contrib.boosted_trees.python.ops import model_ops from tensorflow.contrib.boosted_trees.python.training.functions import gbdt_batch from tensorflow.contrib.boosted_trees.python.utils import losses +from tensorflow.python.feature_column import feature_column_lib as core_feature_column from tensorflow.contrib.layers.python.layers import feature_column as feature_column_lib from tensorflow.contrib.learn.python.learn.estimators import model_fn + from tensorflow.python.framework import dtypes from tensorflow.python.framework import sparse_tensor from tensorflow.python.framework import test_util @@ -99,7 +101,8 @@ class GbdtTest(test_util.TensorFlowTestCase): array_ops.zeros([2], dtypes.int64)) (fc_names, dense_floats, sparse_float_indices, sparse_float_values, sparse_float_shapes, sparse_int_indices, sparse_int_values, - sparse_int_shapes) = (gbdt_batch.extract_features(features, None)) + sparse_int_shapes) = ( + gbdt_batch.extract_features(features, None, use_core_columns=False)) self.assertEqual(len(fc_names), 3) self.assertAllEqual(fc_names, ["dense_float", "sparse_float", "sparse_int"]) @@ -148,8 +151,9 @@ class GbdtTest(test_util.TensorFlowTestCase): "sparse_categorical", hash_bucket_size=1000000)) (fc_names, dense_floats, sparse_float_indices, sparse_float_values, sparse_float_shapes, sparse_int_indices, sparse_int_values, - sparse_int_shapes) = (gbdt_batch.extract_features( - features, feature_columns)) + sparse_int_shapes) = ( + gbdt_batch.extract_features( + features, feature_columns, use_core_columns=False)) self.assertEqual(len(fc_names), 3) self.assertAllEqual(fc_names, ["dense_float", "sparse_float", "sparse_categorical"]) @@ -174,6 +178,41 @@ class GbdtTest(test_util.TensorFlowTestCase): self.assertAllEqual(sparse_int_shapes[0].eval(), features["sparse_categorical"].dense_shape.eval()) + def testExtractFeaturesFromCoreFeatureColumns(self): + """Tests feature extraction when using core columns.""" + with self.test_session(): + features = {} + # Sparse float column does not exist in core, so only dense numeric and + # categorical. + features["dense_float"] = array_ops.zeros([2, 1], dtypes.float32) + features["sparse_categorical"] = sparse_tensor.SparseTensor( + array_ops.zeros([2, 2], dtypes.int64), + array_ops.zeros([2], dtypes.string), array_ops.zeros([2], + dtypes.int64)) + + feature_columns = set() + feature_columns.add(core_feature_column.numeric_column("dense_float")) + feature_columns.add( + core_feature_column.categorical_column_with_hash_bucket( + "sparse_categorical", hash_bucket_size=1000000)) + (fc_names, dense_floats, _, _, _, sparse_int_indices, sparse_int_values, + sparse_int_shapes) = ( + gbdt_batch.extract_features( + features, feature_columns, use_core_columns=True)) + self.assertEqual(len(fc_names), 2) + self.assertAllEqual(fc_names, ["dense_float", "sparse_categorical"]) + self.assertEqual(len(dense_floats), 1) + self.assertEqual(len(sparse_int_indices), 1) + self.assertEqual(len(sparse_int_values), 1) + self.assertEqual(len(sparse_int_shapes), 1) + self.assertAllEqual(dense_floats[0].eval(), + features["dense_float"].eval()) + self.assertAllEqual(sparse_int_indices[0].eval(), + features["sparse_categorical"].indices.eval()) + self.assertAllEqual(sparse_int_values[0].eval(), [397263, 397263]) + self.assertAllEqual(sparse_int_shapes[0].eval(), + features["sparse_categorical"].dense_shape.eval()) + def testTrainFnChiefNoBiasCentering(self): """Tests the train function running on chief without bias centering.""" with self.test_session() as sess: -- GitLab From d6e2513d60999bf0cf315c42a14c0e45eb49cda2 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 15:59:47 -0700 Subject: [PATCH 194/791] support profiling multiple tpu through one grpc and one session. data are saved with host prefix. PiperOrigin-RevId: 192523668 --- tensorflow/contrib/tpu/profiler/BUILD | 1 + .../tpu/profiler/capture_tpu_profile.cc | 53 ++++++++++++++++--- .../contrib/tpu/profiler/dump_tpu_profile.cc | 3 +- .../contrib/tpu/profiler/tpu_profiler.proto | 7 ++- .../tpu/profiler/tpu_profiler_analysis.proto | 6 ++- 5 files changed, 60 insertions(+), 10 deletions(-) diff --git a/tensorflow/contrib/tpu/profiler/BUILD b/tensorflow/contrib/tpu/profiler/BUILD index 1c32993e8e..dbf1ab6bbf 100644 --- a/tensorflow/contrib/tpu/profiler/BUILD +++ b/tensorflow/contrib/tpu/profiler/BUILD @@ -46,6 +46,7 @@ tf_cc_binary( visibility = ["//visibility:public"], deps = [ ":dump_tpu_profile", + ":tpu_profiler_analysis_proto_cc", ":tpu_profiler_proto_cc", ":version", "//tensorflow/core:framework_internal", diff --git a/tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc b/tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc index 6b198dbc16..a535884263 100644 --- a/tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc +++ b/tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc @@ -26,6 +26,7 @@ limitations under the License. #include "tensorflow/contrib/tpu/profiler/dump_tpu_profile.h" #include "tensorflow/contrib/tpu/profiler/tpu_profiler.grpc.pb.h" +#include "tensorflow/contrib/tpu/profiler/tpu_profiler_analysis.grpc.pb.h" #include "tensorflow/contrib/tpu/profiler/version.h" #include "tensorflow/core/distributed_runtime/rpc/grpc_util.h" #include "tensorflow/core/lib/core/errors.h" @@ -40,6 +41,7 @@ namespace tensorflow { namespace tpu { namespace { +using ::tensorflow::grpc::TPUProfileAnalysis; using ::tensorflow::TPUProfiler; constexpr uint64 kMaxEvents = 1000000; @@ -64,11 +66,10 @@ Status ValidateHostPortPair(const string& host_port) { return Status::OK(); } -// Returns whether the returned trace is empty. -// Failure are handled by CHECK, i.e. abort() -bool Profile(const string& service_addr, const string& logdir, int duration_ms, - const string& repository_root, const string& session_id, - const ProfileOptions& opts) { +ProfileRequest PopulateProfileRequest(int duration_ms, + const string& repository_root, + const string& session_id, + const ProfileOptions& opts) { ProfileRequest request; request.set_duration_ms(duration_ms); request.set_max_events(kMaxEvents); @@ -83,6 +84,17 @@ bool Profile(const string& service_addr, const string& logdir, int duration_ms, *request.mutable_opts() = opts; std::cout << "Limiting the number of trace events to " << kMaxEvents << std::endl; + return request; +} + +// Returns whether the returned trace is empty. +// Failure are handled by CHECK, i.e. abort() +bool Profile(const string& service_addr, const string& logdir, int duration_ms, + const string& repository_root, const string& session_id, + const ProfileOptions& opts) { + ProfileRequest request = + PopulateProfileRequest(duration_ms, repository_root, session_id, opts); + ::grpc::ClientContext context; ::grpc::ChannelArguments channel_args; // TODO(ioeric): use `SetMaxReceiveMessageSize` instead once it's available. @@ -120,7 +132,36 @@ bool NewSession(const string& service_addr, const std::vector& hostnames, int duration_ms, const string& repository_root, const string& session_id, const ProfileOptions& opts) { - return true; + NewProfileSessionRequest new_session_request; + *new_session_request.mutable_request() = + PopulateProfileRequest(duration_ms, repository_root, session_id, opts); + new_session_request.set_repository_root(repository_root); + new_session_request.set_session_id(session_id); + std::copy( + hostnames.begin(), hostnames.end(), + proto2::RepeatedFieldBackInserter(new_session_request.mutable_hosts())); + + ::grpc::ClientContext context; + ::grpc::ChannelArguments channel_args; + // TODO(qiuminxu): use `NewHostPortGrpcChannel` instead once their + // `ValidateHostPortPair` checks for empty host string case. + channel_args.SetMaxReceiveMessageSize(std::numeric_limits::max()); + // TODO(jiesun): GRPC support following relevant naming scheme: + // 1. dns:///host:port + // 2. ipv4:host:port or ipv6:[host]:port + // We might need to change the prefix which depends on what TPU name resolver + // will give us. + std::unique_ptr stub = + TPUProfileAnalysis::NewStub(::grpc::CreateCustomChannel( + "dns:///" + service_addr, ::grpc::InsecureChannelCredentials(), + channel_args)); + NewProfileSessionResponse new_session_response; + TF_QCHECK_OK(FromGrpcStatus( + stub->NewSession(&context, new_session_request, &new_session_response))); + + std::cout << "Profile session succeed for hosts:" + << str_util::Join(hostnames, ","); + return new_session_response.empty_trace(); } } // namespace diff --git a/tensorflow/contrib/tpu/profiler/dump_tpu_profile.cc b/tensorflow/contrib/tpu/profiler/dump_tpu_profile.cc index ae508583f8..b53f9be2e2 100644 --- a/tensorflow/contrib/tpu/profiler/dump_tpu_profile.cc +++ b/tensorflow/contrib/tpu/profiler/dump_tpu_profile.cc @@ -64,7 +64,8 @@ Status WriteGzippedDataToFile(const string& filename, const string& data) { Status DumpTraceToLogDirectory(StringPiece run_dir, const string& host_prefix, const string& encoded_trace, std::ostream* os) { - string proto_path = JoinPath(run_dir, kProtoTraceFileName); + string proto_path = + JoinPath(run_dir, StrCat(host_prefix, kProtoTraceFileName)); TF_RETURN_IF_ERROR( WriteStringToFile(Env::Default(), proto_path, encoded_trace)); LOG(INFO) << "Dumped raw-proto trace data to " << proto_path; diff --git a/tensorflow/contrib/tpu/profiler/tpu_profiler.proto b/tensorflow/contrib/tpu/profiler/tpu_profiler.proto index 8505c4bc69..7be694e866 100644 --- a/tensorflow/contrib/tpu/profiler/tpu_profiler.proto +++ b/tensorflow/contrib/tpu/profiler/tpu_profiler.proto @@ -96,5 +96,10 @@ message ProfileResponse { // Data payload for each required tools. repeated ProfileToolData tool_data = 6; - // next-field: 7 + + // When we write profiling data directly to repository directory, we need a + // way to figure out whether the captured trace is empty (due to idle TPU). + bool empty_trace = 7; + + // next-field: 8 } diff --git a/tensorflow/contrib/tpu/profiler/tpu_profiler_analysis.proto b/tensorflow/contrib/tpu/profiler/tpu_profiler_analysis.proto index a4fc8d4e87..8b0bbde98e 100644 --- a/tensorflow/contrib/tpu/profiler/tpu_profiler_analysis.proto +++ b/tensorflow/contrib/tpu/profiler/tpu_profiler_analysis.proto @@ -7,13 +7,15 @@ message NewProfileSessionRequest { ProfileRequest request = 1; string repository_root = 2; repeated string hosts = 3; + string session_id = 4; } message NewProfileSessionResponse { // Auxiliary error_message. string error_message = 1; - // If success, return session identifier for future reference. - string session_id = 2; + + // Whether all hosts had returned a empty trace. + bool empty_trace = 2; } message EnumProfileSessionsAndToolsRequest { -- GitLab From e7cfede7bb75f22de890f6e94851121c949d8ba9 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 16:05:42 -0700 Subject: [PATCH 195/791] Speed up computing mean confidence intervals by avoiding tf.while_loop. Implement a vectorized way to compute the same thing instead. PiperOrigin-RevId: 192524667 --- .../kernel_tests/statistical_testing_test.py | 23 +++++++++ .../python/ops/statistical_testing.py | 48 +++++++++---------- 2 files changed, 45 insertions(+), 26 deletions(-) diff --git a/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py b/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py index c4fb669ebb..ce6cf702d5 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import numpy as np from tensorflow.contrib.distributions.python.ops import statistical_testing as st +from tensorflow.python.framework import ops from tensorflow.python.platform import test @@ -215,6 +216,28 @@ class StatisticalTestingTest(test.TestCase): samples, [[0., 1.]], [[1., 2.]], error_rate=0.5) _ = sess.run(op) + def test_do_maximum_mean(self): + n = 117 + envelope = 0.02 # > 2 / n, but < 3 / n + rng = np.random.RandomState(seed=8) + samples = rng.uniform(size=n).astype(np.float32) + + # Compute the answer in TF using the code under test + with self.test_session() as sess: + envelope_t = ops.convert_to_tensor(envelope) + max_mean = st._do_maximum_mean(samples, envelope_t, 1) + max_mean = sess.run(max_mean) + + # Compute the correct answer for this case in numpy. In this + # example, `n` and `envelope` are such that `samples[2]` is the + # element that should be taken partially, regardless of the + # content of the `samples` array (see algorithm description in + # `../ops/statistical_testing.py`). + samples = sorted(samples) + weight = 1. / n - (envelope - 2. / n) + answer = samples[2] * weight + sum(samples[3:]) / n + envelope * 1. + self.assertAllClose(max_mean, answer, rtol=1e-9) + if __name__ == '__main__': test.main() diff --git a/tensorflow/contrib/distributions/python/ops/statistical_testing.py b/tensorflow/contrib/distributions/python/ops/statistical_testing.py index 9b9fff0afa..9c69435fac 100644 --- a/tensorflow/contrib/distributions/python/ops/statistical_testing.py +++ b/tensorflow/contrib/distributions/python/ops/statistical_testing.py @@ -130,7 +130,7 @@ import itertools from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import clip_ops from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn_ops @@ -169,31 +169,27 @@ def _do_maximum_mean(samples, envelope, high, name=None): samples = array_ops.transpose(samples, perm) samples = _batch_sort_vector(samples) - batch_shape = array_ops.shape(samples)[:-1] - n = array_ops.shape(samples)[-1] - step = 1. / math_ops.cast(n, dtype=samples.dtype.base_dtype) - - def _loop_body(iter_, total, to_skip): - total = array_ops.where( - step <= to_skip, - total, - array_ops.where( - to_skip > 0., - total + (step - to_skip) * samples[..., iter_], - total + step * samples[..., iter_])) - to_skip = array_ops.where(step <= to_skip, to_skip - step, 0.) - return [iter_ + 1, total, to_skip] - - _, total, _ = control_flow_ops.while_loop( - cond=lambda iter_, *args: iter_ < n, - body=_loop_body, - loop_vars=[ - 0, - array_ops.zeros(batch_shape, dtype=samples.dtype.base_dtype), - envelope, # to_skip - ]) - - return total + envelope * high + + # The maximum mean is given by taking `envelope`-worth of + # probability from the smallest samples and moving it to the + # maximum value. This amounts to: + # - ignoring the smallest k samples, where `k/n < envelope` + # - taking a `1/n - (envelope - k/n)` part of the index k sample + # - taking all the other samples + # - and adding `envelope * high` at the end. + # The following is a vectorized and batched way of computing this. + # `max_mean_contrib` is a mask implementing the previous. + batch_size = array_ops.shape(samples)[-1] + batch_size = math_ops.cast(batch_size, dtype=samples.dtype.base_dtype) + step = 1. / batch_size + cum_steps = step * math_ops.range( + 1, batch_size + 1, dtype=samples.dtype.base_dtype) + max_mean_contrib = clip_ops.clip_by_value( + cum_steps - envelope[..., array_ops.newaxis], + clip_value_min=0., + clip_value_max=step) + return math_ops.reduce_sum( + samples * max_mean_contrib, axis=-1) + envelope * high def _maximum_mean(samples, envelope, high, name=None): -- GitLab From 2b94b444d53cfa6875f7874197cbc584a06d7a30 Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Wed, 11 Apr 2018 16:43:33 -0700 Subject: [PATCH 196/791] Move callback into bound function to avoid copying. PiperOrigin-RevId: 192530231 --- .../core/common_runtime/rendezvous_mgr.cc | 51 +++++++++++-------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/tensorflow/core/common_runtime/rendezvous_mgr.cc b/tensorflow/core/common_runtime/rendezvous_mgr.cc index 60263d1471..93f24a3217 100644 --- a/tensorflow/core/common_runtime/rendezvous_mgr.cc +++ b/tensorflow/core/common_runtime/rendezvous_mgr.cc @@ -121,27 +121,36 @@ void IntraProcessRendezvous::RecvAsync(const ParsedKey& parsed, // Recv the tensor from local_. local_->RecvAsync( parsed, recv_args, - [this, parsed, done]( - const Status& status, const Rendezvous::Args& send_args, - const Rendezvous::Args& recv_args, const Tensor& in, bool is_dead) { - // If "in" is an uninitialized tensor, do copy-construction to preserve - // the uninitialized state, along with data type and shape info, which - // is useful for debugger purposes. - Tensor* out = in.IsInitialized() ? new Tensor : new Tensor(in); - - StatusCallback final_callback = [done, send_args, recv_args, out, - is_dead](const Status& s) { - done(s, send_args, recv_args, *out, is_dead); - delete out; - }; - - if (status.ok() && in.IsInitialized()) { - SameWorkerRecvDone(parsed, send_args, recv_args, in, out, - std::move(final_callback)); - } else { - final_callback(status); - } - }); + std::bind( + [this, parsed](DoneCallback done, + // Begin unbound arguments. + const Status& status, + const Rendezvous::Args& send_args, + const Rendezvous::Args& recv_args, const Tensor& in, + bool is_dead) { + // If "in" is an uninitialized tensor, do copy-construction to + // preserve the uninitialized state, along with data type and shape + // info, which is useful for debugger purposes. + Tensor* out = in.IsInitialized() ? new Tensor : new Tensor(in); + + auto final_callback = std::bind( + [send_args, recv_args, out, is_dead](DoneCallback done, + // Begin unbound arguments. + const Status& s) { + done(s, send_args, recv_args, *out, is_dead); + delete out; + }, + std::move(done), std::placeholders::_1); + + if (status.ok() && in.IsInitialized()) { + SameWorkerRecvDone(parsed, send_args, recv_args, in, out, + std::move(final_callback)); + } else { + final_callback(status); + } + }, + std::move(done), std::placeholders::_1, std::placeholders::_2, + std::placeholders::_3, std::placeholders::_4, std::placeholders::_5)); } void IntraProcessRendezvous::StartAbort(const Status& s) { -- GitLab From 3734bb6ca9f5df8dbbf4bceb80b28d69452bdc61 Mon Sep 17 00:00:00 2001 From: Younghee Kwon Date: Wed, 11 Apr 2018 16:59:45 -0700 Subject: [PATCH 197/791] boosted_trees: make sure ensemble deserialization happens for the non-TRAIN modes too. PiperOrigin-RevId: 192532297 --- .../python/estimator/canned/boosted_trees.py | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/tensorflow/python/estimator/canned/boosted_trees.py b/tensorflow/python/estimator/canned/boosted_trees.py index 58af59dbb1..0ecc8c7089 100644 --- a/tensorflow/python/estimator/canned/boosted_trees.py +++ b/tensorflow/python/estimator/canned/boosted_trees.py @@ -317,27 +317,28 @@ def _bt_model_fn( head.logits_dimension) # Create Ensemble resources. - if is_single_machine: - tree_ensemble = boosted_trees_ops.TreeEnsemble(name=name) - local_tree_ensemble = tree_ensemble - ensemble_reload = control_flow_ops.no_op() - else: - tree_ensemble = boosted_trees_ops.TreeEnsemble(name=name) - with ops.device(worker_device): - local_tree_ensemble = boosted_trees_ops.TreeEnsemble( - name=name + '_local', is_local=True) - # TODO(soroush): Do partial updates if this becomes a bottleneck. - ensemble_reload = local_tree_ensemble.deserialize( - *tree_ensemble.serialize()) - + tree_ensemble = boosted_trees_ops.TreeEnsemble(name=name) # Create logits. if mode != model_fn.ModeKeys.TRAIN: logits = boosted_trees_ops.predict( - tree_ensemble_handle=local_tree_ensemble.resource_handle, + # For non-TRAIN mode, ensemble doesn't change after initialization, + # so no local copy is needed; using tree_ensemble directly. + tree_ensemble_handle=tree_ensemble.resource_handle, bucketized_features=input_feature_list, logits_dimension=head.logits_dimension, max_depth=tree_hparams.max_depth) else: + if is_single_machine: + local_tree_ensemble = tree_ensemble + ensemble_reload = control_flow_ops.no_op() + else: + # Have a local copy of ensemble for the distributed setting. + with ops.device(worker_device): + local_tree_ensemble = boosted_trees_ops.TreeEnsemble( + name=name + '_local', is_local=True) + # TODO(soroush): Do partial updates if this becomes a bottleneck. + ensemble_reload = local_tree_ensemble.deserialize( + *tree_ensemble.serialize()) if cache: cached_tree_ids, cached_node_ids, cached_logits = cache.lookup() else: -- GitLab From 81a9ceaf7290b2260f636609a83b01b9ab2224d7 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 17:19:20 -0700 Subject: [PATCH 198/791] Update ops-related pbtxt files. PiperOrigin-RevId: 192534931 --- tensorflow/core/ops/ops.pbtxt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index 6af77be148..43fd09fb72 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -4135,6 +4135,10 @@ op { name: "num_attempted_layers" type: DT_INT32 } + output_arg { + name: "last_layer_nodes_range" + type: DT_INT32 + } is_stateful: true } op { -- GitLab From d62a5a11e99b391f2e61e80c4f0a80def6ff6508 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 17:29:32 -0700 Subject: [PATCH 199/791] Automated g4 rollback of changelist 192516190 PiperOrigin-RevId: 192536085 --- tensorflow/core/grappler/op_types.cc | 8 +- tensorflow/core/grappler/op_types.h | 1 - .../grappler/optimizers/constant_folding.cc | 95 ++----------------- .../optimizers/constant_folding_test.cc | 80 +--------------- 4 files changed, 16 insertions(+), 168 deletions(-) diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc index cfe1329dbf..9c45aed62f 100644 --- a/tensorflow/core/grappler/op_types.cc +++ b/tensorflow/core/grappler/op_types.cc @@ -249,10 +249,6 @@ bool IsPrint(const NodeDef& node) { return node.op() == "Print"; } bool IsProd(const NodeDef& node) { return node.op() == "Prod"; } -bool IsRandomShuffle(const NodeDef& node) { - return node.op() == "RandomShuffle"; -} - bool IsReal(const NodeDef& node) { return node.op() == "Real"; } bool IsRealDiv(const NodeDef& node) { return node.op() == "RealDiv"; } @@ -302,7 +298,9 @@ bool IsShape(const NodeDef& node) { return node.op() == "Shape"; } bool IsShapeN(const NodeDef& node) { return node.op() == "ShapeN"; } -bool IsShuffle(const NodeDef& node) { return node.op() == "Shuffle"; } +bool IsShuffle(const NodeDef& node) { + return node.op() == "Shuffle" || node.op() == "RandomShuffle"; +} bool IsSigmoidGrad(const NodeDef& node) { return node.op() == "SigmoidGrad"; } diff --git a/tensorflow/core/grappler/op_types.h b/tensorflow/core/grappler/op_types.h index 0573b02604..79fd05e187 100644 --- a/tensorflow/core/grappler/op_types.h +++ b/tensorflow/core/grappler/op_types.h @@ -98,7 +98,6 @@ bool IsPolygamma(const NodeDef& node); bool IsPrint(const NodeDef& node); bool IsProd(const NodeDef& node); bool IsPow(const NodeDef& node); -bool IsRandomShuffle(const NodeDef& node); bool IsReal(const NodeDef& node); bool IsRealDiv(const NodeDef& node); bool IsRelu6Grad(const NodeDef& node); diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc index 17d8b7421c..b2a1ce6ab6 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding.cc +++ b/tensorflow/core/grappler/optimizers/constant_folding.cc @@ -1574,99 +1574,24 @@ Status ConstantFolding::SimplifyGraph(GraphDef* optimized_graph, continue; } - // Remove Shuffle or Transpose op over dimensions of size 1. - if (use_shape_info && (IsShuffle(*node) || IsTranspose(*node)) && - !properties->GetInputProperties(node->name()).empty()) { - const auto& shape = - properties->GetInputProperties(node->name())[0].shape(); - if (shape.unknown_rank()) { - // Not optimizable. - continue; - } - const auto& p = properties->GetInputProperties(node->name())[1]; - if (TensorShape::IsValid(p.shape()) && p.has_value()) { - Tensor perm(p.dtype(), p.shape()); - if (!perm.FromProto(p.value())) { - return errors::InvalidArgument("Cannot parse tensor from proto: ", - p.value().DebugString()); - } - std::vector permutation; - for (int j = 0; j < perm.NumElements(); ++j) { - if (perm.dtype() == DT_INT64) { - permutation.push_back(perm.vec()(j)); - } else { - permutation.push_back(perm.vec()(j)); - } - } - if (permutation.size() != shape.dim_size()) { - // Number of elements in perm should be same as dim_size. Skip if not. - continue; - } - // The node is replaceable iff - // dim_size == 0 || all dims have size 1 || - // all dims with > 1 size are not permuted. - bool replaceable = true; - for (int j = 0; replaceable && j < shape.dim_size(); ++j) { - replaceable &= shape.dim(j).size() == 1 || j == permutation[j]; - } - if (replaceable) { - ReplaceOperationWithIdentity(0, node, optimized_graph); - continue; - } - } - } - - // Remove RandomShuffle op if it is scalar or first dimension is of size 1. - if (use_shape_info && IsRandomShuffle(*node) && - !properties->GetInputProperties(node->name()).empty()) { + // Remove Shuffle or Reverse op over scalar values. + if (use_shape_info && + !properties->GetInputProperties(node->name()).empty() && + (IsShuffle(*node) || IsReverse(*node) || IsTranspose(*node))) { const auto& shape = properties->GetInputProperties(node->name())[0].shape(); // The node is replaceable iff - // unknown_rank == false && (dim_size == 0 || first dim is of size 1) - if (!shape.unknown_rank() && - (shape.dim_size() == 0 || shape.dim(0).size() == 1)) { + // unknown_rank == false && (dim_size == 0 || all dims have size 1) + bool replaceable = !shape.unknown_rank(); + for (int j = 0; replaceable && j < shape.dim_size(); ++j) { + replaceable &= shape.dim(j).size() == 1; + } + if (replaceable) { ReplaceOperationWithIdentity(0, node, optimized_graph); continue; } } - // Remove Reverse op over dimensions with size 1. - if (use_shape_info && IsReverse(*node) && - !properties->GetInputProperties(node->name()).empty()) { - const auto& shape = - properties->GetInputProperties(node->name())[0].shape(); - const auto& a = properties->GetInputProperties(node->name())[1]; - if (TensorShape::IsValid(a.shape()) && a.has_value()) { - Tensor axis(a.dtype(), a.shape()); - if (!axis.FromProto(a.value())) { - return errors::InvalidArgument("Cannot parse tensor from proto: ", - a.value().DebugString()); - } - std::set target_axes; - for (int j = 0; j < axis.NumElements(); ++j) { - if (axis.dtype() == DT_INT64) { - target_axes.insert(axis.vec()(j)); - } else { - target_axes.insert(axis.vec()(j)); - } - } - - // The node is replaceable iff - // unknown_rank == false && - // (dim_size == 0 || all dims have size 1 || - // all dims with > 1 size are not in target_axes) - bool replaceable = !shape.unknown_rank(); - for (int j = 0; replaceable && j < shape.dim_size(); ++j) { - replaceable &= shape.dim(j).size() == 1 || - target_axes.find(j) == target_axes.end(); - } - if (replaceable) { - ReplaceOperationWithIdentity(0, node, optimized_graph); - continue; - } - } - } - if (use_shape_info && IsSlice(*node) && properties->GetInputProperties(node->name()).size() == 3) { const auto& input = properties->GetInputProperties(node->name())[0]; diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc index 7453fb6731..31abe43846 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc +++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc @@ -1389,6 +1389,8 @@ TEST_F(ConstantFoldingTest, SplitVRemoval) { ops::SplitV s1(scope.WithOpName("s1"), in1, size_splits1, split_dim, 1); ops::SplitV s2(scope.WithOpName("s2"), in2, size_splits2, split_dim, 2); + LOG(INFO) << s1.output.size(); + LOG(INFO) << s2.output.size(); ops::Add out(scope.WithOpName("out"), s1[0], s2[0]); GrapplerItem item; @@ -1416,45 +1418,7 @@ TEST_F(ConstantFoldingTest, SplitVRemoval) { CompareGraphs(want, got); } -TEST_F(ConstantFoldingTest, TransposeOnSize1DimsRemoval) { - tensorflow::Scope scope = tensorflow::Scope::NewRootScope(); - - Output in1 = ops::Variable(scope.WithOpName("in1"), TensorShape({1, 2, 4, 1}), - DT_FLOAT); - Output p1 = ops::Const(scope.WithOpName("p1"), {3, 2, 1, 0}, {4}); - Output in2 = ops::Variable(scope.WithOpName("in2"), TensorShape({1, 4, 2, 1}), - DT_FLOAT); - Output p2 = ops::Const(scope.WithOpName("p2"), {3, 1, 2, 0}, {4}); - ops::Transpose t1(scope.WithOpName("t1"), in1, p1); - ops::Transpose t2(scope.WithOpName("t2").WithControlDependencies({in1}), in2, - p2); - - ops::Add out1(scope.WithOpName("out1"), t1, t2); - - GrapplerItem item; - item.fetch = {"out1"}; - TF_CHECK_OK(scope.ToGraphDef(&item.graph)); - - ConstantFolding optimizer(nullptr /* cpu_device */); - GraphDef got; - Status status = optimizer.Optimize(nullptr, item, &got); - TF_EXPECT_OK(status); - - GraphDef want; - AddNode("in1", "VariableV2", {}, {}, &want); - AddNode("in2", "VariableV2", {}, {}, &want); - AddNode("p1", "Const", {}, {}, &want); - AddNode("p2", "Const", {}, {}, &want); - AddNode("t1", "Transpose", {"in1", "p1"}, {}, &want); - AddNode("t2", "Identity", - {"in2", AsControlDependency("in1"), AsControlDependency("p2")}, {}, - &want); - AddNode("out1", "Add", {"t1", "t2"}, {}, &want); - - CompareGraphs(want, got); -} - -TEST_F(ConstantFoldingTest, RandomShuffleOnScalarRemoval) { +TEST_F(ConstantFoldingTest, ShuffleReverseOnScalarRemoval) { tensorflow::Scope scope = tensorflow::Scope::NewRootScope(); Output in1 = @@ -1488,44 +1452,6 @@ TEST_F(ConstantFoldingTest, RandomShuffleOnScalarRemoval) { CompareGraphs(want, got); } -TEST_F(ConstantFoldingTest, ReverseOnSize1DimsRemoval) { - tensorflow::Scope scope = tensorflow::Scope::NewRootScope(); - - Output in1 = ops::Variable(scope.WithOpName("in1"), TensorShape({1, 2, 4, 1}), - DT_FLOAT); - Output a1 = ops::Const(scope.WithOpName("a1"), {3, 2, 1, 0}, {4}); - Output in2 = ops::Variable(scope.WithOpName("in2"), TensorShape({1, 2, 4, 1}), - DT_FLOAT); - Output a2 = ops::Const(scope.WithOpName("a2"), {0, 3}, {2}); - ops::Reverse r1(scope.WithOpName("r1"), in1, a1); - ops::Reverse r2(scope.WithOpName("r2").WithControlDependencies({in1}), in2, - a2); - - ops::Add out1(scope.WithOpName("out1"), r1, r2); - - GrapplerItem item; - item.fetch = {"out1"}; - TF_CHECK_OK(scope.ToGraphDef(&item.graph)); - - ConstantFolding optimizer(nullptr /* cpu_device */); - GraphDef got; - Status status = optimizer.Optimize(nullptr, item, &got); - TF_EXPECT_OK(status); - - GraphDef want; - AddNode("in1", "VariableV2", {}, {}, &want); - AddNode("in2", "VariableV2", {}, {}, &want); - AddNode("a1", "Const", {}, {}, &want); - AddNode("a2", "Const", {}, {}, &want); - AddNode("r1", "ReverseV2", {"in1", "a1"}, {}, &want); - AddNode("r2", "Identity", - {"in2", AsControlDependency("in1"), AsControlDependency("a2")}, {}, - &want); - AddNode("out1", "Add", {"r1", "r2"}, {}, &want); - - CompareGraphs(want, got); -} - TEST_F(ConstantFoldingTest, SliceWithSameDimensionRemoval) { { // size = {3, 5} tensorflow::Scope scope = tensorflow::Scope::NewRootScope(); -- GitLab From 7de7245a7b102107b6f6cd20912db5f5be2c955c Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 11 Apr 2018 17:49:47 -0700 Subject: [PATCH 200/791] Update docs of reduce_max/reduce_min for real numeric type (#18422) Both reduce_max and reduce_min only work for real numeric type as complex numbers do not apply. This fix update the docs with `numeric type` -> `real numeric type`. Note that the current kernel registration in reduction_ops_max.cc and reduction_ops_min.cc use `TF_CALL_REAL_NUMBER_TYPES` so it is good. The op registraton for Max and Min inside math_ops.cc should be `.Attr("T: realnumbertype")` instead of `numbertype`. However, such a change will break API compatibility so leave it alone. Signed-off-by: Yong Tang --- tensorflow/python/ops/math_ops.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py index 01d670ea2d..a38ecb2acb 100644 --- a/tensorflow/python/ops/math_ops.py +++ b/tensorflow/python/ops/math_ops.py @@ -1632,7 +1632,7 @@ def reduce_min(input_tensor, tensor with a single element is returned. Args: - input_tensor: The tensor to reduce. Should have numeric type. + input_tensor: The tensor to reduce. Should have real numeric type. axis: The dimensions to reduce. If `None` (the default), reduces all dimensions. Must be in the range `[-rank(input_tensor), rank(input_tensor))`. @@ -1681,7 +1681,7 @@ def reduce_max(input_tensor, tensor with a single element is returned. Args: - input_tensor: The tensor to reduce. Should have numeric type. + input_tensor: The tensor to reduce. Should have real numeric type. axis: The dimensions to reduce. If `None` (the default), reduces all dimensions. Must be in the range `[-rank(input_tensor), rank(input_tensor))`. -- GitLab From 1a721ecd9a9992d48c0deb3008b1fc8df297d300 Mon Sep 17 00:00:00 2001 From: Austin Anderson Date: Wed, 11 Apr 2018 17:46:08 -0700 Subject: [PATCH 201/791] Internal testing changes PiperOrigin-RevId: 192537874 --- tensorflow/contrib/lite/schema/BUILD | 3 +++ tensorflow/contrib/lite/testing/BUILD | 3 +++ tensorflow/contrib/lite/tools/BUILD | 3 +++ 3 files changed, 9 insertions(+) diff --git a/tensorflow/contrib/lite/schema/BUILD b/tensorflow/contrib/lite/schema/BUILD index 246ec85fe4..9717a4a1a4 100644 --- a/tensorflow/contrib/lite/schema/BUILD +++ b/tensorflow/contrib/lite/schema/BUILD @@ -63,6 +63,9 @@ cc_test( "schema.fbs", "schema_v3.fbs", ], + tags = [ + "tflite_not_portable_android", + ], deps = [ "//tensorflow/core:lib_platform", "@com_google_googletest//:gtest", diff --git a/tensorflow/contrib/lite/testing/BUILD b/tensorflow/contrib/lite/testing/BUILD index 1ce89a25fd..2c226e76d4 100644 --- a/tensorflow/contrib/lite/testing/BUILD +++ b/tensorflow/contrib/lite/testing/BUILD @@ -161,6 +161,9 @@ cc_test( size = "small", srcs = ["tflite_driver_test.cc"], data = ["//tensorflow/contrib/lite:testdata/multi_add.bin"], + tags = [ + "tflite_not_portable_android", + ], deps = [ ":tflite_driver", "@com_google_googletest//:gtest_main", diff --git a/tensorflow/contrib/lite/tools/BUILD b/tensorflow/contrib/lite/tools/BUILD index 44fde69a1e..7b3569ea9c 100644 --- a/tensorflow/contrib/lite/tools/BUILD +++ b/tensorflow/contrib/lite/tools/BUILD @@ -78,6 +78,9 @@ cc_test( "//tensorflow/contrib/lite:testdata/test_model.bin", "//tensorflow/contrib/lite:testdata/test_model_broken.bin", ], + tags = [ + "tflite_not_portable_android", + ], deps = [ ":gen_op_registration", "@com_google_googletest//:gtest", -- GitLab From 9d7eee0d7fee883ffa3711f4e80b2c93ff5aecbc Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 11 Apr 2018 17:50:45 -0700 Subject: [PATCH 202/791] Imporve shape function of RandomUniformInt (#18420) * Imporve shape function of RandomUniformInt The input of `minval` and `maxval` of `RandomUniformInt` should be scalar though it is not checked in the shape function. This fix improves the shape function with the rank check, and adds test case for it. Signed-off-by: Yong Tang * Add test case for maxval and minval for RandomUniformInt Signed-off-by: Yong Tang --- tensorflow/core/ops/random_ops.cc | 7 ++++++- .../python/kernel_tests/random/random_ops_test.py | 11 +++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/ops/random_ops.cc b/tensorflow/core/ops/random_ops.cc index f6c668f5c9..416ce9c0d8 100644 --- a/tensorflow/core/ops/random_ops.cc +++ b/tensorflow/core/ops/random_ops.cc @@ -43,7 +43,12 @@ REGISTER_OP("RandomUniformInt") .Attr("seed2: int = 0") .Attr("Tout: {int32, int64}") .Attr("T: {int32, int64}") - .SetShapeFn(shape_inference::RandomShape); + .SetShapeFn([](InferenceContext* c) { + ShapeHandle unused; + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused)); + return shape_inference::RandomShape(c); + }); REGISTER_OP("RandomStandardNormal") .Input("shape: T") diff --git a/tensorflow/python/kernel_tests/random/random_ops_test.py b/tensorflow/python/kernel_tests/random/random_ops_test.py index df37dd98ec..e4b5c3832a 100644 --- a/tensorflow/python/kernel_tests/random/random_ops_test.py +++ b/tensorflow/python/kernel_tests/random/random_ops_test.py @@ -228,6 +228,17 @@ class RandomUniformTest(test.TestCase): print("count = ", count) self.assertTrue(count < count_limit) + def testUniformIntsWithInvalidShape(self): + for dtype in dtypes.int32, dtypes.int64: + with self.assertRaisesRegexp( + ValueError, "Shape must be rank 0 but is rank 1"): + random_ops.random_uniform( + [1000], minval=[1, 2], maxval=3, dtype=dtype) + with self.assertRaisesRegexp( + ValueError, "Shape must be rank 0 but is rank 1"): + random_ops.random_uniform( + [1000], minval=1, maxval=[2, 3], dtype=dtype) + # Check that uniform ints actually follow a uniform distribution. def testUniformInts(self): minv = -2 -- GitLab From da0fed895c2cb8d8f16d0a8083bb635f623cfa75 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 11 Apr 2018 17:51:41 -0700 Subject: [PATCH 203/791] Add deprecated_args decoration to expand_dims (#18419) * Add deprecated_args decoration to expand_dims This fix adds deprecated_args decoration to expand_dims as `dims` has been deprecated and in favor of `axis` Signed-off-by: Yong Tang * Enhance deprecated args with deprecation.deprecated_argument_lookup Signed-off-by: Yong Tang --- tensorflow/python/ops/array_ops.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index fa26e07c85..9e136937f6 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -144,6 +144,7 @@ def identity(input, name=None): # pylint: disable=redefined-builtin # pylint: disable=redefined-builtin,protected-access @tf_export("expand_dims") +@deprecation.deprecated_args(None, "Use the `axis` argument instead", "dim") def expand_dims(input, axis=None, name=None, dim=None): """Inserts a dimension of 1 into a tensor's shape. @@ -193,11 +194,7 @@ def expand_dims(input, axis=None, name=None, dim=None): Raises: ValueError: if both `dim` and `axis` are specified. """ - # TODO(aselle): Remove argument dim - if dim is not None: - if axis is not None: - raise ValueError("can't specify both 'dim' and 'axis'") - axis = dim + axis = deprecation.deprecated_argument_lookup("axis", axis, "dim", dim) return gen_array_ops.expand_dims(input, axis, name) -- GitLab From a75a5e48a4f9240a02a45119e77b28363e772bef Mon Sep 17 00:00:00 2001 From: Jonathan Hseu Date: Wed, 11 Apr 2018 17:54:10 -0700 Subject: [PATCH 204/791] Improve comment --- tensorflow/contrib/lite/toco/model.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/lite/toco/model.h b/tensorflow/contrib/lite/toco/model.h index 8a936842d9..d0ae8d389f 100644 --- a/tensorflow/contrib/lite/toco/model.h +++ b/tensorflow/contrib/lite/toco/model.h @@ -151,9 +151,9 @@ enum class AxesOrder { }; // The type of the scalars in an array. -// Note that does not by itself tell whether the values in the array are -// real (are literally interpreted as real numbers) or quantized (only acquire -// a meaning as real numbers in conjunction with QuantizationParams). +// Note that the type does not by itself tell whether the values in the array +// are real (are literally interpreted as real numbers) or quantized (only +// acquire a meaning as real numbers in conjunction with QuantizationParams). // // In practice though: // float values are always real -- GitLab From 40c40bbc4b52a2036b2f6a504f2b3895d789639f Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 11 Apr 2018 18:01:38 -0700 Subject: [PATCH 205/791] Add negative axis support for tf.manip.roll (#18409) * Add negative axis support for tf.manip.roll This fix tries to support negative axis for tf.manip.roll. The tf.manip.roll is supposed to be compatible with numpy.roll which does support negative axis. Signed-off-by: Yong Tang * Add test case for negative axis support for tf.manip.roll Signed-off-by: Yong Tang * Add axis check so that negative axis is within the range Negative axis should be 0 <= axis + dims < dims Signed-off-by: Yong Tang * Add additional test cases Signed-off-by: Yong Tang * Fix pylint issue Signed-off-by: Yong Tang --- tensorflow/core/kernels/roll_op.cc | 7 +++++-- tensorflow/python/kernel_tests/manip_ops_test.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/kernels/roll_op.cc b/tensorflow/core/kernels/roll_op.cc index bcbdbee058..4b630809c5 100644 --- a/tensorflow/core/kernels/roll_op.cc +++ b/tensorflow/core/kernels/roll_op.cc @@ -254,8 +254,11 @@ class RollOp : public OpKernel { // total modulo sum of shifts for each dimension gtl::InlinedVector shift_mod_sum(num_dims, 0); for (int i = 0; i < num_shifts; i++) { - const int axis = axis_flat(i); - OP_REQUIRES(context, axis < num_dims, + int axis = axis_flat(i); + if (axis < 0) { + axis += num_dims; + } + OP_REQUIRES(context, 0 <= axis && axis < num_dims, errors::InvalidArgument("axis ", axis, " is out of range")); const int ds = std::max(static_cast(input.dim_size(axis)), 1); const int sum = shift_mod_sum[axis] + static_cast(shift_flat(i)); diff --git a/tensorflow/python/kernel_tests/manip_ops_test.py b/tensorflow/python/kernel_tests/manip_ops_test.py index b8200ac0cb..7948a475bb 100644 --- a/tensorflow/python/kernel_tests/manip_ops_test.py +++ b/tensorflow/python/kernel_tests/manip_ops_test.py @@ -88,6 +88,16 @@ class RollTest(test_util.TensorFlowTestCase): x = np.random.rand(3, 2, 1, 1).astype(t) self._testAll(x + 1j * x, [2, 1, 1, 0], [0, 3, 1, 2]) + def testNegativeAxis(self): + self._testAll(np.random.randint(-100, 100, (5)).astype(np.int32), 3, -1) + self._testAll(np.random.randint(-100, 100, (4, 4)).astype(np.int32), 3, -2) + # Make sure negative axis shoudl be 0 <= axis + dims < dims + with self.test_session(): + with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, + "is out of range"): + manip_ops.roll(np.random.randint(-100, 100, (4, 4)).astype(np.int32), + 3, -10).eval() + def testRollInputMustVectorHigherRaises(self): tensor = 7 shift = 1 -- GitLab From 7b0b7bbe9519a5dee55d9e83d681411495aad45a Mon Sep 17 00:00:00 2001 From: Mahmoud Abuzaina Date: Wed, 11 Apr 2018 18:04:26 -0700 Subject: [PATCH 206/791] Fixing non-mkl builds (#18401) --- tensorflow/core/kernels/BUILD | 36 ++++++++++++----------------------- 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 1018e8d25c..2bbedfff73 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -5943,8 +5943,7 @@ tf_mkl_kernel_library( "//tensorflow/core:lib_internal", "//tensorflow/core:nn_ops_op_lib", "//third_party/mkl:intel_binary_blob", - "@mkl_dnn", - ], + ] + if_mkl(["@mkl_dnn"]), ) tf_mkl_kernel_library( @@ -5959,8 +5958,7 @@ tf_mkl_kernel_library( "//tensorflow/core:lib_internal", "//tensorflow/core:nn_ops_op_lib", "//third_party/mkl:intel_binary_blob", - "@mkl_dnn", - ], + ] + if_mkl(["@mkl_dnn"]), ) tf_mkl_kernel_library( @@ -5976,8 +5974,7 @@ tf_mkl_kernel_library( "//tensorflow/core:lib_internal", "//tensorflow/core:nn_ops_op_lib", "//third_party/mkl:intel_binary_blob", - "@mkl_dnn", - ], + ] + if_mkl(["@mkl_dnn"]), ) tf_mkl_kernel_library( @@ -5997,8 +5994,7 @@ tf_mkl_kernel_library( "//tensorflow/core:lib_internal", "//tensorflow/core:nn_ops_op_lib", "//third_party/mkl:intel_binary_blob", - "@mkl_dnn", - ], + ] + if_mkl(["@mkl_dnn"]), ) tf_mkl_kernel_library( @@ -6014,8 +6010,7 @@ tf_mkl_kernel_library( "//tensorflow/core:nn_ops_op_lib", "//third_party/eigen3", "//third_party/mkl:intel_binary_blob", - "@mkl_dnn", - ], + ] + if_mkl(["@mkl_dnn"]), ) tf_mkl_kernel_library( @@ -6031,8 +6026,7 @@ tf_mkl_kernel_library( "//tensorflow/core:nn_ops_op_lib", "//third_party/eigen3", "//third_party/mkl:intel_binary_blob", - "@mkl_dnn", - ], + ] + if_mkl(["@mkl_dnn"]), ) tf_mkl_kernel_library( @@ -6040,8 +6034,7 @@ tf_mkl_kernel_library( srcs = ["mkl_fused_batch_norm_op.cc"], deps = NN_DEPS + [ "//third_party/mkl:intel_binary_blob", - "@mkl_dnn", - ], + ] + if_mkl(["@mkl_dnn"]), ) tf_mkl_kernel_library( @@ -6049,8 +6042,7 @@ tf_mkl_kernel_library( prefix = "mkl_aggregate_ops", deps = MATH_DEPS + [ "//third_party/mkl:intel_binary_blob", - "@mkl_dnn", - ], + ] + if_mkl(["@mkl_dnn"]), ) tf_mkl_kernel_library( @@ -6058,8 +6050,7 @@ tf_mkl_kernel_library( prefix = "mkl_concat_op", deps = ARRAY_DEPS + [ "//third_party/mkl:intel_binary_blob", - "@mkl_dnn", - ], + ] + if_mkl(["@mkl_dnn"]), ) tf_mkl_kernel_library( @@ -6067,8 +6058,7 @@ tf_mkl_kernel_library( prefix = "mkl_reshape_op", deps = ARRAY_DEPS + [ "//third_party/mkl:intel_binary_blob", - "@mkl_dnn", - ], + ] + if_mkl(["@mkl_dnn"]), ) tf_mkl_kernel_library( @@ -6076,8 +6066,7 @@ tf_mkl_kernel_library( prefix = "mkl_identity_op", deps = ARRAY_DEPS + [ "//third_party/mkl:intel_binary_blob", - "@mkl_dnn", - ], + ] + if_mkl(["@mkl_dnn"]), ) tf_mkl_kernel_library( @@ -6085,8 +6074,7 @@ tf_mkl_kernel_library( prefix = "mkl_lrn_op", deps = NN_DEPS + [ "//third_party/mkl:intel_binary_blob", - "@mkl_dnn", - ], + ] + if_mkl(["@mkl_dnn"]), ) tf_mkl_kernel_library( -- GitLab From d68ceefaba6972221bc6b3f86a76c4d07565fbdb Mon Sep 17 00:00:00 2001 From: Vadim Markovtsev Date: Thu, 12 Apr 2018 03:08:00 +0200 Subject: [PATCH 207/791] Replace print with logging (#18392) --- tensorflow/python/framework/graph_util_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/framework/graph_util_impl.py b/tensorflow/python/framework/graph_util_impl.py index 910364364c..394fac6c85 100644 --- a/tensorflow/python/framework/graph_util_impl.py +++ b/tensorflow/python/framework/graph_util_impl.py @@ -285,7 +285,7 @@ def convert_variables_to_constants(sess, output_graph_def.node.extend([output_node]) output_graph_def.library.CopyFrom(inference_graph.library) - print("Converted %d variables to const ops." % how_many_converted) + logging.info("Converted %d variables to const ops.", how_many_converted) return output_graph_def -- GitLab From d1ee6aa01090614ea53bc88ddf5edc1d44215a72 Mon Sep 17 00:00:00 2001 From: ngc92 <7938269+ngc92@users.noreply.github.com> Date: Thu, 12 Apr 2018 03:08:42 +0200 Subject: [PATCH 208/791] unified flip_* and random_flip_* functions (#18364) --- tensorflow/python/ops/image_ops_impl.py | 74 ++++++++++++++++--------- 1 file changed, 48 insertions(+), 26 deletions(-) diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py index 3369fe3c9b..601010bce9 100644 --- a/tensorflow/python/ops/image_ops_impl.py +++ b/tensorflow/python/ops/image_ops_impl.py @@ -269,17 +269,7 @@ def random_flip_up_down(image, seed=None): Raises: ValueError: if the shape of `image` not supported. """ - with ops.name_scope(None, 'random_flip_up_down', [image]) as scope: - image = ops.convert_to_tensor(image, name='image') - image = _Assert3DImage(image) - uniform_random = random_ops.random_uniform([], 0, 1.0, seed=seed) - mirror_cond = math_ops.less(uniform_random, .5) - result = control_flow_ops.cond( - mirror_cond, - lambda: array_ops.reverse(image, [0]), - lambda: image, - name=scope) - return fix_image_flip_shape(image, result) + return _random_flip(image, 0, seed, 'random_flip_up_down') @tf_export('image.random_flip_left_right') @@ -301,14 +291,34 @@ def random_flip_left_right(image, seed=None): Raises: ValueError: if the shape of `image` not supported. """ - with ops.name_scope(None, 'random_flip_left_right', [image]) as scope: + return _random_flip(image, 1, seed, 'random_flip_left_right') + + +def _random_flip(image, flip_index, seed, scope_name): + """Randomly (50% chance) flip an image along axis `flip_index`. + Args: + image: A 3-D tensor of shape `[height, width, channels].` + flip_index: The dimension along which to flip the image. + Vertical: 0, Horizontal: 1 + seed: A Python integer. Used to create a random seed. See + @{tf.set_random_seed} + for behavior. + scope_name: Name of the scope in which the ops are added. + + Returns: + A 3-D tensor of the same type and shape as `image`. + + Raises: + ValueError: if the shape of `image` not supported. + """ + with ops.name_scope(None, scope_name, [image]) as scope: image = ops.convert_to_tensor(image, name='image') image = _Assert3DImage(image) uniform_random = random_ops.random_uniform([], 0, 1.0, seed=seed) mirror_cond = math_ops.less(uniform_random, .5) result = control_flow_ops.cond( mirror_cond, - lambda: array_ops.reverse(image, [1]), + lambda: array_ops.reverse(image, [flip_index]), lambda: image, name=scope) return fix_image_flip_shape(image, result) @@ -332,16 +342,7 @@ def flip_left_right(image): Raises: ValueError: if the shape of `image` not supported. """ - with ops.name_scope(None, 'flip_left_right', [image]): - image = ops.convert_to_tensor(image, name='image') - image = _AssertAtLeast3DImage(image) - shape = image.get_shape() - if shape.ndims == 3 or shape.ndims is None: - return fix_image_flip_shape(image, array_ops.reverse(image, [1])) - elif shape.ndims == 4: - return array_ops.reverse(image, [2]) - else: - raise ValueError('\'image\' must have either 3 or 4 dimensions.') + return _flip(image, 1, 'flip_left_right') @tf_export('image.flip_up_down') @@ -362,14 +363,35 @@ def flip_up_down(image): Raises: ValueError: if the shape of `image` not supported. """ - with ops.name_scope(None, 'flip_up_down', [image]): + return _flip(image, 0, 'flip_up_down') + + +def _flip(image, flip_index, scope_name): + """Flip an image either horizontally or vertically. + + Outputs the contents of `image` flipped along the dimension `flip_index`. + + See also `reverse()`. + + Args: + image: 4-D Tensor of shape `[batch, height, width, channels]` or + 3-D Tensor of shape `[height, width, channels]`. + flip_index: 0 For vertical, 1 for horizontal. + + Returns: + A tensor of the same type and shape as `image`. + + Raises: + ValueError: if the shape of `image` not supported. + """ + with ops.name_scope(None, scope_name, [image]): image = ops.convert_to_tensor(image, name='image') image = _AssertAtLeast3DImage(image) shape = image.get_shape() if shape.ndims == 3 or shape.ndims is None: - return fix_image_flip_shape(image, array_ops.reverse(image, [0])) + return fix_image_flip_shape(image, array_ops.reverse(image, [flip_index])) elif shape.ndims == 4: - return array_ops.reverse(image, [1]) + return array_ops.reverse(image, [flip_index+1]) else: raise ValueError('\'image\' must have either 3 or 4 dimensions.') -- GitLab From 85b5d2eeb2dd876cb70b4c053110552553ade44b Mon Sep 17 00:00:00 2001 From: ImSheridan Date: Thu, 12 Apr 2018 09:09:17 +0800 Subject: [PATCH 209/791] Fix broken links in /extend/language_bindings (#18346) --- tensorflow/docs_src/extend/language_bindings.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tensorflow/docs_src/extend/language_bindings.md b/tensorflow/docs_src/extend/language_bindings.md index b9fd72978d..9a968d365b 100644 --- a/tensorflow/docs_src/extend/language_bindings.md +++ b/tensorflow/docs_src/extend/language_bindings.md @@ -112,11 +112,11 @@ There are a few ways to get a list of the `OpDef`s for the registered ops: to interpret the `OpDef` messages. - The C++ function `OpRegistry::Global()->GetRegisteredOps()` returns the same list of all registered `OpDef`s (defined in - [`tensorflow/core/framework/op.h`]). This can be used to write the generator + [`tensorflow/core/framework/op.h`](https://www.tensorflow.org/code/tensorflow/core/framework/op.h)). This can be used to write the generator in C++ (particularly useful for languages that do not have protocol buffer support). - The ASCII-serialized version of that list is periodically checked in to - [`tensorflow/core/ops/ops.pbtxt`] by an automated process. + [`tensorflow/core/ops/ops.pbtxt`](https://www.tensorflow.org/code/tensorflow/core/ops/ops.pbtxt) by an automated process. The `OpDef` specifies the following: @@ -159,7 +159,7 @@ between the generated code and the `OpDef`s checked into the repository, but is useful for languages where code is expected to be generated ahead of time like `go get` for Go and `cargo ops` for Rust. At the other end of the spectrum, for some languages the code could be generated dynamically from -[`tensorflow/core/ops/ops.pbtxt`]. +[`tensorflow/core/ops/ops.pbtxt`](https://www.tensorflow.org/code/tensorflow/core/ops/ops.pbtxt). #### Handling Constants @@ -229,6 +229,3 @@ and "while") is not available in languages other than Python. This will be updated when the [C API] provides necessary support. [C API]: https://www.tensorflow.org/code/tensorflow/c/c_api.h -[`tensorflow/core/ops/ops.pbtxt`]: https://www.tensorflow.org/code/tensorflow/core/ops/ops.pbtxt -[`tensorflow/python/BUILD`]: https://www.tensorflow.org/code/tensorflow/python/BUILD -[`tensorflow/core/framework/op.h`]: https://www.tensorflow.org/code/tensorflow/core/framework/op.h -- GitLab From 91baf5056f02d235e2516b0c066c473ab77a8955 Mon Sep 17 00:00:00 2001 From: David Norman Date: Thu, 12 Apr 2018 02:09:31 +0100 Subject: [PATCH 210/791] Disable int64 test for backends which don't support it (#18344) --- tensorflow/compiler/tests/binary_ops_test.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tensorflow/compiler/tests/binary_ops_test.py b/tensorflow/compiler/tests/binary_ops_test.py index d1d7379c0a..1e4dd32916 100644 --- a/tensorflow/compiler/tests/binary_ops_test.py +++ b/tensorflow/compiler/tests/binary_ops_test.py @@ -360,11 +360,13 @@ class BinaryOpsTest(XLATestCase): np.array([2, -1], dtype=dtype), expected=np.array([[[[3, 1], [5, 3]]]], dtype=dtype)) - self._testBinary( - math_ops.add, - np.array([0xffffffff, 0xfffffffff, 1, 1], dtype=np.int64), - np.array([1, 1, 0xffffffff, 0xfffffffff], dtype=np.int64), - expected=np.array([1 << 32, 1 << 36, 1 << 32, 1 << 36], dtype=np.int64)) + if np.int64 in self.numeric_types: + self._testBinary( + math_ops.add, + np.array([0xffffffff, 0xfffffffff, 1, 1], dtype=np.int64), + np.array([1, 1, 0xffffffff, 0xfffffffff], dtype=np.int64), + expected=np.array([1 << 32, 1 << 36, 1 << 32, 1 << 36], + dtype=np.int64)) def testComplexOps(self): for dtype in self.complex_types: -- GitLab From 70d99359fcb9aa9efa955fab06227373c734728b Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Wed, 11 Apr 2018 18:09:42 -0700 Subject: [PATCH 211/791] Add `tf.contrib.stateless.stateless_multinomial()`. This is a starting point for Dataset-compatible weighted sampling across a list of datasets. PiperOrigin-RevId: 192540412 --- tensorflow/contrib/stateless/__init__.py | 2 + .../kernel_tests/stateless_random_ops_test.py | 46 ++++++ .../api_def_StatelessMultinomial.pbtxt | 30 ++++ tensorflow/core/kernels/BUILD | 1 + tensorflow/core/kernels/multinomial_op.cc | 131 +++++++++++++++--- .../core/kernels/stateless_random_ops.cc | 68 +++++---- .../core/kernels/stateless_random_ops.h | 34 +++++ tensorflow/core/ops/stateless_random_ops.cc | 28 +++- tensorflow/core/util/guarded_philox_random.cc | 8 ++ tensorflow/core/util/guarded_philox_random.h | 2 + 10 files changed, 296 insertions(+), 54 deletions(-) create mode 100644 tensorflow/core/api_def/base_api/api_def_StatelessMultinomial.pbtxt create mode 100644 tensorflow/core/kernels/stateless_random_ops.h diff --git a/tensorflow/contrib/stateless/__init__.py b/tensorflow/contrib/stateless/__init__.py index ca937546f5..0cca40f071 100644 --- a/tensorflow/contrib/stateless/__init__.py +++ b/tensorflow/contrib/stateless/__init__.py @@ -22,6 +22,7 @@ WARNING: These ops are in contrib, and are not stable. They should be consistent across multiple runs on the same hardware, but only for the same version of the code. +@@stateless_multinomial @@stateless_random_uniform @@stateless_random_normal @@stateless_truncated_normal @@ -37,6 +38,7 @@ from tensorflow.contrib.stateless.gen_stateless_random_ops import * from tensorflow.python.framework import ops from tensorflow.python.util.all_util import remove_undocumented +ops.NotDifferentiable("StatelessMultinomial") ops.NotDifferentiable("StatelessRandomNormal") ops.NotDifferentiable("StatelessRandomUniform") ops.NotDifferentiable("StatelessTruncatedNormal") diff --git a/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py b/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py index bea6341cfd..d724a5c014 100644 --- a/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py +++ b/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py @@ -96,6 +96,52 @@ class StatelessOpsTest(test.TestCase): for s1, v1 in values: self.assertEqual(s0 == s1, np.all(v0 == v1)) + def testMatchStatefulMultinomial(self): + # Stateless ops should be the same as stateful ops on the first call + # after seed scrambling. + key = 0x3ec8f720, 0x02461e29 + num_samples = 4 + for logits_dtype in np.float16, np.float32, np.float64: + for output_dtype in dtypes.int32, dtypes.int64: + for seed in (7, 17), (11, 5), (2, 3): + preseed = invert_philox(key, + (seed[0], 0, seed[1], 0)).astype(np.uint64) + preseed = preseed[::2] | preseed[1::2] << 32 + random_seed.set_random_seed(seed[0]) + with self.test_session(use_gpu=True): + for logits in ([[0.1, 0.25, 0.5, 0.15]], [[0.5, 0.5], [0.8, 0.2], + [0.25, 0.75]]): + logits_t = constant_op.constant(logits, dtype=logits_dtype) + stateful = random_ops.multinomial( + logits_t, + num_samples, + seed=seed[1], + output_dtype=output_dtype) + pure = stateless.stateless_multinomial( + logits_t, + num_samples, + seed=preseed, + output_dtype=output_dtype) + self.assertAllEqual(stateful.eval(), pure.eval()) + + def testDeterminismMultinomial(self): + # Stateless values should be equal iff the seeds are equal (roughly) + num_samples = 10 + with self.test_session(use_gpu=True): + for seed_type in [dtypes.int32, dtypes.int64]: + seed_t = array_ops.placeholder(seed_type, shape=[2]) + seeds = [(x, y) for x in range(5) for y in range(5)] * 3 + for logits in ([[0.1, 0.25, 0.5, 0.15]], [[0.5, 0.5], [0.8, 0.2], + [0.25, 0.75]]): + pure = stateless.stateless_multinomial( + logits, num_samples, seed=seed_t) + values = [ + (seed, pure.eval(feed_dict={seed_t: seed})) for seed in seeds + ] + for s0, v0 in values: + for s1, v1 in values: + self.assertEqual(s0 == s1, np.all(v0 == v1)) + if __name__ == '__main__': test.main() diff --git a/tensorflow/core/api_def/base_api/api_def_StatelessMultinomial.pbtxt b/tensorflow/core/api_def/base_api/api_def_StatelessMultinomial.pbtxt new file mode 100644 index 0000000000..c4e6c1fddd --- /dev/null +++ b/tensorflow/core/api_def/base_api/api_def_StatelessMultinomial.pbtxt @@ -0,0 +1,30 @@ +op { + graph_op_name: "StatelessMultinomial" + in_arg { + name: "logits" + description: < { } // namespace functor +namespace { + // Samples from a multinomial distribution. template class MultinomialOp : public OpKernel { public: - explicit MultinomialOp(OpKernelConstruction* context) : OpKernel(context) { - OP_REQUIRES_OK(context, generator_.Init(context)); - } - - void Compute(OpKernelContext* ctx) override { - const Tensor& logits_t = ctx->input(0); - const Tensor& num_samples_t = ctx->input(1); + explicit MultinomialOp(OpKernelConstruction* context) : OpKernel(context) {} + void DoCompute(OpKernelContext* ctx, const Tensor& logits_t, + const Tensor& num_samples_t, GuardedPhiloxRandom* generator) { OP_REQUIRES(ctx, TensorShapeUtils::IsMatrix(logits_t.shape()), errors::InvalidArgument("logits should be a matrix, got shape ", logits_t.shape().DebugString())); @@ -194,7 +193,7 @@ class MultinomialOp : public OpKernel { // CPU generates doubles = 2 samples per number. if (std::is_same::value) num_samples_ceil_4 *= 2; auto rng = - generator_.ReserveRandomOutputs(batch_size * num_samples_ceil_4, 256); + generator->ReserveRandomOutputs(batch_size * num_samples_ceil_4, 256); functor::MultinomialFunctor()( ctx, ctx->eigen_device(), logits_t.matrix(), noises.flat(), scores.flat(), scratch.flat(), @@ -202,24 +201,38 @@ class MultinomialOp : public OpKernel { samples_t->matrix()); } } +}; + +template +class StatefulMultinomialOp : public MultinomialOp { + public: + explicit StatefulMultinomialOp(OpKernelConstruction* ctx) + : MultinomialOp(ctx) { + OP_REQUIRES_OK(ctx, generator_.Init(ctx)); + } + + void Compute(OpKernelContext* ctx) override { + const Tensor& logits_t = ctx->input(0); + const Tensor& num_samples_t = ctx->input(1); + this->DoCompute(ctx, logits_t, num_samples_t, &generator_); + } private: GuardedPhiloxRandom generator_; - - TF_DISALLOW_COPY_AND_ASSIGN(MultinomialOp); }; -#define REGISTER(TYPE) \ - REGISTER_KERNEL_BUILDER(Name("Multinomial") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("output_dtype", DT_INT32), \ - MultinomialOp); \ - REGISTER_KERNEL_BUILDER(Name("Multinomial") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("output_dtype", DT_INT64), \ - MultinomialOp); +// TODO(b/77906027): Add a TPU implementation. +#define REGISTER(TYPE) \ + REGISTER_KERNEL_BUILDER(Name("Multinomial") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("output_dtype", DT_INT32), \ + StatefulMultinomialOp); \ + REGISTER_KERNEL_BUILDER(Name("Multinomial") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("output_dtype", DT_INT64), \ + StatefulMultinomialOp); TF_CALL_half(REGISTER); TF_CALL_float(REGISTER); @@ -233,13 +246,83 @@ TF_CALL_double(REGISTER); .HostMemory("num_samples") \ .TypeConstraint("T") \ .TypeConstraint("output_dtype", DT_INT32), \ - MultinomialOp) \ + StatefulMultinomialOp) \ REGISTER_KERNEL_BUILDER(Name("Multinomial") \ .Device(DEVICE_GPU) \ .HostMemory("num_samples") \ .TypeConstraint("T") \ .TypeConstraint("output_dtype", DT_INT64), \ - MultinomialOp) + StatefulMultinomialOp) + +TF_CALL_half(REGISTER); +TF_CALL_float(REGISTER); +TF_CALL_double(REGISTER); +#undef REGISTER + +#endif // GOOGLE_CUDA + +template +class StatelessMultinomialOp : public MultinomialOp { + public: + explicit StatelessMultinomialOp(OpKernelConstruction* ctx) + : MultinomialOp(ctx) {} + + void Compute(OpKernelContext* ctx) override { + const Tensor& logits_t = ctx->input(0); + const Tensor& num_samples_t = ctx->input(1); + + const Tensor& seed_t = ctx->input(2); + OP_REQUIRES(ctx, seed_t.dims() == 1 && seed_t.dim_size(0) == 2, + errors::InvalidArgument("seed must have shape [2], not ", + seed_t.shape().DebugString())); + + random::PhiloxRandom::Key key; + random::PhiloxRandom::ResultType counter; + OP_REQUIRES_OK(ctx, GenerateKey(seed_t, &key, &counter)); + + GuardedPhiloxRandom generator; + generator.Init(counter, key); + + this->DoCompute(ctx, logits_t, num_samples_t, &generator); + } + + private: + GuardedPhiloxRandom generator_; +}; + +#define REGISTER(TYPE) \ + REGISTER_KERNEL_BUILDER(Name("StatelessMultinomial") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("output_dtype", DT_INT32), \ + StatelessMultinomialOp); \ + REGISTER_KERNEL_BUILDER(Name("StatelessMultinomial") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("output_dtype", DT_INT64), \ + StatelessMultinomialOp); + +TF_CALL_half(REGISTER); +TF_CALL_float(REGISTER); +TF_CALL_double(REGISTER); +#undef REGISTER + +#if GOOGLE_CUDA +#define REGISTER(TYPE) \ + REGISTER_KERNEL_BUILDER(Name("StatelessMultinomial") \ + .Device(DEVICE_GPU) \ + .HostMemory("num_samples") \ + .HostMemory("seed") \ + .TypeConstraint("T") \ + .TypeConstraint("output_dtype", DT_INT32), \ + StatelessMultinomialOp) \ + REGISTER_KERNEL_BUILDER(Name("StatelessMultinomial") \ + .Device(DEVICE_GPU) \ + .HostMemory("num_samples") \ + .HostMemory("seed") \ + .TypeConstraint("T") \ + .TypeConstraint("output_dtype", DT_INT64), \ + StatelessMultinomialOp) TF_CALL_half(REGISTER); TF_CALL_float(REGISTER); @@ -248,4 +331,6 @@ TF_CALL_double(REGISTER); #endif // GOOGLE_CUDA +} // end namespace + } // end namespace tensorflow diff --git a/tensorflow/core/kernels/stateless_random_ops.cc b/tensorflow/core/kernels/stateless_random_ops.cc index 88fcf542fb..eab176c7fb 100644 --- a/tensorflow/core/kernels/stateless_random_ops.cc +++ b/tensorflow/core/kernels/stateless_random_ops.cc @@ -27,6 +27,41 @@ namespace tensorflow { using CPUDevice = Eigen::ThreadPoolDevice; using GPUDevice = Eigen::GpuDevice; +Status GenerateKey(Tensor seed, random::PhiloxRandom::Key* out_key, + random::PhiloxRandom::ResultType* out_counter) { + // Grab the two seeds + uint64 seed0; + uint64 seed1; + if (seed.dtype() == DT_INT32) { + const auto seed_vals = seed.flat(); + seed0 = internal::SubtleMustCopy(seed_vals(0)); + seed1 = internal::SubtleMustCopy(seed_vals(1)); + } else if (seed.dtype() == DT_INT64) { + const auto seed_vals = seed.flat(); + seed0 = internal::SubtleMustCopy(seed_vals(0)); + seed1 = internal::SubtleMustCopy(seed_vals(1)); + } else { + return errors::InvalidArgument("Invalid seed type: ", + DataTypeString(seed.dtype())); + } + + // Scramble the seeds so that the user doesn't need to worry about which + // part of the seed needs to be strong. + (*out_key)[0] = 0x3ec8f720; + (*out_key)[1] = 0x02461e29; + (*out_counter)[0] = static_cast(seed0); + (*out_counter)[1] = static_cast(seed0 >> 32); + (*out_counter)[2] = static_cast(seed1); + (*out_counter)[3] = static_cast(seed1 >> 32); + const auto mix = random::PhiloxRandom(*out_counter, *out_key)(); + (*out_key)[0] = mix[0]; + (*out_key)[1] = mix[1]; + (*out_counter)[0] = (*out_counter)[1] = 0; + (*out_counter)[2] = mix[2]; + (*out_counter)[3] = mix[3]; + return Status::OK(); +} + namespace { class StatelessRandomOpBase : public OpKernel { @@ -49,36 +84,9 @@ class StatelessRandomOpBase : public OpKernel { OP_REQUIRES_OK(context, context->allocate_output(0, shape, &output)); if (shape.num_elements() == 0) return; - // Grab the two seeds - uint64 seed0; - uint64 seed1; - if (context->input_dtype(1) == DT_INT32) { - const auto seed = seed_t.flat(); - seed0 = internal::SubtleMustCopy(seed(0)); - seed1 = internal::SubtleMustCopy(seed(1)); - } else { - CHECK_EQ(DT_INT64, context->input_dtype(1)); - const auto seed = seed_t.flat(); - seed0 = internal::SubtleMustCopy(seed(0)); - seed1 = internal::SubtleMustCopy(seed(1)); - } - - // Scramble the seeds so that the user doesn't need to worry about which - // part of the seed needs to be strong. random::PhiloxRandom::Key key; random::PhiloxRandom::ResultType counter; - key[0] = 0x3ec8f720; - key[1] = 0x02461e29; - counter[0] = static_cast(seed0); - counter[1] = static_cast(seed0 >> 32); - counter[2] = static_cast(seed1); - counter[3] = static_cast(seed1 >> 32); - const auto mix = random::PhiloxRandom(counter, key)(); - key[0] = mix[0]; - key[1] = mix[1]; - counter[0] = counter[1] = 0; - counter[2] = mix[2]; - counter[3] = mix[3]; + OP_REQUIRES_OK(context, GenerateKey(seed_t, &key, &counter)); // Fill in the random numbers Fill(context, random::PhiloxRandom(counter, key), output); @@ -105,8 +113,6 @@ class StatelessRandomOp : public StatelessRandomOpBase { } }; -} // namespace - #define REGISTER(TYPE) \ REGISTER_KERNEL_BUILDER( \ Name("StatelessRandomUniform") \ @@ -176,4 +182,6 @@ TF_CALL_double(REGISTER); #endif // GOOGLE_CUDA +} // namespace + } // namespace tensorflow diff --git a/tensorflow/core/kernels/stateless_random_ops.h b/tensorflow/core/kernels/stateless_random_ops.h new file mode 100644 index 0000000000..bcd29c4873 --- /dev/null +++ b/tensorflow/core/kernels/stateless_random_ops.h @@ -0,0 +1,34 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_CORE_KERNELS_STATELESS_RANDOM_OPS_H_ +#define TENSORFLOW_CORE_KERNELS_STATELESS_RANDOM_OPS_H_ + +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/lib/random/random_distributions.h" + +namespace tensorflow { + +// Generates a key and counter that can be used to seed a PhiloxRandom, +// generator, based on the seed value in `seed_t`. +// +// REQUIRES: `seed_t` must be a length-2 vector of type DT_INT{32,64}. +// `out_key` and `out_counter` must be non-null. +Status GenerateKey(Tensor seed_t, random::PhiloxRandom::Key* out_key, + random::PhiloxRandom::ResultType* out_counter); + +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_KERNELS_STATELESS_RANDOM_OPS_H_ diff --git a/tensorflow/core/ops/stateless_random_ops.cc b/tensorflow/core/ops/stateless_random_ops.cc index 553850610a..742709fb18 100644 --- a/tensorflow/core/ops/stateless_random_ops.cc +++ b/tensorflow/core/ops/stateless_random_ops.cc @@ -29,7 +29,7 @@ static Status StatelessShape(shape_inference::InferenceContext* context) { TF_RETURN_IF_ERROR(context->WithValue(context->Dim(seed, 0), 2, &unused)); // Set output shape - shape_inference::ShapeHandle out; + ShapeHandle out; TF_RETURN_IF_ERROR(context->MakeShapeFromShapeTensor(0, &out)); context->set_output(0, out); return Status::OK(); @@ -54,6 +54,32 @@ REGISTER_STATELESS_OP("StatelessRandomNormal"); // This op is exposed through contrib/stateless only. The interface may change. REGISTER_STATELESS_OP("StatelessTruncatedNormal"); +// This op is exposed through contrib/stateless only. The interface may change. +REGISTER_OP("StatelessMultinomial") + .Input("logits: T") + .Input("num_samples: int32") + .Input("seed: Tseed") + .Output("output: output_dtype") + .Attr("T: realnumbertype") + .Attr("Tseed: {int32, int64} = DT_INT64") + .Attr("output_dtype: {int32, int64} = DT_INT64") + .SetShapeFn([](shape_inference::InferenceContext* c) { + // Check seed shape + ShapeHandle seed; + TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 1, &seed)); + DimensionHandle unused_dim; + TF_RETURN_IF_ERROR(c->WithValue(c->Dim(seed, 0), 2, &unused_dim)); + + ShapeHandle logits_shape; + ShapeHandle unused; + DimensionHandle num_samples; + TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 2, &logits_shape)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); + TF_RETURN_IF_ERROR(c->MakeDimForScalarInput(1, &num_samples)); + c->set_output(0, c->Matrix(c->Dim(logits_shape, 0), num_samples)); + return Status::OK(); + }); + #undef REGISTER_STATELESS_OP } // namespace tensorflow diff --git a/tensorflow/core/util/guarded_philox_random.cc b/tensorflow/core/util/guarded_philox_random.cc index 2d1e9a293e..7c7ba4cef6 100644 --- a/tensorflow/core/util/guarded_philox_random.cc +++ b/tensorflow/core/util/guarded_philox_random.cc @@ -43,6 +43,14 @@ void GuardedPhiloxRandom::Init(int64 seed, int64 seed2) { initialized_ = true; } +void GuardedPhiloxRandom::Init(random::PhiloxRandom::ResultType counter, + random::PhiloxRandom::Key key) { + CHECK(!initialized_); + mutex_lock lock(mu_); + generator_ = random::PhiloxRandom(counter, key); + initialized_ = true; +} + random::PhiloxRandom GuardedPhiloxRandom::ReserveSamples128(int64 samples) { CHECK(initialized_); mutex_lock lock(mu_); diff --git a/tensorflow/core/util/guarded_philox_random.h b/tensorflow/core/util/guarded_philox_random.h index 5b94a76777..44970eb949 100644 --- a/tensorflow/core/util/guarded_philox_random.h +++ b/tensorflow/core/util/guarded_philox_random.h @@ -49,6 +49,8 @@ class GuardedPhiloxRandom { // Initialize with given seeds. void Init(int64 seed, int64 seed2); + void Init(random::PhiloxRandom::ResultType counter, + random::PhiloxRandom::Key key); // Reserve a certain number of 128-bit samples. // This function is thread safe. The returned generator is valid for the -- GitLab From 7f39b18febda4513eb9b869396bad3ac9e8f64a8 Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Wed, 11 Apr 2018 21:18:01 -0400 Subject: [PATCH 212/791] Fix typo in error message (#18319) --- tensorflow/python/estimator/estimator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py index 4d3eff71ad..301a360636 100644 --- a/tensorflow/python/estimator/estimator.py +++ b/tensorflow/python/estimator/estimator.py @@ -723,7 +723,7 @@ class Estimator(object): batch_length = batch_length or value.shape[0] if value.shape[0] != batch_length: raise ValueError('Batch length of predictions should be same. %s has ' - 'different batch length then others.' % key) + 'different batch length than others.' % key) return batch_length def _extract_keys(self, predictions, predict_keys): -- GitLab From 41308f454f39d4a5fe5e87b97045d9867a5e7ac2 Mon Sep 17 00:00:00 2001 From: jinghuangintel Date: Wed, 11 Apr 2018 18:20:55 -0700 Subject: [PATCH 213/791] added missing shapefn to several operators (#18298) --- tensorflow/core/ops/nn_ops.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc index 12d6dc5eaf..18165fb6ed 100644 --- a/tensorflow/core/ops/nn_ops.cc +++ b/tensorflow/core/ops/nn_ops.cc @@ -1533,6 +1533,7 @@ REGISTER_OP("__MklDummyConv2DWithBias") .Attr(GetPaddingAttrString()) .Attr(GetConvnetDataFormatAttrString()) .Attr("dilations: list(int) = [1, 1, 1, 1]") + .SetShapeFn(shape_inference::Conv2DShape) .Doc(R"doc( Dummy node that enables fusing Conv2D and BiasAdd operator for MKL. This node does not perform anything. It is just created as an intermediate output of @@ -1559,6 +1560,7 @@ REGISTER_OP("_MklConv2DWithBias") .Attr(GetPaddingAttrString()) .Attr(GetConvnetDataFormatAttrString()) .Attr("dilations: list(int) = [1, 1, 1, 1]") + .SetShapeFn(shape_inference::Conv2DShape) .Doc(R"doc( MKL version of Conv2D and BiasAdd operator. Uses MKL DNN APIs to perform 2D convolution and add Bias to the output of convolution. @@ -1681,6 +1683,7 @@ NOTE Do not invoke this operator directly in Python. Graph rewrite pass is expected to invoke these operators. )doc"); +#ifdef INTEL_MKL_ML REGISTER_OP("_MklConv2DWithBiasBackpropBias") .Input("out_backprop: T") .Input("mkl_out_backprop: uint8") @@ -1697,6 +1700,7 @@ gradients of convolution with respect to the bias. NOTE Do not invoke this operator directly in Python. Graph rewrite pass is expected to invoke these operators. )doc"); +#endif REGISTER_OP("_MklConv2DBackpropInput") .Input("input_sizes: int32") @@ -2154,6 +2158,7 @@ REGISTER_OP("_MklToTf") .Output("output: T") .Attr("T: {half, float, double}") .Attr(GetConvnetDataFormatAttrString()) + .SetShapeFn(shape_inference::UnknownShape) .Doc(R"doc( MKL operator to convert a tensor from MKL layout to TensorFlow layout. @@ -2175,6 +2180,7 @@ REGISTER_OP("_MklInputConversion") "T: {half, float, double, uint8, int8, uint16, int16, int32, int64, " "complex64, complex128}") .Attr(GetConvnetDataFormatAttrString()) + .SetShapeFn(shape_inference::UnknownShape) .Doc(R"doc( MKL operator to process the inputs to an elementwise MKL op. Both inputs need to be either in TF or in MKL format. This op is added before every -- GitLab From 58029d1d0b13dbe91db12cb130303bfaaf566d8a Mon Sep 17 00:00:00 2001 From: Yifei Feng Date: Wed, 11 Apr 2018 18:20:19 -0700 Subject: [PATCH 214/791] In model_to_estimator, only run get_weights when there are initialized Keras variables(which assumes there exists a session). Otherwise create a session so that we can run get_config(). Actually fix #18193. PiperOrigin-RevId: 192541442 --- .../python/keras/_impl/keras/estimator.py | 45 +++++++++----- .../keras/_impl/keras/estimator_test.py | 61 ++++++++++--------- 2 files changed, 63 insertions(+), 43 deletions(-) diff --git a/tensorflow/python/keras/_impl/keras/estimator.py b/tensorflow/python/keras/_impl/keras/estimator.py index 8043242b70..b922a6c683 100644 --- a/tensorflow/python/keras/_impl/keras/estimator.py +++ b/tensorflow/python/keras/_impl/keras/estimator.py @@ -26,7 +26,6 @@ from tensorflow.python.estimator import estimator as estimator_lib from tensorflow.python.estimator import export as export_lib from tensorflow.python.estimator import model_fn as model_fn_lib from tensorflow.python.estimator import run_config as run_config_lib -from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import random_seed from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib @@ -38,6 +37,7 @@ from tensorflow.python.keras._impl.keras.engine.network import Network from tensorflow.python.keras._impl.keras.utils.generic_utils import CustomObjectScope from tensorflow.python.ops import math_ops from tensorflow.python.ops import metrics as metrics_module +from tensorflow.python.ops import variables as variables_module from tensorflow.python.platform import tf_logging as logging from tensorflow.python.saved_model import signature_constants from tensorflow.python.training import saver as saver_lib @@ -55,6 +55,19 @@ def _cast_tensor_to_floatx(x): return math_ops.cast(x, K.floatx()) +def _any_variable_initalized(): + """Check if any variable has been initialized in the Keras model. + + Returns: + boolean, True if at least one variable has been initalized, else False. + """ + variables = variables_module.global_variables() + for v in variables: + if getattr(v, '_keras_initialized', False): + return True + return False + + def _create_ordered_io(keras_model, estimator_io, is_input=True): """Create a list of tensors from IO dictionary based on Keras IO order. @@ -396,7 +409,8 @@ def _save_first_checkpoint(keras_model, estimator, custom_objects, custom_objects) # save to checkpoint with session.Session(config=estimator._session_config) as sess: - model.set_weights(keras_weights) + if keras_weights: + model.set_weights(keras_weights) # Make update ops and initialize all variables. if not model.train_function: # pylint: disable=protected-access @@ -466,20 +480,21 @@ def model_to_estimator(keras_model=None, estimator = estimator_lib.Estimator( keras_model_fn, model_dir=model_dir, config=config) - old_session = K._SESSION - # Pass the config into keras backend's default session. - sess = session.Session(config=estimator._session_config) - K.set_session(sess) - try: - keras_weights = keras_model.get_weights() - except errors.FailedPreconditionError as e: - if old_session is None: - raise e - logging.warning( - 'The Keras backend session has already been ' - 'set. The _session_config passed to model_to_estimator is not used.') - K.set_session(old_session) + # Check if we need to call get_weights: + if _any_variable_initalized(): keras_weights = keras_model.get_weights() + # Warn if config passed to estimator tries to update GPUOptions. If a + # session has already been created, the GPUOptions passed to the first + # session sticks. + if estimator._session_config.HasField('gpu_options'): + logging.warning( + 'The Keras backend session has already been set. ' + 'The _session_config passed to model_to_estimator will not be used.') + else: + # Pass the config into keras backend's default session. + sess = session.Session(config=estimator._session_config) + K.set_session(sess) + keras_weights = None if keras_model._is_graph_network: # TODO(yifeif): move checkpoint initialization to scaffold.init_fn diff --git a/tensorflow/python/keras/_impl/keras/estimator_test.py b/tensorflow/python/keras/_impl/keras/estimator_test.py index 27b7ec7dd4..653cdc01e2 100644 --- a/tensorflow/python/keras/_impl/keras/estimator_test.py +++ b/tensorflow/python/keras/_impl/keras/estimator_test.py @@ -27,10 +27,12 @@ import numpy as np from tensorflow.core.protobuf import config_pb2 from tensorflow.python.estimator import run_config as run_config_lib from tensorflow.python.estimator.inputs import numpy_io +from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.keras._impl import keras from tensorflow.python.keras._impl.keras import testing_utils from tensorflow.python.keras._impl.keras.applications import mobilenet +from tensorflow.python.keras._impl.keras.optimizers import SGD from tensorflow.python.platform import gfile from tensorflow.python.platform import test from tensorflow.python.summary.writer import writer_cache @@ -443,8 +445,9 @@ class TestKerasEstimator(test_util.TensorFlowTestCase): model = simple_functional_model() model.compile( loss='categorical_crossentropy', optimizer='adam', metrics=['acc']) - est_keras = keras.estimator.model_to_estimator( - keras_model=model, config=self._config) + with self.test_session(): + est_keras = keras.estimator.model_to_estimator( + keras_model=model, config=self._config) with self.test_session(): with self.assertRaises(ValueError): @@ -497,20 +500,22 @@ class TestKerasEstimator(test_util.TensorFlowTestCase): model_dir=tempfile.mkdtemp(dir=self._base_dir)) def test_gpu_config(self): - keras_model, (_, _), (_, _), _, _ = get_resource_for_simple_model() - keras_model.compile( - loss='categorical_crossentropy', - optimizer='rmsprop', - metrics=['mse', keras.metrics.categorical_accuracy]) + with ops.Graph().as_default(): + keras_model, (_, _), (_, _), _, _ = get_resource_for_simple_model() + keras_model.compile( + loss='categorical_crossentropy', + optimizer='rmsprop', + metrics=['mse', keras.metrics.categorical_accuracy]) - gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.3) - sess_config = config_pb2.ConfigProto(gpu_options=gpu_options) - self._config._session_config = sess_config - keras.estimator.model_to_estimator( - keras_model=keras_model, config=self._config) - self.assertEqual(keras.backend.get_session() - ._config.gpu_options.per_process_gpu_memory_fraction, - gpu_options.per_process_gpu_memory_fraction) + gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.3) + sess_config = config_pb2.ConfigProto(gpu_options=gpu_options) + self._config._session_config = sess_config + keras.estimator.model_to_estimator( + keras_model=keras_model, config=self._config) + self.assertEqual( + keras.backend.get_session() + ._config.gpu_options.per_process_gpu_memory_fraction, + gpu_options.per_process_gpu_memory_fraction) def test_pretrained_weights(self): keras_model, (_, _), (_, _), _, _ = get_resource_for_simple_model() @@ -518,19 +523,19 @@ class TestKerasEstimator(test_util.TensorFlowTestCase): loss='categorical_crossentropy', optimizer=rmsprop.RMSPropOptimizer(1e-3), metrics=['mse', keras.metrics.categorical_accuracy]) - - keras_model.train_on_batch( - np.random.random((10,) + _INPUT_SIZE), np.random.random((10, - _NUM_CLASS))) - weights = keras_model.get_weights() - keras_model, (_, _), (_, _), _, _ = get_resource_for_simple_model() - keras_model.set_weights(weights) - keras_model.compile( - loss='categorical_crossentropy', - optimizer=rmsprop.RMSPropOptimizer(1e-3), - metrics=['mse', keras.metrics.categorical_accuracy]) - keras.estimator.model_to_estimator( - keras_model=keras_model, config=self._config) + with self.test_session(): + keras_model.train_on_batch( + np.random.random((10,) + _INPUT_SIZE), + np.random.random((10, _NUM_CLASS))) + weights = keras_model.get_weights() + keras_model, (_, _), (_, _), _, _ = get_resource_for_simple_model() + keras_model.set_weights(weights) + keras_model.compile( + loss='categorical_crossentropy', + optimizer=SGD(lr=0.0001, momentum=0.9), + metrics=['mse', keras.metrics.categorical_accuracy]) + keras.estimator.model_to_estimator( + keras_model=keras_model, config=self._config) if __name__ == '__main__': -- GitLab From 9c2e04411ec1dbcf7aaf604dbc218489928bb2cc Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 11 Apr 2018 18:26:12 -0700 Subject: [PATCH 215/791] Check input dimension for contrib.layers.conv2d/conv3d (#18251) * Check input dimension for contrib.layers.conv2d/conv3d This fix tries to fix the issue raised in 14583 where the input dimension was not checked for contrib.layers.conv2d/conv3d and contrib.slim.conv2d/conv3d. The issue was that conv2d/conv3d were just aliases of convolution. This fix wrap the conv2d/conv3d with the input dimension check so that incorrect usage will return ValueError. This fix fixes 14583. Signed-off-by: Yong Tang * Add test case for conv2d/conv3d shape check Signed-off-by: Yong Tang * Fix impacted tests. Signed-off-by: Yong Tang * Update convolution instead of adding _convolution, based on review feedback Signed-off-by: Yong Tang * Add convolution1d and additional update Signed-off-by: Yong Tang --- .../contrib/layers/python/layers/layers.py | 138 +++++++++++++++++- .../layers/python/layers/layers_test.py | 15 +- 2 files changed, 148 insertions(+), 5 deletions(-) diff --git a/tensorflow/contrib/layers/python/layers/layers.py b/tensorflow/contrib/layers/python/layers/layers.py index 10d7f6d076..949e73deff 100644 --- a/tensorflow/contrib/layers/python/layers/layers.py +++ b/tensorflow/contrib/layers/python/layers/layers.py @@ -932,7 +932,8 @@ def convolution(inputs, variables_collections=None, outputs_collections=None, trainable=True, - scope=None): + scope=None, + conv_dims=None): """Adds an N-D convolution followed by an optional batch_norm layer. It is required that 1 <= N <= 3. @@ -993,6 +994,10 @@ def convolution(inputs, trainable: If `True` also add variables to the graph collection `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable). scope: Optional scope for `variable_scope`. + conv_dims: Optional convolution dimensionality, when set it would use the + corresponding convolution (e.g. 2 for Conv 2D, 3 for Conv 3D, ..). When + leaved to None it would select the convolution dimensionality based on + the input rank (i.e. Conv ND, with N = input_rank - 2). Returns: A tensor representing the output of the operation. @@ -1015,6 +1020,9 @@ def convolution(inputs, inputs = ops.convert_to_tensor(inputs) input_rank = inputs.get_shape().ndims + if conv_dims is not None and conv_dims + 2 != input_rank: + raise ValueError('Convolution expects input with rank %d, got %d' % + (conv_dims + 2, input_rank)) if input_rank == 3: layer_class = convolutional_layers.Convolution1D elif input_rank == 4: @@ -1061,10 +1069,134 @@ def convolution(inputs, outputs = activation_fn(outputs) return utils.collect_named_outputs(outputs_collections, sc.name, outputs) +@add_arg_scope +def convolution1d(inputs, + num_outputs, + kernel_size, + stride=1, + padding='SAME', + data_format=None, + rate=1, + activation_fn=nn.relu, + normalizer_fn=None, + normalizer_params=None, + weights_initializer=initializers.xavier_initializer(), + weights_regularizer=None, + biases_initializer=init_ops.zeros_initializer(), + biases_regularizer=None, + reuse=None, + variables_collections=None, + outputs_collections=None, + trainable=True, + scope=None): + return convolution(inputs, + num_outputs, + kernel_size, + stride, + padding, + data_format, + rate, + activation_fn, + normalizer_fn, + normalizer_params, + weights_initializer, + weights_regularizer, + biases_initializer, + biases_regularizer, + reuse, + variables_collections, + outputs_collections, + trainable, + scope, + conv_dims=1) + +convolution1d.__doc__ = convolution.__doc__ -convolution2d = convolution -convolution3d = convolution +@add_arg_scope +def convolution2d(inputs, + num_outputs, + kernel_size, + stride=1, + padding='SAME', + data_format=None, + rate=1, + activation_fn=nn.relu, + normalizer_fn=None, + normalizer_params=None, + weights_initializer=initializers.xavier_initializer(), + weights_regularizer=None, + biases_initializer=init_ops.zeros_initializer(), + biases_regularizer=None, + reuse=None, + variables_collections=None, + outputs_collections=None, + trainable=True, + scope=None): + return convolution(inputs, + num_outputs, + kernel_size, + stride, + padding, + data_format, + rate, + activation_fn, + normalizer_fn, + normalizer_params, + weights_initializer, + weights_regularizer, + biases_initializer, + biases_regularizer, + reuse, + variables_collections, + outputs_collections, + trainable, + scope, + conv_dims=2) + +convolution2d.__doc__ = convolution.__doc__ +@add_arg_scope +def convolution3d(inputs, + num_outputs, + kernel_size, + stride=1, + padding='SAME', + data_format=None, + rate=1, + activation_fn=nn.relu, + normalizer_fn=None, + normalizer_params=None, + weights_initializer=initializers.xavier_initializer(), + weights_regularizer=None, + biases_initializer=init_ops.zeros_initializer(), + biases_regularizer=None, + reuse=None, + variables_collections=None, + outputs_collections=None, + trainable=True, + scope=None): + return convolution(inputs, + num_outputs, + kernel_size, + stride, + padding, + data_format, + rate, + activation_fn, + normalizer_fn, + normalizer_params, + weights_initializer, + weights_regularizer, + biases_initializer, + biases_regularizer, + reuse, + variables_collections, + outputs_collections, + trainable, + scope, + conv_dims=3) + +convolution3d.__doc__ = convolution.__doc__ @add_arg_scope def convolution2d_in_plane( diff --git a/tensorflow/contrib/layers/python/layers/layers_test.py b/tensorflow/contrib/layers/python/layers/layers_test.py index 997f910a2a..b01fd5d5c9 100644 --- a/tensorflow/contrib/layers/python/layers/layers_test.py +++ b/tensorflow/contrib/layers/python/layers/layers_test.py @@ -310,6 +310,17 @@ class BiasAddTest(test.TestCase): class ConvolutionTest(test.TestCase): + def testInvalidShape(self): + with self.test_session(): + images_2d = random_ops.random_uniform((5, 7, 9, 3), seed=1) + with self.assertRaisesRegexp( + ValueError, 'Convolution expects input with rank 5, got 4'): + layers_lib.convolution3d(images_2d, 32, 3) + images_3d = random_ops.random_uniform((5, 6, 7, 9, 3), seed=1) + with self.assertRaisesRegexp( + ValueError, 'Convolution expects input with rank 4, got 5'): + layers_lib.convolution2d(images_3d, 32, 3) + def testInvalidDataFormat(self): height, width = 7, 9 with self.test_session(): @@ -3155,7 +3166,7 @@ class RepeatTests(test.TestCase): with self.test_session(): images = np.random.uniform(size=(5, height, width, 3)).astype(np.float32) output = _layers.repeat(images, 3, layers_lib.conv2d, 32, [3, 3]) - self.assertEqual(output.op.name, 'Repeat/convolution_3/Relu') + self.assertEqual(output.op.name, 'Repeat/convolution2d_3/Relu') self.assertListEqual(output.get_shape().as_list(), [5, 3, 3, 32]) def testRepeatWithScope(self): @@ -3749,7 +3760,7 @@ class StackTests(test.TestCase): layers_lib.convolution2d, [10, 20, 30], kernel_size=[3, 3], padding='SAME') - self.assertEqual(output.op.name, 'Stack/convolution_3/Relu') + self.assertEqual(output.op.name, 'Stack/convolution2d_3/Relu') self.assertListEqual(output.get_shape().as_list(), [5, 3, 3, 30]) def testStackWithScope(self): -- GitLab From a843ec33a8fe8feb41f3733d2bea34691bb02a1e Mon Sep 17 00:00:00 2001 From: brett koonce Date: Wed, 11 Apr 2018 18:27:16 -0700 Subject: [PATCH 216/791] contrib/image: minor spelling tweaks (#18162) --- .../contrib/image/kernels/adjust_hsv_in_yiq_op_gpu.cu.cc | 2 +- tensorflow/contrib/image/ops/distort_image_ops.cc | 4 ++-- tensorflow/contrib/image/ops/image_ops.cc | 2 +- .../image/ops/single_image_random_dot_stereograms_ops.cc | 4 ++-- tensorflow/contrib/image/python/ops/image_ops.py | 2 +- .../image/python/ops/single_image_random_dot_stereograms.py | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op_gpu.cu.cc b/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op_gpu.cu.cc index b71ff9cd50..645abbf0b0 100644 --- a/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op_gpu.cu.cc +++ b/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op_gpu.cu.cc @@ -53,7 +53,7 @@ void AdjustHsvInYiqGPU::operator()(OpKernelContext* ctx, int channel_count, OP_REQUIRES_OK(ctx, ctx->allocate_temp( DT_FLOAT, TensorShape({kChannelSize * kChannelSize}), &tranformation_matrix)); - // TODO(huangyp): It takes about 3.5 us to comute tranformation_matrix + // TODO(huangyp): It takes about 3.5 us to compute tranformation_matrix // with one thread. Improve its performance if necessary. internal::compute_tranformation_matrix_cuda<<<1, 1, 0, cu_stream>>>( delta_h, scale_s, scale_v, tranformation_matrix.flat().data(), diff --git a/tensorflow/contrib/image/ops/distort_image_ops.cc b/tensorflow/contrib/image/ops/distort_image_ops.cc index b169b0b2b2..ca49635d5d 100644 --- a/tensorflow/contrib/image/ops/distort_image_ops.cc +++ b/tensorflow/contrib/image/ops/distort_image_ops.cc @@ -36,9 +36,9 @@ REGISTER_OP("AdjustHsvInYiq") Adjust the YIQ hue of one or more images. `images` is a tensor of at least 3 dimensions. The last dimension is -interpretted as channels, and must be three. +interpreted as channels, and must be three. -We used linear transfomation described in: +We used linear transformation described in: beesbuzz.biz/code/hsv_color_transforms.php The input image is considered in the RGB colorspace. Conceptually, the RGB colors are first mapped into YIQ space, rotated around the Y channel by diff --git a/tensorflow/contrib/image/ops/image_ops.cc b/tensorflow/contrib/image/ops/image_ops.cc index 68771b3d05..ebdcaea7ab 100644 --- a/tensorflow/contrib/image/ops/image_ops.cc +++ b/tensorflow/contrib/image/ops/image_ops.cc @@ -93,7 +93,7 @@ row_to_col_match_indices: A vector of length num_rows, which is the number of If `row_to_col_match_indices[i]` is not -1, row i is matched to column `row_to_col_match_indices[i]`. col_to_row_match_indices: A vector of length num_columns, which is the number - of columns of the input ditance matrix. + of columns of the input distance matrix. If `col_to_row_match_indices[j]` is not -1, column j is matched to row `col_to_row_match_indices[j]`. )doc"); diff --git a/tensorflow/contrib/image/ops/single_image_random_dot_stereograms_ops.cc b/tensorflow/contrib/image/ops/single_image_random_dot_stereograms_ops.cc index 8139d4272d..bd784c6bda 100755 --- a/tensorflow/contrib/image/ops/single_image_random_dot_stereograms_ops.cc +++ b/tensorflow/contrib/image/ops/single_image_random_dot_stereograms_ops.cc @@ -69,7 +69,7 @@ Outputs a single image random dot stereogram for export via encode_PNG/JPG OP. Given the 2-D tensor 'depth_values' with encoded Z values, this operation will encode 3-D data into a 2-D image. The output of this Op is suitable for the encode_PNG/JPG ops. Be careful with image compression as this may corrupt the -encode 3-D data witin the image. +encode 3-D data within the image. This Op is based upon: 'http://www.learningace.com/doc/4331582/b6ab058d1e206d68ab60e4e1ead2fe6e/sirds-paper' @@ -111,7 +111,7 @@ output_image_shape: Output size of returned image in X,Y, Channels 1-grayscale, output_data_window: Size of "DATA" window, must be equal to or smaller than 'output_image_shape', will be centered and use 'convergence_dots_size' for best fit to avoid overlap if possible -image:= A tensor of size 'output_image_shape' with the encloded 'depth_values' +image:= A tensor of size 'output_image_shape' with the encoded 'depth_values' )doc"); } // namespace tensorflow diff --git a/tensorflow/contrib/image/python/ops/image_ops.py b/tensorflow/contrib/image/python/ops/image_ops.py index c139ae89d8..cd984c8054 100644 --- a/tensorflow/contrib/image/python/ops/image_ops.py +++ b/tensorflow/contrib/image/python/ops/image_ops.py @@ -433,7 +433,7 @@ def bipartite_match(distance_mat, of rows of the input `distance_matrix`. If `row_to_col_match_indices[i]` is not -1, row i is matched to column `row_to_col_match_indices[i]`. col_to_row_match_indices: A vector of length num_columns, which is the - number of columns of the input ditance matrix. + number of columns of the input distance matrix. If `col_to_row_match_indices[j]` is not -1, column j is matched to row `col_to_row_match_indices[j]`. """ diff --git a/tensorflow/contrib/image/python/ops/single_image_random_dot_stereograms.py b/tensorflow/contrib/image/python/ops/single_image_random_dot_stereograms.py index d4a6a5bcbb..0ceb683ff4 100755 --- a/tensorflow/contrib/image/python/ops/single_image_random_dot_stereograms.py +++ b/tensorflow/contrib/image/python/ops/single_image_random_dot_stereograms.py @@ -45,7 +45,7 @@ def single_image_random_dot_stereograms(depth_values, Given the 2-D tensor 'depth_values' with encoded Z values, this operation will encode 3-D data into a 2-D image. The output of this Op is suitable for the encode_PNG/JPG ops. Be careful with image compression as this may - corrupt the encode 3-D data witin the image. + corrupt the encode 3-D data within the image. Based upon [this paper](http://www.learningace.com/doc/4331582/b6ab058d1e206d68ab60e4e1ead2fe6e/sirds-paper). -- GitLab From fd934f119deba4543555c3dac2c8c75936ac12d0 Mon Sep 17 00:00:00 2001 From: tamimaddari82 <37008274+tamimaddari82@users.noreply.github.com> Date: Thu, 12 Apr 2018 09:28:18 +0800 Subject: [PATCH 217/791] Add parallel implementation of CTC greedy decoder (#17982) --- tensorflow/core/kernels/ctc_decoder_ops.cc | 34 ++++++++++++++-------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/tensorflow/core/kernels/ctc_decoder_ops.cc b/tensorflow/core/kernels/ctc_decoder_ops.cc index 96bdb6a241..8cadeac68d 100644 --- a/tensorflow/core/kernels/ctc_decoder_ops.cc +++ b/tensorflow/core/kernels/ctc_decoder_ops.cc @@ -27,6 +27,7 @@ limitations under the License. #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/util/ctc/ctc_beam_search.h" #include "tensorflow/core/util/sparse/sparse_tensor.h" +#include "tensorflow/core/util/work_sharder.h" namespace tensorflow { @@ -213,20 +214,29 @@ class CTCGreedyDecoderOp : public OpKernel { // Perform best path decoding std::vector > > sequences(batch_size); - for (int b = 0; b < batch_size; ++b) { - sequences[b].resize(1); - auto& sequence = sequences[b][0]; - int prev_indices = -1; - for (int t = 0; t < seq_len_t(b); ++t) { - int max_class_indices; - log_prob_t(b, 0) += -RowMax(input_list_t[t], b, &max_class_indices); - if (max_class_indices != blank_index && - !(merge_repeated_ && max_class_indices == prev_indices)) { - sequence.push_back(max_class_indices); + auto decode = [&](const int64 begin, const int64 end) { + for (int b = begin; b < end; ++b) { + sequences[b].resize(1); + auto &sequence = sequences[b][0]; + int prev_indices = -1; + for (int t = 0; t < seq_len_t(b); ++t) { + int max_class_indices; + log_prob_t(b, 0) += -RowMax(input_list_t[t], b, &max_class_indices); + if (max_class_indices != blank_index && + !(merge_repeated_ && max_class_indices == prev_indices)) { + sequence.push_back(max_class_indices); + } + prev_indices = max_class_indices; } - prev_indices = max_class_indices; } - } + }; + + const int64 kCostPerUnit = 50 * max_time * num_classes; + const int64 total = batch_size; + const DeviceBase::CpuWorkerThreads& worker_threads = + *ctx->device()->tensorflow_cpu_worker_threads(); + Shard(worker_threads.num_threads, worker_threads.workers, total, + kCostPerUnit, decode); OP_REQUIRES_OK( ctx, decode_helper_.StoreAllDecodedSequences( -- GitLab From de72c8cccef2ee77667c041b68a34be6fb61ea65 Mon Sep 17 00:00:00 2001 From: Michal Turek Date: Thu, 12 Apr 2018 03:32:10 +0200 Subject: [PATCH 218/791] Add comment to examples to prevent resource leaks (#17820) Issue #17374 --- tensorflow/docs_src/install/install_java.md | 2 ++ .../java/src/main/java/org/tensorflow/examples/LabelImage.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md index cdde45a6f4..0dcb059793 100644 --- a/tensorflow/docs_src/install/install_java.md +++ b/tensorflow/docs_src/install/install_java.md @@ -93,6 +93,7 @@ As an example, these steps will create a Maven project that uses TensorFlow: // Execute the "MyConst" operation in a Session. try (Session s = new Session(g); + // Generally, there may be multiple output tensors, all of them must be closed to prevent resource leaks. Tensor output = s.runner().fetch("MyConst").run().get(0)) { System.out.println(new String(output.bytesValue(), "UTF-8")); } @@ -207,6 +208,7 @@ public class HelloTF { // Execute the "MyConst" operation in a Session. try (Session s = new Session(g); + // Generally, there may be multiple output tensors, all of them must be closed to prevent resource leaks. Tensor output = s.runner().fetch("MyConst").run().get(0)) { System.out.println(new String(output.bytesValue(), "UTF-8")); } diff --git a/tensorflow/java/src/main/java/org/tensorflow/examples/LabelImage.java b/tensorflow/java/src/main/java/org/tensorflow/examples/LabelImage.java index 489e95c310..3948991c84 100644 --- a/tensorflow/java/src/main/java/org/tensorflow/examples/LabelImage.java +++ b/tensorflow/java/src/main/java/org/tensorflow/examples/LabelImage.java @@ -101,6 +101,7 @@ public class LabelImage { b.constant("mean", mean)), b.constant("scale", scale)); try (Session s = new Session(g)) { + // Generally, there may be multiple output tensors, all of them must be closed to prevent resource leaks. return s.runner().fetch(output.op().name()).run().get(0).expect(Float.class); } } @@ -110,6 +111,7 @@ public class LabelImage { try (Graph g = new Graph()) { g.importGraphDef(graphDef); try (Session s = new Session(g); + // Generally, there may be multiple output tensors, all of them must be closed to prevent resource leaks. Tensor result = s.runner().feed("input", image).fetch("output").run().get(0).expect(Float.class)) { final long[] rshape = result.shape(); -- GitLab From c98f8c59b924b87bebe991607a5fb7d3cb90c5ee Mon Sep 17 00:00:00 2001 From: "Seungwoo Choi (Biggie)" Date: Thu, 12 Apr 2018 10:33:41 +0900 Subject: [PATCH 219/791] Replace wrong variable (#17738) --- tensorflow/contrib/quantize/python/fold_batch_norms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/quantize/python/fold_batch_norms.py b/tensorflow/contrib/quantize/python/fold_batch_norms.py index 4a8f8a04cc..aa0ef64308 100644 --- a/tensorflow/contrib/quantize/python/fold_batch_norms.py +++ b/tensorflow/contrib/quantize/python/fold_batch_norms.py @@ -545,7 +545,7 @@ def _GetBatchNormParams(graph, context, has_scaling): gamma_tensor = graph.get_tensor_by_name(op.name + ':0') if not has_scaling: - gamma_tensor = array_ops.ones(batch_mean_tensor.shape) + gamma_tensor = array_ops.ones(moving_mean_tensor.shape) return _BatchNormMatch( layer_op=None, -- GitLab From b47ff5f95d42d5321864359bd559fec0c1d81a69 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 11 Apr 2018 18:34:20 -0700 Subject: [PATCH 220/791] Enhancement with deprecated_argument_lookup (#17527) * Enhancement with deprecated_argument_lookup The tf.losses.cosine_distance deprecated dim and switched to axis. This fix adds the enhancement of using deprecated_argument_lookup, which is used in all other arguments deprecations. Signed-off-by: Yong Tang * Add missing import Signed-off-by: Yong Tang --- tensorflow/python/ops/losses/losses_impl.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tensorflow/python/ops/losses/losses_impl.py b/tensorflow/python/ops/losses/losses_impl.py index 34ca1adc3e..19a8eaf22c 100644 --- a/tensorflow/python/ops/losses/losses_impl.py +++ b/tensorflow/python/ops/losses/losses_impl.py @@ -29,6 +29,7 @@ from tensorflow.python.ops import nn_ops from tensorflow.python.ops import weights_broadcast_ops from tensorflow.python.ops.losses import util from tensorflow.python.util.deprecation import deprecated_args +from tensorflow.python.util.deprecation import deprecated_argument_lookup from tensorflow.python.util.tf_export import tf_export @@ -306,11 +307,8 @@ def cosine_distance( ValueError: If `predictions` shape doesn't match `labels` shape, or `axis`, `labels`, `predictions` or `weights` is `None`. """ - if dim is not None: - if axis is not None: - raise ValueError("Cannot specify both 'axis' and 'dim'") - axis = dim - if axis is None and dim is None: + axis = deprecated_argument_lookup("axis", axis, "dim", dim) + if axis is None: raise ValueError("You must specify 'axis'.") if labels is None: raise ValueError("labels must not be None.") -- GitLab From ff6c11008213424b7a1dd77346f996be693b004a Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Wed, 11 Apr 2018 18:37:47 -0700 Subject: [PATCH 221/791] Increase size of //tensorflow/python/kernel_tests:linalg_ops_test to "medium". PiperOrigin-RevId: 192542956 --- tensorflow/python/kernel_tests/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index 5738e79b27..e504a9fd21 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -1607,7 +1607,7 @@ cuda_py_test( cuda_py_test( name = "linalg_ops_test", - size = "small", + size = "medium", srcs = ["linalg_ops_test.py"], additional_deps = [ "//third_party/py/numpy", -- GitLab From 1caeb2086e7e9d7e3cb85883f0af316cddcf1285 Mon Sep 17 00:00:00 2001 From: fo40225 Date: Thu, 12 Apr 2018 09:41:48 +0800 Subject: [PATCH 222/791] fix tf.GIT_VERSION always 'unknown' on windows cmake build (#16730) --- tensorflow/contrib/cmake/tf_core_framework.cmake | 2 +- tensorflow/tools/git/gen_git_source.py | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/tensorflow/contrib/cmake/tf_core_framework.cmake b/tensorflow/contrib/cmake/tf_core_framework.cmake index a1c320347f..bcfb4f0819 100644 --- a/tensorflow/contrib/cmake/tf_core_framework.cmake +++ b/tensorflow/contrib/cmake/tf_core_framework.cmake @@ -276,7 +276,7 @@ add_custom_command(OUTPUT __force_rebuild COMMAND ${CMAKE_COMMAND} -E echo) add_custom_command(OUTPUT ${VERSION_INFO_CC} COMMAND ${PYTHON_EXECUTABLE} ${tensorflow_source_dir}/tensorflow/tools/git/gen_git_source.py - --raw_generate ${VERSION_INFO_CC} + ARGS --raw_generate ${VERSION_INFO_CC} --source_dir ${tensorflow_source_dir} DEPENDS __force_rebuild) set(tf_version_srcs ${tensorflow_source_dir}/tensorflow/core/util/version_info.cc) diff --git a/tensorflow/tools/git/gen_git_source.py b/tensorflow/tools/git/gen_git_source.py index cbcdbf5b80..6a1f126131 100755 --- a/tensorflow/tools/git/gen_git_source.py +++ b/tensorflow/tools/git/gen_git_source.py @@ -238,7 +238,7 @@ def generate(arglist): write_version_info(dest_file, git_version) -def raw_generate(output_file): +def raw_generate(output_file, source_dir): """Simple generator used for cmake/make build systems. This does not create any symlinks. It requires the build system @@ -246,9 +246,10 @@ def raw_generate(output_file): Args: output_file: Output filename for the version info cc + source_dir: Base path of the source code """ - git_version = get_git_version(".") + git_version = get_git_version(source_dir) write_version_info(output_file, git_version) @@ -281,6 +282,11 @@ parser.add_argument( type=str, help="Generate version_info.cc (simpler version used for cmake/make)") +parser.add_argument( + "--source_dir", + type=str, + help="Base path of the source code (used for cmake/make)") + args = parser.parse_args() if args.configure is not None: @@ -290,7 +296,10 @@ if args.configure is not None: elif args.generate is not None: generate(args.generate) elif args.raw_generate is not None: - raw_generate(args.raw_generate) + source_path = "." + if args.source_dir is not None: + source_path = args.source_dir + raw_generate(args.raw_generate, source_path) else: raise RuntimeError("--configure or --generate or --raw_generate " "must be used") -- GitLab From 4e29ebd67cd4409cbdfa6510b06acd780166aa9d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 18:38:38 -0700 Subject: [PATCH 223/791] [XLA] Redesign: test sharding. Also set the sharding to the instruction when created from proto. PiperOrigin-RevId: 192543024 --- .../xla/client/xla_client/xla_builder.h | 31 +++++++++++++++++++ .../compiler/xla/service/hlo_instruction.cc | 6 ++++ 2 files changed, 37 insertions(+) diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.h b/tensorflow/compiler/xla/client/xla_client/xla_builder.h index 24e0be2ac1..e583b4fe48 100644 --- a/tensorflow/compiler/xla/client/xla_client/xla_builder.h +++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.h @@ -959,6 +959,37 @@ XlaOp XlaBuilder::ConstantR4FromArray4D(const Array4D& values) { return ConstantFromArray(values); } +// RAII-style object: sets the current sharding assignment in builder on +// construction, and sets back to the previous assignment on destruction. +// +// TODO(b/74197823): This is a part of a NOT YET ready refactor. +class XlaScopedShardingAssignment { + public: + XlaScopedShardingAssignment(xla::XlaBuilder* builder, + tensorflow::gtl::optional sharding) + : builder_(builder), prev_sharding_(builder->sharding()) { + SetSharding(sharding); + } + + XlaScopedShardingAssignment(const XlaScopedShardingAssignment&) = delete; + XlaScopedShardingAssignment& operator=(const XlaScopedShardingAssignment&) = + delete; + + ~XlaScopedShardingAssignment() { SetSharding(prev_sharding_); } + + private: + void SetSharding(const tensorflow::gtl::optional& sharding) { + if (sharding.has_value()) { + builder_->SetSharding(sharding.value()); + } else { + builder_->ClearSharding(); + } + } + + xla::XlaBuilder* const builder_; + tensorflow::gtl::optional prev_sharding_; +}; + } // namespace xla #endif // TENSORFLOW_COMPILER_XLA_CLIENT_XLA_CLIENT_XLA_BUILDER_H_ diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index a986bbd511..5d2d7a9727 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -159,6 +159,12 @@ StatusOr> HloInstruction::CreateFromProto( instruction->fft_length_.push_back(fft_len); } + if (proto.has_sharding()) { + TF_ASSIGN_OR_RETURN(const auto& sharding, + HloSharding::FromProto(proto.sharding())); + instruction->set_sharding(sharding); + } + if (proto.has_gather_dimension_numbers()) { instruction->gather_dimension_numbers_ = MakeUnique(proto.gather_dimension_numbers()); -- GitLab From 079539b2e7acb1813cbfcdd2ab39f7bb77bc0467 Mon Sep 17 00:00:00 2001 From: Yanbo Liang Date: Wed, 11 Apr 2018 18:42:50 -0700 Subject: [PATCH 224/791] Correct argument doc for BasicLSTMCell.call (#16554) * Correct argument doc for BasicLSTMCell.call * change self._num_units to num_units. --- tensorflow/python/ops/rnn_cell_impl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/ops/rnn_cell_impl.py b/tensorflow/python/ops/rnn_cell_impl.py index cbc2dcf419..54f4e0f240 100644 --- a/tensorflow/python/ops/rnn_cell_impl.py +++ b/tensorflow/python/ops/rnn_cell_impl.py @@ -599,9 +599,9 @@ class BasicLSTMCell(LayerRNNCell): Args: inputs: `2-D` tensor with shape `[batch_size, input_size]`. state: An `LSTMStateTuple` of state tensors, each shaped - `[batch_size, self.state_size]`, if `state_is_tuple` has been set to + `[batch_size, num_units]`, if `state_is_tuple` has been set to `True`. Otherwise, a `Tensor` shaped - `[batch_size, 2 * self.state_size]`. + `[batch_size, 2 * num_units]`. Returns: A pair containing the new hidden state, and the new state (either a -- GitLab From b52b5a47148b6f05ed9439840dff9e3f189b3b19 Mon Sep 17 00:00:00 2001 From: Jonathan Hseu Date: Wed, 11 Apr 2018 18:57:49 -0700 Subject: [PATCH 225/791] Switch to WaitForNotification to fix the flaky test. See: https://source.cloud.google.com/results/invocations/31632a30-3728-4635-a456-f89b9e8b9dfe/log PiperOrigin-RevId: 192544848 --- tensorflow/core/platform/cloud/ram_file_block_cache_test.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tensorflow/core/platform/cloud/ram_file_block_cache_test.cc b/tensorflow/core/platform/cloud/ram_file_block_cache_test.cc index d555b682a6..10203783fc 100644 --- a/tensorflow/core/platform/cloud/ram_file_block_cache_test.cc +++ b/tensorflow/core/platform/cloud/ram_file_block_cache_test.cc @@ -487,8 +487,7 @@ TEST(RamFileBlockCacheTest, CoalesceConcurrentReads) { TF_EXPECT_OK(ReadCache(&cache, "", 0, block_size / 2, &out)); EXPECT_EQ(out.size(), block_size / 2); })); - EXPECT_TRUE(WaitForNotificationWithTimeout(¬ification, 10000)) - << "Timeout waiting for concurrent thread to start."; + notification.WaitForNotification(); std::vector out; TF_EXPECT_OK(ReadCache(&cache, "", block_size / 2, block_size / 2, &out)); EXPECT_EQ(out.size(), block_size / 2); -- GitLab From e7e01ac2597346f9dda2fb8fdb155fe784a1eebd Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 19:14:47 -0700 Subject: [PATCH 226/791] [XLA] Redesign: fix GetComputationGraphStats. CreateFromProto requires that the config has proper entry_computation_layout, so give the config the program shape. PiperOrigin-RevId: 192546316 --- tensorflow/compiler/xla/service/service.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc index 70af1c44ea..52500e4e79 100644 --- a/tensorflow/compiler/xla/service/service.cc +++ b/tensorflow/compiler/xla/service/service.cc @@ -1661,7 +1661,14 @@ tensorflow::Status Service::GetComputationStats( tensorflow::Status Service::GetComputationGraphStats( const ComputationGraphStatsRequest* arg, ComputationStatsResponse* result) { - HloModuleConfig config; + if (!arg->has_computation()) { + return InvalidArgument("Computations may not be empty."); + } + if (!arg->computation().has_program_shape()) { + return InvalidArgument("Program shape may not be empty."); + } + + HloModuleConfig config(arg->computation().program_shape()); config.set_debug_options(arg->debug_options()); TF_ASSIGN_OR_RETURN(std::unique_ptr module, HloModule::CreateFromProto(arg->computation(), config)); -- GitLab From 6f678934828a988ea06caf419dd97b9140f7c022 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 19:18:20 -0700 Subject: [PATCH 227/791] Update ops-related pbtxt files. PiperOrigin-RevId: 192546579 --- .../core/ops/compat/ops_history.v1.pbtxt | 65 +++++++++++++++++++ tensorflow/core/ops/ops.pbtxt | 65 +++++++++++++++++++ 2 files changed, 130 insertions(+) diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt index ba442a0582..30d4296326 100644 --- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt +++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt @@ -65536,6 +65536,71 @@ op { } is_stateful: true } +op { + name: "StatelessMultinomial" + input_arg { + name: "logits" + type_attr: "T" + } + input_arg { + name: "num_samples" + type: DT_INT32 + } + input_arg { + name: "seed" + type_attr: "Tseed" + } + output_arg { + name: "output" + type_attr: "output_dtype" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT32 + type: DT_UINT8 + type: DT_INT16 + type: DT_INT8 + type: DT_INT64 + type: DT_BFLOAT16 + type: DT_UINT16 + type: DT_HALF + type: DT_UINT32 + type: DT_UINT64 + } + } + } + attr { + name: "Tseed" + type: "type" + default_value { + type: DT_INT64 + } + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } + attr { + name: "output_dtype" + type: "type" + default_value { + type: DT_INT64 + } + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } +} op { name: "StatelessRandomNormal" input_arg { diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index 43fd09fb72..0ed039ac2e 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -29832,6 +29832,71 @@ op { } is_stateful: true } +op { + name: "StatelessMultinomial" + input_arg { + name: "logits" + type_attr: "T" + } + input_arg { + name: "num_samples" + type: DT_INT32 + } + input_arg { + name: "seed" + type_attr: "Tseed" + } + output_arg { + name: "output" + type_attr: "output_dtype" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT32 + type: DT_UINT8 + type: DT_INT16 + type: DT_INT8 + type: DT_INT64 + type: DT_BFLOAT16 + type: DT_UINT16 + type: DT_HALF + type: DT_UINT32 + type: DT_UINT64 + } + } + } + attr { + name: "Tseed" + type: "type" + default_value { + type: DT_INT64 + } + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } + attr { + name: "output_dtype" + type: "type" + default_value { + type: DT_INT64 + } + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } +} op { name: "StatelessRandomNormal" input_arg { -- GitLab From 6c9f8825096a76b395b01e07b8d611b3e2a23489 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 19:45:42 -0700 Subject: [PATCH 228/791] Go: Update generated wrapper functions for TensorFlow ops. PiperOrigin-RevId: 192548367 --- tensorflow/go/op/wrappers.go | 4640 +++++++++++++++++----------------- 1 file changed, 2320 insertions(+), 2320 deletions(-) diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index 09da8c1892..2d3e369328 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -2505,39 +2505,6 @@ func SparseToDense(scope *Scope, sparse_indices tf.Output, output_shape tf.Outpu return op.Output(0) } -// Counts the number of occurrences of each value in an integer array. -// -// Outputs a vector with length `size` and the same dtype as `weights`. If -// `weights` are empty, then index `i` stores the number of times the value `i` is -// counted in `arr`. If `weights` are non-empty, then index `i` stores the sum of -// the value in `weights` at each index where the corresponding value in `arr` is -// `i`. -// -// Values in `arr` outside of the range [0, size) are ignored. -// -// Arguments: -// arr: int32 `Tensor`. -// size: non-negative int32 scalar `Tensor`. -// weights: is an int32, int64, float32, or float64 `Tensor` with the same -// shape as `arr`, or a length-0 `Tensor`, in which case it acts as all weights -// equal to 1. -// -// Returns 1D `Tensor` with length equal to `size`. The counts or summed weights for -// each value in the range [0, size). -func Bincount(scope *Scope, arr tf.Output, size tf.Output, weights tf.Output) (bins tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "Bincount", - Input: []tf.Input{ - arr, size, weights, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // Computes the sum along sparse segments of a tensor. // // Read @{$math_ops#segmentation$the section on segmentation} for an explanation of @@ -6567,6 +6534,85 @@ func OneHot(scope *Scope, indices tf.Output, depth tf.Output, on_value tf.Output return op.Output(0) } +// Transforms a vector of brain.Example protos (as strings) into typed tensors. +// +// Arguments: +// serialized: A vector containing a batch of binary serialized Example protos. +// names: A vector containing the names of the serialized protos. +// May contain, for example, table key (descriptive) names for the +// corresponding serialized protos. These are purely useful for debugging +// purposes, and the presence of values here has no effect on the output. +// May also be an empty vector if no names are available. +// If non-empty, this vector must be the same length as "serialized". +// sparse_keys: A list of Nsparse string Tensors (scalars). +// The keys expected in the Examples' features associated with sparse values. +// dense_keys: A list of Ndense string Tensors (scalars). +// The keys expected in the Examples' features associated with dense values. +// dense_defaults: A list of Ndense Tensors (some may be empty). +// dense_defaults[j] provides default values +// when the example's feature_map lacks dense_key[j]. If an empty Tensor is +// provided for dense_defaults[j], then the Feature dense_keys[j] is required. +// The input type is inferred from dense_defaults[j], even when it's empty. +// If dense_defaults[j] is not empty, and dense_shapes[j] is fully defined, +// then the shape of dense_defaults[j] must match that of dense_shapes[j]. +// If dense_shapes[j] has an undefined major dimension (variable strides dense +// feature), dense_defaults[j] must contain a single element: +// the padding element. +// sparse_types: A list of Nsparse types; the data types of data in each Feature +// given in sparse_keys. +// Currently the ParseExample supports DT_FLOAT (FloatList), +// DT_INT64 (Int64List), and DT_STRING (BytesList). +// dense_shapes: A list of Ndense shapes; the shapes of data in each Feature +// given in dense_keys. +// The number of elements in the Feature corresponding to dense_key[j] +// must always equal dense_shapes[j].NumEntries(). +// If dense_shapes[j] == (D0, D1, ..., DN) then the shape of output +// Tensor dense_values[j] will be (|serialized|, D0, D1, ..., DN): +// The dense outputs are just the inputs row-stacked by batch. +// This works for dense_shapes[j] = (-1, D1, ..., DN). In this case +// the shape of the output Tensor dense_values[j] will be +// (|serialized|, M, D1, .., DN), where M is the maximum number of blocks +// of elements of length D1 * .... * DN, across all minibatch entries +// in the input. Any minibatch entry with less than M blocks of elements of +// length D1 * ... * DN will be padded with the corresponding default_value +// scalar element along the second dimension. +func ParseExample(scope *Scope, serialized tf.Output, names tf.Output, sparse_keys []tf.Output, dense_keys []tf.Output, dense_defaults []tf.Output, sparse_types []tf.DataType, dense_shapes []tf.Shape) (sparse_indices []tf.Output, sparse_values []tf.Output, sparse_shapes []tf.Output, dense_values []tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"sparse_types": sparse_types, "dense_shapes": dense_shapes} + opspec := tf.OpSpec{ + Type: "ParseExample", + Input: []tf.Input{ + serialized, names, tf.OutputList(sparse_keys), tf.OutputList(dense_keys), tf.OutputList(dense_defaults), + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + if scope.Err() != nil { + return + } + var idx int + var err error + if sparse_indices, idx, err = makeOutputList(op, idx, "sparse_indices"); err != nil { + scope.UpdateErr("ParseExample", err) + return + } + if sparse_values, idx, err = makeOutputList(op, idx, "sparse_values"); err != nil { + scope.UpdateErr("ParseExample", err) + return + } + if sparse_shapes, idx, err = makeOutputList(op, idx, "sparse_shapes"); err != nil { + scope.UpdateErr("ParseExample", err) + return + } + if dense_values, idx, err = makeOutputList(op, idx, "dense_values"); err != nil { + scope.UpdateErr("ParseExample", err) + return + } + return sparse_indices, sparse_values, sparse_shapes, dense_values +} + // Real-valued fast Fourier transform. // // Computes the 1-dimensional discrete Fourier transform of a real-valued signal @@ -7333,6 +7379,29 @@ func SparseSegmentSqrtN(scope *Scope, data tf.Output, indices tf.Output, segment return op.Output(0) } +// Adds up a `SparseTensor` and a dense `Tensor`, producing a dense `Tensor`. +// +// This Op does not require `a_indices` be sorted in standard lexicographic order. +// +// Arguments: +// a_indices: 2-D. The `indices` of the `SparseTensor`, with shape `[nnz, ndims]`. +// a_values: 1-D. The `values` of the `SparseTensor`, with shape `[nnz]`. +// a_shape: 1-D. The `shape` of the `SparseTensor`, with shape `[ndims]`. +// b: `ndims`-D Tensor. With shape `a_shape`. +func SparseTensorDenseAdd(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b tf.Output) (output tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "SparseTensorDenseAdd", + Input: []tf.Input{ + a_indices, a_values, a_shape, b, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // StatelessTruncatedNormalAttr is an optional argument to StatelessTruncatedNormal. type StatelessTruncatedNormalAttr func(optionalAttr) @@ -8414,98 +8483,49 @@ func StringToHashBucketStrong(scope *Scope, input tf.Output, num_buckets int64, return op.Output(0) } -// Applies softmax to a batched N-D `SparseTensor`. -// -// The inputs represent an N-D SparseTensor with logical shape `[..., B, C]` -// (where `N >= 2`), and with indices sorted in the canonical lexicographic order. -// -// This op is equivalent to applying the normal `tf.nn.softmax()` to each innermost -// logical submatrix with shape `[B, C]`, but with the catch that *the implicitly -// zero elements do not participate*. Specifically, the algorithm is equivalent -// to the following: -// -// (1) Applies `tf.nn.softmax()` to a densified view of each innermost submatrix -// with shape `[B, C]`, along the size-C dimension; -// (2) Masks out the original implicitly-zero locations; -// (3) Renormalizes the remaining elements. -// -// Hence, the `SparseTensor` result has exactly the same non-zero indices and -// shape. -// -// Arguments: -// sp_indices: 2-D. `NNZ x R` matrix with the indices of non-empty values in a -// SparseTensor, in canonical ordering. -// sp_values: 1-D. `NNZ` non-empty values corresponding to `sp_indices`. -// sp_shape: 1-D. Shape of the input SparseTensor. +// Computes numerical negative value element-wise. // -// Returns 1-D. The `NNZ` values for the result `SparseTensor`. -func SparseSoftmax(scope *Scope, sp_indices tf.Output, sp_values tf.Output, sp_shape tf.Output) (output tf.Output) { +// I.e., \\(y = -x\\). +func Neg(scope *Scope, x tf.Output) (y tf.Output) { if scope.Err() != nil { return } opspec := tf.OpSpec{ - Type: "SparseSoftmax", + Type: "Neg", Input: []tf.Input{ - sp_indices, sp_values, sp_shape, + x, }, } op := scope.AddOperation(opspec) return op.Output(0) } -// Partitions `data` into `num_partitions` tensors using indices from `partitions`. -// -// For each index tuple `js` of size `partitions.ndim`, the slice `data[js, ...]` -// becomes part of `outputs[partitions[js]]`. The slices with `partitions[js] = i` -// are placed in `outputs[i]` in lexicographic order of `js`, and the first -// dimension of `outputs[i]` is the number of entries in `partitions` equal to `i`. -// In detail, -// -// ```python -// outputs[i].shape = [sum(partitions == i)] + data.shape[partitions.ndim:] -// -// outputs[i] = pack([data[js, ...] for js if partitions[js] == i]) -// ``` -// -// `data.shape` must start with `partitions.shape`. -// -// For example: -// -// ```python -// # Scalar partitions. -// partitions = 1 -// num_partitions = 2 -// data = [10, 20] -// outputs[0] = [] # Empty with shape [0, 2] -// outputs[1] = [[10, 20]] -// -// # Vector partitions. -// partitions = [0, 0, 1, 1, 0] -// num_partitions = 2 -// data = [10, 20, 30, 40, 50] -// outputs[0] = [10, 20, 50] -// outputs[1] = [30, 40] -// ``` -// -// See `dynamic_stitch` for an example on how to merge partitions back. +// Execute a sub graph on a remote processor. // -//

-// -//
+// The graph specifications(such as graph itself, input tensors and output names) +// are stored as a serialized protocol buffer of RemoteFusedGraphExecuteInfo +// as serialized_remote_fused_graph_execute_info. +// The specifications will be passed to a dedicated registered +// remote fused graph executor. The executor will send the graph specifications +// to a remote processor and execute that graph. The execution results +// will be passed to consumer nodes as outputs of this node. // // Arguments: +// inputs: Arbitrary number of tensors with arbitrary data types // -// partitions: Any shape. Indices in the range `[0, num_partitions)`. -// num_partitions: The number of partitions to output. -func DynamicPartition(scope *Scope, data tf.Output, partitions tf.Output, num_partitions int64) (outputs []tf.Output) { +// serialized_remote_fused_graph_execute_info: Serialized protocol buffer +// of RemoteFusedGraphExecuteInfo which contains graph specifications. +// +// Returns Arbitrary number of tensors with arbitrary data types +func RemoteFusedGraphExecute(scope *Scope, inputs []tf.Output, Toutputs []tf.DataType, serialized_remote_fused_graph_execute_info string) (outputs []tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{"num_partitions": num_partitions} + attrs := map[string]interface{}{"Toutputs": Toutputs, "serialized_remote_fused_graph_execute_info": serialized_remote_fused_graph_execute_info} opspec := tf.OpSpec{ - Type: "DynamicPartition", + Type: "RemoteFusedGraphExecute", Input: []tf.Input{ - data, partitions, + tf.OutputList(inputs), }, Attrs: attrs, } @@ -8516,119 +8536,117 @@ func DynamicPartition(scope *Scope, data tf.Output, partitions tf.Output, num_pa var idx int var err error if outputs, idx, err = makeOutputList(op, idx, "outputs"); err != nil { - scope.UpdateErr("DynamicPartition", err) + scope.UpdateErr("RemoteFusedGraphExecute", err) return } return outputs } -// ResourceApplyAdagradAttr is an optional argument to ResourceApplyAdagrad. -type ResourceApplyAdagradAttr func(optionalAttr) +// MaxPool3DGradGradAttr is an optional argument to MaxPool3DGradGrad. +type MaxPool3DGradGradAttr func(optionalAttr) -// ResourceApplyAdagradUseLocking sets the optional use_locking attribute to value. +// MaxPool3DGradGradDataFormat sets the optional data_format attribute to value. // -// value: If `True`, updating of the var and accum tensors will be protected -// by a lock; otherwise the behavior is undefined, but may exhibit less -// contention. -// If not specified, defaults to false -func ResourceApplyAdagradUseLocking(value bool) ResourceApplyAdagradAttr { +// value: The data format of the input and output data. With the +// default format "NDHWC", the data is stored in the order of: +// [batch, in_depth, in_height, in_width, in_channels]. +// Alternatively, the format could be "NCDHW", the data storage order is: +// [batch, in_channels, in_depth, in_height, in_width]. +// If not specified, defaults to "NDHWC" +func MaxPool3DGradGradDataFormat(value string) MaxPool3DGradGradAttr { return func(m optionalAttr) { - m["use_locking"] = value + m["data_format"] = value } } -// Update '*var' according to the adagrad scheme. -// -// accum += grad * grad -// var -= lr * grad * (1 / sqrt(accum)) +// Computes second-order gradients of the maxpooling function. // // Arguments: -// var_: Should be from a Variable(). -// accum: Should be from a Variable(). -// lr: Scaling factor. Must be a scalar. -// grad: The gradient. +// orig_input: The original input tensor. +// orig_output: The original output tensor. +// grad: Output backprop of shape `[batch, depth, rows, cols, channels]`. +// ksize: 1-D tensor of length 5. The size of the window for each dimension of +// the input tensor. Must have `ksize[0] = ksize[4] = 1`. +// strides: 1-D tensor of length 5. The stride of the sliding window for each +// dimension of `input`. Must have `strides[0] = strides[4] = 1`. +// padding: The type of padding algorithm to use. // -// Returns the created operation. -func ResourceApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, optional ...ResourceApplyAdagradAttr) (o *tf.Operation) { +// Returns Gradients of gradients w.r.t. the input to `max_pool`. +func MaxPool3DGradGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPool3DGradGradAttr) (output tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{} + attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding} for _, a := range optional { a(attrs) } opspec := tf.OpSpec{ - Type: "ResourceApplyAdagrad", + Type: "MaxPool3DGradGrad", Input: []tf.Input{ - var_, accum, lr, grad, + orig_input, orig_output, grad, }, Attrs: attrs, } - return scope.AddOperation(opspec) -} - -// Return the shape of s0 op s1 with broadcast. -// -// Given `s0` and `s1`, tensors that represent shapes, compute `r0`, the -// broadcasted shape. `s0`, `s1` and `r0` are all integer vectors. -func BroadcastArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "BroadcastArgs", - Input: []tf.Input{ - s0, s1, - }, - } op := scope.AddOperation(opspec) return op.Output(0) } -// DataFormatDimMapAttr is an optional argument to DataFormatDimMap. -type DataFormatDimMapAttr func(optionalAttr) +// Conv3DBackpropFilterV2Attr is an optional argument to Conv3DBackpropFilterV2. +type Conv3DBackpropFilterV2Attr func(optionalAttr) -// DataFormatDimMapSrcFormat sets the optional src_format attribute to value. +// Conv3DBackpropFilterV2DataFormat sets the optional data_format attribute to value. // -// value: source data format. -// If not specified, defaults to "NHWC" -func DataFormatDimMapSrcFormat(value string) DataFormatDimMapAttr { +// value: The data format of the input and output data. With the +// default format "NDHWC", the data is stored in the order of: +// [batch, in_depth, in_height, in_width, in_channels]. +// Alternatively, the format could be "NCDHW", the data storage order is: +// [batch, in_channels, in_depth, in_height, in_width]. +// If not specified, defaults to "NDHWC" +func Conv3DBackpropFilterV2DataFormat(value string) Conv3DBackpropFilterV2Attr { return func(m optionalAttr) { - m["src_format"] = value + m["data_format"] = value } } -// DataFormatDimMapDstFormat sets the optional dst_format attribute to value. +// Conv3DBackpropFilterV2Dilations sets the optional dilations attribute to value. // -// value: destination data format. -// If not specified, defaults to "NCHW" -func DataFormatDimMapDstFormat(value string) DataFormatDimMapAttr { +// value: 1-D tensor of length 5. The dilation factor for each dimension of +// `input`. If set to k > 1, there will be k-1 skipped cells between each +// filter element on that dimension. The dimension order is determined by the +// value of `data_format`, see above for details. Dilations in the batch and +// depth dimensions must be 1. +// If not specified, defaults to +func Conv3DBackpropFilterV2Dilations(value []int64) Conv3DBackpropFilterV2Attr { return func(m optionalAttr) { - m["dst_format"] = value + m["dilations"] = value } } -// Returns the dimension index in the destination data format given the one in -// -// the source data format. +// Computes the gradients of 3-D convolution with respect to the filter. // // Arguments: -// x: A Tensor with each element as a dimension index in source data format. -// Must be in the range [-4, 4). -// -// Returns A Tensor with each element as a dimension index in destination data format. -func DataFormatDimMap(scope *Scope, x tf.Output, optional ...DataFormatDimMapAttr) (y tf.Output) { +// input: Shape `[batch, depth, rows, cols, in_channels]`. +// filter_sizes: An integer vector representing the tensor shape of `filter`, +// where `filter` is a 5-D +// `[filter_depth, filter_height, filter_width, in_channels, out_channels]` +// tensor. +// out_backprop: Backprop signal of shape `[batch, out_depth, out_rows, out_cols, +// out_channels]`. +// strides: 1-D tensor of length 5. The stride of the sliding window for each +// dimension of `input`. Must have `strides[0] = strides[4] = 1`. +// padding: The type of padding algorithm to use. +func Conv3DBackpropFilterV2(scope *Scope, input tf.Output, filter_sizes tf.Output, out_backprop tf.Output, strides []int64, padding string, optional ...Conv3DBackpropFilterV2Attr) (output tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{} + attrs := map[string]interface{}{"strides": strides, "padding": padding} for _, a := range optional { a(attrs) } opspec := tf.OpSpec{ - Type: "DataFormatDimMap", + Type: "Conv3DBackpropFilterV2", Input: []tf.Input{ - x, + input, filter_sizes, out_backprop, }, Attrs: attrs, } @@ -8636,38 +8654,38 @@ func DataFormatDimMap(scope *Scope, x tf.Output, optional ...DataFormatDimMapAtt return op.Output(0) } -// ResourceApplyPowerSignAttr is an optional argument to ResourceApplyPowerSign. -type ResourceApplyPowerSignAttr func(optionalAttr) +// FakeQuantWithMinMaxVarsAttr is an optional argument to FakeQuantWithMinMaxVars. +type FakeQuantWithMinMaxVarsAttr func(optionalAttr) -// ResourceApplyPowerSignUseLocking sets the optional use_locking attribute to value. -// -// value: If `True`, updating of the var and m tensors is -// protected by a lock; otherwise the behavior is undefined, but may exhibit less -// contention. +// FakeQuantWithMinMaxVarsNumBits sets the optional num_bits attribute to value. +// If not specified, defaults to 8 +func FakeQuantWithMinMaxVarsNumBits(value int64) FakeQuantWithMinMaxVarsAttr { + return func(m optionalAttr) { + m["num_bits"] = value + } +} + +// FakeQuantWithMinMaxVarsNarrowRange sets the optional narrow_range attribute to value. // If not specified, defaults to false -func ResourceApplyPowerSignUseLocking(value bool) ResourceApplyPowerSignAttr { +func FakeQuantWithMinMaxVarsNarrowRange(value bool) FakeQuantWithMinMaxVarsAttr { return func(m optionalAttr) { - m["use_locking"] = value + m["narrow_range"] = value } } -// Update '*var' according to the AddSign update. +// Fake-quantize the 'inputs' tensor of type float via global float scalars `min` // -// m_t <- beta1 * m_{t-1} + (1 - beta1) * g -// update <- exp(logbase * sign_decay * sign(g) * sign(m_t)) * g -// variable <- variable - lr_t * update +// and `max` to 'outputs' tensor of same shape as `inputs`. // -// Arguments: -// var_: Should be from a Variable(). -// m: Should be from a Variable(). -// lr: Scaling factor. Must be a scalar. -// logbase: Must be a scalar. -// sign_decay: Must be a scalar. -// beta: Must be a scalar. -// grad: The gradient. +// `[min; max]` define the clamping range for the `inputs` data. +// `inputs` values are quantized into the quantization range (`[0; 2^num_bits - 1]` +// when `narrow_range` is false and `[1; 2^num_bits - 1]` when it is true) and +// then de-quantized and output as floats in `[min; max]` interval. +// `num_bits` is the bitwidth of the quantization; between 2 and 8, inclusive. // -// Returns the created operation. -func ResourceApplyPowerSign(scope *Scope, var_ tf.Output, m tf.Output, lr tf.Output, logbase tf.Output, sign_decay tf.Output, beta tf.Output, grad tf.Output, optional ...ResourceApplyPowerSignAttr) (o *tf.Operation) { +// This operation has a gradient and thus allows for training `min` and `max` +// values. +func FakeQuantWithMinMaxVars(scope *Scope, inputs tf.Output, min tf.Output, max tf.Output, optional ...FakeQuantWithMinMaxVarsAttr) (outputs tf.Output) { if scope.Err() != nil { return } @@ -8676,225 +8694,152 @@ func ResourceApplyPowerSign(scope *Scope, var_ tf.Output, m tf.Output, lr tf.Out a(attrs) } opspec := tf.OpSpec{ - Type: "ResourceApplyPowerSign", + Type: "FakeQuantWithMinMaxVars", Input: []tf.Input{ - var_, m, lr, logbase, sign_decay, beta, grad, + inputs, min, max, }, Attrs: attrs, } - return scope.AddOperation(opspec) + op := scope.AddOperation(opspec) + return op.Output(0) } -// Computes the mean along segments of a tensor. +// Applies softmax to a batched N-D `SparseTensor`. // -// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of -// segments. +// The inputs represent an N-D SparseTensor with logical shape `[..., B, C]` +// (where `N >= 2`), and with indices sorted in the canonical lexicographic order. // -// Computes a tensor such that -// \\(output_i = \frac{\sum_j data_j}{N}\\) where `mean` is -// over `j` such that `segment_ids[j] == i` and `N` is the total number of -// values summed. +// This op is equivalent to applying the normal `tf.nn.softmax()` to each innermost +// logical submatrix with shape `[B, C]`, but with the catch that *the implicitly +// zero elements do not participate*. Specifically, the algorithm is equivalent +// to the following: // -// If the mean is empty for a given segment ID `i`, `output[i] = 0`. +// (1) Applies `tf.nn.softmax()` to a densified view of each innermost submatrix +// with shape `[B, C]`, along the size-C dimension; +// (2) Masks out the original implicitly-zero locations; +// (3) Renormalizes the remaining elements. // -//
-// -//
+// Hence, the `SparseTensor` result has exactly the same non-zero indices and +// shape. // // Arguments: +// sp_indices: 2-D. `NNZ x R` matrix with the indices of non-empty values in a +// SparseTensor, in canonical ordering. +// sp_values: 1-D. `NNZ` non-empty values corresponding to `sp_indices`. +// sp_shape: 1-D. Shape of the input SparseTensor. // -// segment_ids: A 1-D tensor whose rank is equal to the rank of `data`'s -// first dimension. Values should be sorted and can be repeated. -// -// Returns Has same shape as data, except for dimension 0 which -// has size `k`, the number of segments. -func SegmentMean(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) { +// Returns 1-D. The `NNZ` values for the result `SparseTensor`. +func SparseSoftmax(scope *Scope, sp_indices tf.Output, sp_values tf.Output, sp_shape tf.Output) (output tf.Output) { if scope.Err() != nil { return } opspec := tf.OpSpec{ - Type: "SegmentMean", + Type: "SparseSoftmax", Input: []tf.Input{ - data, segment_ids, + sp_indices, sp_values, sp_shape, }, } op := scope.AddOperation(opspec) return op.Output(0) } -// ResourceSparseApplyCenteredRMSPropAttr is an optional argument to ResourceSparseApplyCenteredRMSProp. -type ResourceSparseApplyCenteredRMSPropAttr func(optionalAttr) - -// ResourceSparseApplyCenteredRMSPropUseLocking sets the optional use_locking attribute to value. +// Partitions `data` into `num_partitions` tensors using indices from `partitions`. // -// value: If `True`, updating of the var, mg, ms, and mom tensors is -// protected by a lock; otherwise the behavior is undefined, but may exhibit less -// contention. -// If not specified, defaults to false -func ResourceSparseApplyCenteredRMSPropUseLocking(value bool) ResourceSparseApplyCenteredRMSPropAttr { - return func(m optionalAttr) { - m["use_locking"] = value - } -} - -// Update '*var' according to the centered RMSProp algorithm. +// For each index tuple `js` of size `partitions.ndim`, the slice `data[js, ...]` +// becomes part of `outputs[partitions[js]]`. The slices with `partitions[js] = i` +// are placed in `outputs[i]` in lexicographic order of `js`, and the first +// dimension of `outputs[i]` is the number of entries in `partitions` equal to `i`. +// In detail, // -// The centered RMSProp algorithm uses an estimate of the centered second moment -// (i.e., the variance) for normalization, as opposed to regular RMSProp, which -// uses the (uncentered) second moment. This often helps with training, but is -// slightly more expensive in terms of computation and memory. +// ```python +// outputs[i].shape = [sum(partitions == i)] + data.shape[partitions.ndim:] // -// Note that in dense implementation of this algorithm, mg, ms, and mom will -// update even if the grad is zero, but in this sparse implementation, mg, ms, -// and mom will not update in iterations during which the grad is zero. +// outputs[i] = pack([data[js, ...] for js if partitions[js] == i]) +// ``` // -// mean_square = decay * mean_square + (1-decay) * gradient ** 2 -// mean_grad = decay * mean_grad + (1-decay) * gradient -// Delta = learning_rate * gradient / sqrt(mean_square + epsilon - mean_grad ** 2) +// `data.shape` must start with `partitions.shape`. // -// ms <- rho * ms_{t-1} + (1-rho) * grad * grad -// mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon) -// var <- var - mom +// For example: // -// Arguments: -// var_: Should be from a Variable(). -// mg: Should be from a Variable(). -// ms: Should be from a Variable(). -// mom: Should be from a Variable(). -// lr: Scaling factor. Must be a scalar. -// rho: Decay rate. Must be a scalar. +// ```python +// # Scalar partitions. +// partitions = 1 +// num_partitions = 2 +// data = [10, 20] +// outputs[0] = [] # Empty with shape [0, 2] +// outputs[1] = [[10, 20]] // -// epsilon: Ridge term. Must be a scalar. -// grad: The gradient. -// indices: A vector of indices into the first dimension of var, ms and mom. +// # Vector partitions. +// partitions = [0, 0, 1, 1, 0] +// num_partitions = 2 +// data = [10, 20, 30, 40, 50] +// outputs[0] = [10, 20, 50] +// outputs[1] = [30, 40] +// ``` // -// Returns the created operation. -func ResourceSparseApplyCenteredRMSProp(scope *Scope, var_ tf.Output, mg tf.Output, ms tf.Output, mom tf.Output, lr tf.Output, rho tf.Output, momentum tf.Output, epsilon tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyCenteredRMSPropAttr) (o *tf.Operation) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "ResourceSparseApplyCenteredRMSProp", - Input: []tf.Input{ - var_, mg, ms, mom, lr, rho, momentum, epsilon, grad, indices, - }, - Attrs: attrs, - } - return scope.AddOperation(opspec) -} - -// Creates a dataset that batches `batch_size` elements from `input_dataset`. -// -// Arguments: +// See `dynamic_stitch` for an example on how to merge partitions back. // -// batch_size: A scalar representing the number of elements to accumulate in a -// batch. +//
+// +//
// +// Arguments: // -func BatchDataset(scope *Scope, input_dataset tf.Output, batch_size tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) { +// partitions: Any shape. Indices in the range `[0, num_partitions)`. +// num_partitions: The number of partitions to output. +func DynamicPartition(scope *Scope, data tf.Output, partitions tf.Output, num_partitions int64) (outputs []tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes} + attrs := map[string]interface{}{"num_partitions": num_partitions} opspec := tf.OpSpec{ - Type: "BatchDataset", + Type: "DynamicPartition", Input: []tf.Input{ - input_dataset, batch_size, + data, partitions, }, Attrs: attrs, } op := scope.AddOperation(opspec) - return op.Output(0) -} - -// Inverse fast Fourier transform. -// -// Computes the inverse 1-dimensional discrete Fourier transform over the -// inner-most dimension of `input`. -// -// Arguments: -// input: A complex64 tensor. -// -// Returns A complex64 tensor of the same shape as `input`. The inner-most -// dimension of `input` is replaced with its inverse 1D Fourier transform. -// -// @compatibility(numpy) -// Equivalent to np.fft.ifft -// @end_compatibility -func IFFT(scope *Scope, input tf.Output) (output tf.Output) { if scope.Err() != nil { return } - opspec := tf.OpSpec{ - Type: "IFFT", - Input: []tf.Input{ - input, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// Generates values in an interval. -// -// A sequence of `num` evenly-spaced values are generated beginning at `start`. -// If `num > 1`, the values in the sequence increase by `stop - start / num - 1`, -// so that the last one is exactly `stop`. -// -// For example: -// -// ``` -// tf.linspace(10.0, 12.0, 3, name="linspace") => [ 10.0 11.0 12.0] -// ``` -// -// Arguments: -// start: First entry in the range. -// stop: Last entry in the range. -// num: Number of values to generate. -// -// Returns 1-D. The generated values. -func LinSpace(scope *Scope, start tf.Output, stop tf.Output, num tf.Output) (output tf.Output) { - if scope.Err() != nil { + var idx int + var err error + if outputs, idx, err = makeOutputList(op, idx, "outputs"); err != nil { + scope.UpdateErr("DynamicPartition", err) return } - opspec := tf.OpSpec{ - Type: "LinSpace", - Input: []tf.Input{ - start, stop, num, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) + return outputs } -// DestroyResourceOpAttr is an optional argument to DestroyResourceOp. -type DestroyResourceOpAttr func(optionalAttr) +// ResourceApplyAdagradAttr is an optional argument to ResourceApplyAdagrad. +type ResourceApplyAdagradAttr func(optionalAttr) -// DestroyResourceOpIgnoreLookupError sets the optional ignore_lookup_error attribute to value. +// ResourceApplyAdagradUseLocking sets the optional use_locking attribute to value. // -// value: whether to ignore the error when the resource -// doesn't exist. -// If not specified, defaults to true -func DestroyResourceOpIgnoreLookupError(value bool) DestroyResourceOpAttr { +// value: If `True`, updating of the var and accum tensors will be protected +// by a lock; otherwise the behavior is undefined, but may exhibit less +// contention. +// If not specified, defaults to false +func ResourceApplyAdagradUseLocking(value bool) ResourceApplyAdagradAttr { return func(m optionalAttr) { - m["ignore_lookup_error"] = value + m["use_locking"] = value } } -// Deletes the resource specified by the handle. +// Update '*var' according to the adagrad scheme. // -// All subsequent operations using the resource will result in a NotFound -// error status. +// accum += grad * grad +// var -= lr * grad * (1 / sqrt(accum)) // // Arguments: -// resource: handle to the resource to delete. +// var_: Should be from a Variable(). +// accum: Should be from a Variable(). +// lr: Scaling factor. Must be a scalar. +// grad: The gradient. // // Returns the created operation. -func DestroyResourceOp(scope *Scope, resource tf.Output, optional ...DestroyResourceOpAttr) (o *tf.Operation) { +func ResourceApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, optional ...ResourceApplyAdagradAttr) (o *tf.Operation) { if scope.Err() != nil { return } @@ -8903,75 +8848,66 @@ func DestroyResourceOp(scope *Scope, resource tf.Output, optional ...DestroyReso a(attrs) } opspec := tf.OpSpec{ - Type: "DestroyResourceOp", + Type: "ResourceApplyAdagrad", Input: []tf.Input{ - resource, + var_, accum, lr, grad, }, Attrs: attrs, } return scope.AddOperation(opspec) } -// LRNAttr is an optional argument to LRN. -type LRNAttr func(optionalAttr) - -// LRNDepthRadius sets the optional depth_radius attribute to value. +// Return the shape of s0 op s1 with broadcast. // -// value: 0-D. Half-width of the 1-D normalization window. -// If not specified, defaults to 5 -func LRNDepthRadius(value int64) LRNAttr { - return func(m optionalAttr) { - m["depth_radius"] = value +// Given `s0` and `s1`, tensors that represent shapes, compute `r0`, the +// broadcasted shape. `s0`, `s1` and `r0` are all integer vectors. +func BroadcastArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Output) { + if scope.Err() != nil { + return } -} - -// LRNBias sets the optional bias attribute to value. -// -// value: An offset (usually positive to avoid dividing by 0). -// If not specified, defaults to 1 -func LRNBias(value float32) LRNAttr { - return func(m optionalAttr) { - m["bias"] = value + opspec := tf.OpSpec{ + Type: "BroadcastArgs", + Input: []tf.Input{ + s0, s1, + }, } + op := scope.AddOperation(opspec) + return op.Output(0) } -// LRNAlpha sets the optional alpha attribute to value. +// DataFormatDimMapAttr is an optional argument to DataFormatDimMap. +type DataFormatDimMapAttr func(optionalAttr) + +// DataFormatDimMapSrcFormat sets the optional src_format attribute to value. // -// value: A scale factor, usually positive. -// If not specified, defaults to 1 -func LRNAlpha(value float32) LRNAttr { +// value: source data format. +// If not specified, defaults to "NHWC" +func DataFormatDimMapSrcFormat(value string) DataFormatDimMapAttr { return func(m optionalAttr) { - m["alpha"] = value + m["src_format"] = value } } -// LRNBeta sets the optional beta attribute to value. +// DataFormatDimMapDstFormat sets the optional dst_format attribute to value. // -// value: An exponent. -// If not specified, defaults to 0.5 -func LRNBeta(value float32) LRNAttr { +// value: destination data format. +// If not specified, defaults to "NCHW" +func DataFormatDimMapDstFormat(value string) DataFormatDimMapAttr { return func(m optionalAttr) { - m["beta"] = value + m["dst_format"] = value } } -// Local Response Normalization. -// -// The 4-D `input` tensor is treated as a 3-D array of 1-D vectors (along the last -// dimension), and each vector is normalized independently. Within a given vector, -// each component is divided by the weighted, squared sum of inputs within -// `depth_radius`. In detail, -// -// sqr_sum[a, b, c, d] = -// sum(input[a, b, c, d - depth_radius : d + depth_radius + 1] ** 2) -// output = input / (bias + alpha * sqr_sum) ** beta +// Returns the dimension index in the destination data format given the one in // -// For details, see [Krizhevsky et al., ImageNet classification with deep -// convolutional neural networks (NIPS 2012)](http://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks). +// the source data format. // // Arguments: -// input: 4-D. -func LRN(scope *Scope, input tf.Output, optional ...LRNAttr) (output tf.Output) { +// x: A Tensor with each element as a dimension index in source data format. +// Must be in the range [-4, 4). +// +// Returns A Tensor with each element as a dimension index in destination data format. +func DataFormatDimMap(scope *Scope, x tf.Output, optional ...DataFormatDimMapAttr) (y tf.Output) { if scope.Err() != nil { return } @@ -8980,26 +8916,9 @@ func LRN(scope *Scope, input tf.Output, optional ...LRNAttr) (output tf.Output) a(attrs) } opspec := tf.OpSpec{ - Type: "LRN", - Input: []tf.Input{ - input, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// Creates a dataset that zips together `input_datasets`. -func ZipDataset(scope *Scope, input_datasets []tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes} - opspec := tf.OpSpec{ - Type: "ZipDataset", + Type: "DataFormatDimMap", Input: []tf.Input{ - tf.OutputList(input_datasets), + x, }, Attrs: attrs, } @@ -9007,36 +8926,38 @@ func ZipDataset(scope *Scope, input_datasets []tf.Output, output_types []tf.Data return op.Output(0) } -// ResourceSparseApplyAdagradAttr is an optional argument to ResourceSparseApplyAdagrad. -type ResourceSparseApplyAdagradAttr func(optionalAttr) +// ResourceApplyPowerSignAttr is an optional argument to ResourceApplyPowerSign. +type ResourceApplyPowerSignAttr func(optionalAttr) -// ResourceSparseApplyAdagradUseLocking sets the optional use_locking attribute to value. +// ResourceApplyPowerSignUseLocking sets the optional use_locking attribute to value. // -// value: If `True`, updating of the var and accum tensors will be protected -// by a lock; otherwise the behavior is undefined, but may exhibit less +// value: If `True`, updating of the var and m tensors is +// protected by a lock; otherwise the behavior is undefined, but may exhibit less // contention. // If not specified, defaults to false -func ResourceSparseApplyAdagradUseLocking(value bool) ResourceSparseApplyAdagradAttr { +func ResourceApplyPowerSignUseLocking(value bool) ResourceApplyPowerSignAttr { return func(m optionalAttr) { m["use_locking"] = value } } -// Update relevant entries in '*var' and '*accum' according to the adagrad scheme. +// Update '*var' according to the AddSign update. // -// That is for rows we have grad for, we update var and accum as follows: -// accum += grad * grad -// var -= lr * grad * (1 / sqrt(accum)) +// m_t <- beta1 * m_{t-1} + (1 - beta1) * g +// update <- exp(logbase * sign_decay * sign(g) * sign(m_t)) * g +// variable <- variable - lr_t * update // // Arguments: // var_: Should be from a Variable(). -// accum: Should be from a Variable(). -// lr: Learning rate. Must be a scalar. +// m: Should be from a Variable(). +// lr: Scaling factor. Must be a scalar. +// logbase: Must be a scalar. +// sign_decay: Must be a scalar. +// beta: Must be a scalar. // grad: The gradient. -// indices: A vector of indices into the first dimension of var and accum. // // Returns the created operation. -func ResourceSparseApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyAdagradAttr) (o *tf.Operation) { +func ResourceApplyPowerSign(scope *Scope, var_ tf.Output, m tf.Output, lr tf.Output, logbase tf.Output, sign_decay tf.Output, beta tf.Output, grad tf.Output, optional ...ResourceApplyPowerSignAttr) (o *tf.Operation) { if scope.Err() != nil { return } @@ -9045,87 +8966,100 @@ func ResourceSparseApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, l a(attrs) } opspec := tf.OpSpec{ - Type: "ResourceSparseApplyAdagrad", + Type: "ResourceApplyPowerSign", Input: []tf.Input{ - var_, accum, lr, grad, indices, + var_, m, lr, logbase, sign_decay, beta, grad, }, Attrs: attrs, } return scope.AddOperation(opspec) } -// 2D real-valued fast Fourier transform. +// Computes the mean along segments of a tensor. // -// Computes the 2-dimensional discrete Fourier transform of a real-valued signal -// over the inner-most 2 dimensions of `input`. +// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of +// segments. // -// Since the DFT of a real signal is Hermitian-symmetric, `RFFT2D` only returns the -// `fft_length / 2 + 1` unique components of the FFT for the inner-most dimension -// of `output`: the zero-frequency term, followed by the `fft_length / 2` -// positive-frequency terms. +// Computes a tensor such that +// \\(output_i = \frac{\sum_j data_j}{N}\\) where `mean` is +// over `j` such that `segment_ids[j] == i` and `N` is the total number of +// values summed. // -// Along each axis `RFFT2D` is computed on, if `fft_length` is smaller than the -// corresponding dimension of `input`, the dimension is cropped. If it is larger, -// the dimension is padded with zeros. +// If the mean is empty for a given segment ID `i`, `output[i] = 0`. +// +//
+// +//
// // Arguments: -// input: A float32 tensor. -// fft_length: An int32 tensor of shape [2]. The FFT length for each dimension. // -// Returns A complex64 tensor of the same rank as `input`. The inner-most 2 -// dimensions of `input` are replaced with their 2D Fourier transform. The -// inner-most dimension contains `fft_length / 2 + 1` unique frequency -// components. +// segment_ids: A 1-D tensor whose rank is equal to the rank of `data`'s +// first dimension. Values should be sorted and can be repeated. // -// @compatibility(numpy) -// Equivalent to np.fft.rfft2 -// @end_compatibility -func RFFT2D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) { +// Returns Has same shape as data, except for dimension 0 which +// has size `k`, the number of segments. +func SegmentMean(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) { if scope.Err() != nil { return } opspec := tf.OpSpec{ - Type: "RFFT2D", + Type: "SegmentMean", Input: []tf.Input{ - input, fft_length, + data, segment_ids, }, } op := scope.AddOperation(opspec) return op.Output(0) } -// ResizeAreaAttr is an optional argument to ResizeArea. -type ResizeAreaAttr func(optionalAttr) +// ResourceSparseApplyCenteredRMSPropAttr is an optional argument to ResourceSparseApplyCenteredRMSProp. +type ResourceSparseApplyCenteredRMSPropAttr func(optionalAttr) -// ResizeAreaAlignCorners sets the optional align_corners attribute to value. +// ResourceSparseApplyCenteredRMSPropUseLocking sets the optional use_locking attribute to value. // -// value: If true, rescale input by (new_height - 1) / (height - 1), which -// exactly aligns the 4 corners of images and resized images. If false, rescale -// by new_height / height. Treat similarly the width dimension. +// value: If `True`, updating of the var, mg, ms, and mom tensors is +// protected by a lock; otherwise the behavior is undefined, but may exhibit less +// contention. // If not specified, defaults to false -func ResizeAreaAlignCorners(value bool) ResizeAreaAttr { +func ResourceSparseApplyCenteredRMSPropUseLocking(value bool) ResourceSparseApplyCenteredRMSPropAttr { return func(m optionalAttr) { - m["align_corners"] = value + m["use_locking"] = value } } -// Resize `images` to `size` using area interpolation. +// Update '*var' according to the centered RMSProp algorithm. // -// Input images can be of different types but output images are always float. +// The centered RMSProp algorithm uses an estimate of the centered second moment +// (i.e., the variance) for normalization, as opposed to regular RMSProp, which +// uses the (uncentered) second moment. This often helps with training, but is +// slightly more expensive in terms of computation and memory. // -// Each output pixel is computed by first transforming the pixel's footprint into -// the input tensor and then averaging the pixels that intersect the footprint. An -// input pixel's contribution to the average is weighted by the fraction of its -// area that intersects the footprint. This is the same as OpenCV's INTER_AREA. +// Note that in dense implementation of this algorithm, mg, ms, and mom will +// update even if the grad is zero, but in this sparse implementation, mg, ms, +// and mom will not update in iterations during which the grad is zero. +// +// mean_square = decay * mean_square + (1-decay) * gradient ** 2 +// mean_grad = decay * mean_grad + (1-decay) * gradient +// Delta = learning_rate * gradient / sqrt(mean_square + epsilon - mean_grad ** 2) +// +// ms <- rho * ms_{t-1} + (1-rho) * grad * grad +// mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon) +// var <- var - mom // // Arguments: -// images: 4-D with shape `[batch, height, width, channels]`. -// size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`. The -// new size for the images. +// var_: Should be from a Variable(). +// mg: Should be from a Variable(). +// ms: Should be from a Variable(). +// mom: Should be from a Variable(). +// lr: Scaling factor. Must be a scalar. +// rho: Decay rate. Must be a scalar. // -// Returns 4-D with shape -// `[batch, new_height, new_width, channels]`. -func ResizeArea(scope *Scope, images tf.Output, size tf.Output, optional ...ResizeAreaAttr) (resized_images tf.Output) { +// epsilon: Ridge term. Must be a scalar. +// grad: The gradient. +// indices: A vector of indices into the first dimension of var, ms and mom. +// +// Returns the created operation. +func ResourceSparseApplyCenteredRMSProp(scope *Scope, var_ tf.Output, mg tf.Output, ms tf.Output, mom tf.Output, lr tf.Output, rho tf.Output, momentum tf.Output, epsilon tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyCenteredRMSPropAttr) (o *tf.Operation) { if scope.Err() != nil { return } @@ -9134,368 +9068,265 @@ func ResizeArea(scope *Scope, images tf.Output, size tf.Output, optional ...Resi a(attrs) } opspec := tf.OpSpec{ - Type: "ResizeArea", + Type: "ResourceSparseApplyCenteredRMSProp", Input: []tf.Input{ - images, size, + var_, mg, ms, mom, lr, rho, momentum, epsilon, grad, indices, }, Attrs: attrs, } - op := scope.AddOperation(opspec) - return op.Output(0) + return scope.AddOperation(opspec) } -// Pads a tensor with zeros. -// -// This operation pads a `input` with zeros according to the `paddings` you -// specify. `paddings` is an integer tensor with shape `[Dn, 2]`, where n is the -// rank of `input`. For each dimension D of `input`, `paddings[D, 0]` indicates -// how many zeros to add before the contents of `input` in that dimension, and -// `paddings[D, 1]` indicates how many zeros to add after the contents of `input` -// in that dimension. +// Creates a dataset that batches `batch_size` elements from `input_dataset`. // -// The padded size of each dimension D of the output is: +// Arguments: // -// `paddings(D, 0) + input.dim_size(D) + paddings(D, 1)` +// batch_size: A scalar representing the number of elements to accumulate in a +// batch. // -// For example: // -// ``` -// # 't' is [[1, 1], [2, 2]] -// # 'paddings' is [[1, 1], [2, 2]] -// # rank of 't' is 2 -// pad(t, paddings) ==> [[0, 0, 0, 0, 0, 0] -// [0, 0, 1, 1, 0, 0] -// [0, 0, 2, 2, 0, 0] -// [0, 0, 0, 0, 0, 0]] -// ``` -func Pad(scope *Scope, input tf.Output, paddings tf.Output) (output tf.Output) { +func BatchDataset(scope *Scope, input_dataset tf.Output, batch_size tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) { if scope.Err() != nil { return } + attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes} opspec := tf.OpSpec{ - Type: "Pad", + Type: "BatchDataset", Input: []tf.Input{ - input, paddings, + input_dataset, batch_size, }, + Attrs: attrs, } op := scope.AddOperation(opspec) return op.Output(0) } -// Checks whether a resource handle-based variable has been initialized. +// Inverse fast Fourier transform. +// +// Computes the inverse 1-dimensional discrete Fourier transform over the +// inner-most dimension of `input`. // // Arguments: -// resource: the input resource handle. +// input: A complex64 tensor. // -// Returns a scalar boolean which is true if the variable has been -// initialized. -func VarIsInitializedOp(scope *Scope, resource tf.Output) (is_initialized tf.Output) { +// Returns A complex64 tensor of the same shape as `input`. The inner-most +// dimension of `input` is replaced with its inverse 1D Fourier transform. +// +// @compatibility(numpy) +// Equivalent to np.fft.ifft +// @end_compatibility +func IFFT(scope *Scope, input tf.Output) (output tf.Output) { if scope.Err() != nil { return } opspec := tf.OpSpec{ - Type: "VarIsInitializedOp", + Type: "IFFT", Input: []tf.Input{ - resource, + input, }, } op := scope.AddOperation(opspec) return op.Output(0) } -// StatelessRandomUniformAttr is an optional argument to StatelessRandomUniform. -type StatelessRandomUniformAttr func(optionalAttr) - -// StatelessRandomUniformDtype sets the optional dtype attribute to value. +// Generates values in an interval. // -// value: The type of the output. -// If not specified, defaults to DT_FLOAT -func StatelessRandomUniformDtype(value tf.DataType) StatelessRandomUniformAttr { - return func(m optionalAttr) { - m["dtype"] = value - } -} - -// Outputs deterministic pseudorandom random values from a uniform distribution. +// A sequence of `num` evenly-spaced values are generated beginning at `start`. +// If `num > 1`, the values in the sequence increase by `stop - start / num - 1`, +// so that the last one is exactly `stop`. // -// The generated values follow a uniform distribution in the range `[0, 1)`. The -// lower bound 0 is included in the range, while the upper bound 1 is excluded. +// For example: // -// The outputs are a deterministic function of `shape` and `seed`. +// ``` +// tf.linspace(10.0, 12.0, 3, name="linspace") => [ 10.0 11.0 12.0] +// ``` // // Arguments: -// shape: The shape of the output tensor. -// seed: 2 seeds (shape [2]). +// start: First entry in the range. +// stop: Last entry in the range. +// num: Number of values to generate. // -// Returns Random values with specified shape. -func StatelessRandomUniform(scope *Scope, shape tf.Output, seed tf.Output, optional ...StatelessRandomUniformAttr) (output tf.Output) { +// Returns 1-D. The generated values. +func LinSpace(scope *Scope, start tf.Output, stop tf.Output, num tf.Output) (output tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } opspec := tf.OpSpec{ - Type: "StatelessRandomUniform", + Type: "LinSpace", Input: []tf.Input{ - shape, seed, + start, stop, num, }, - Attrs: attrs, } op := scope.AddOperation(opspec) return op.Output(0) } -// Makes its input available to the next iteration. +// DestroyResourceOpAttr is an optional argument to DestroyResourceOp. +type DestroyResourceOpAttr func(optionalAttr) + +// DestroyResourceOpIgnoreLookupError sets the optional ignore_lookup_error attribute to value. +// +// value: whether to ignore the error when the resource +// doesn't exist. +// If not specified, defaults to true +func DestroyResourceOpIgnoreLookupError(value bool) DestroyResourceOpAttr { + return func(m optionalAttr) { + m["ignore_lookup_error"] = value + } +} + +// Deletes the resource specified by the handle. +// +// All subsequent operations using the resource will result in a NotFound +// error status. // // Arguments: -// data: The tensor to be made available to the next iteration. +// resource: handle to the resource to delete. // -// Returns The same tensor as `data`. -func NextIteration(scope *Scope, data tf.Output) (output tf.Output) { +// Returns the created operation. +func DestroyResourceOp(scope *Scope, resource tf.Output, optional ...DestroyResourceOpAttr) (o *tf.Operation) { if scope.Err() != nil { return } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } opspec := tf.OpSpec{ - Type: "NextIteration", + Type: "DestroyResourceOp", Input: []tf.Input{ - data, + resource, }, + Attrs: attrs, } - op := scope.AddOperation(opspec) - return op.Output(0) + return scope.AddOperation(opspec) } -// Output a fact about factorials. -func Fact(scope *Scope) (fact tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "Fact", - } - op := scope.AddOperation(opspec) - return op.Output(0) -} +// LRNAttr is an optional argument to LRN. +type LRNAttr func(optionalAttr) -// AngleAttr is an optional argument to Angle. -type AngleAttr func(optionalAttr) - -// AngleTout sets the optional Tout attribute to value. -// If not specified, defaults to DT_FLOAT -func AngleTout(value tf.DataType) AngleAttr { +// LRNDepthRadius sets the optional depth_radius attribute to value. +// +// value: 0-D. Half-width of the 1-D normalization window. +// If not specified, defaults to 5 +func LRNDepthRadius(value int64) LRNAttr { return func(m optionalAttr) { - m["Tout"] = value + m["depth_radius"] = value } } -// Returns the argument of a complex number. -// -// Given a tensor `input` of complex numbers, this operation returns a tensor of -// type `float` that is the argument of each element in `input`. All elements in -// `input` must be complex numbers of the form \\(a + bj\\), where *a* -// is the real part and *b* is the imaginary part. -// -// The argument returned by this operation is of the form \\(atan2(b, a)\\). -// -// For example: -// -// ``` -// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j] -// tf.angle(input) ==> [2.0132, 1.056] -// ``` +// LRNBias sets the optional bias attribute to value. // -// @compatibility(numpy) -// Equivalent to np.angle. -// @end_compatibility -func Angle(scope *Scope, input tf.Output, optional ...AngleAttr) (output tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "Angle", - Input: []tf.Input{ - input, - }, - Attrs: attrs, +// value: An offset (usually positive to avoid dividing by 0). +// If not specified, defaults to 1 +func LRNBias(value float32) LRNAttr { + return func(m optionalAttr) { + m["bias"] = value } - op := scope.AddOperation(opspec) - return op.Output(0) } -// VarHandleOpAttr is an optional argument to VarHandleOp. -type VarHandleOpAttr func(optionalAttr) - -// VarHandleOpContainer sets the optional container attribute to value. +// LRNAlpha sets the optional alpha attribute to value. // -// value: the container this variable is placed in. -// If not specified, defaults to "" -func VarHandleOpContainer(value string) VarHandleOpAttr { +// value: A scale factor, usually positive. +// If not specified, defaults to 1 +func LRNAlpha(value float32) LRNAttr { return func(m optionalAttr) { - m["container"] = value + m["alpha"] = value } } -// VarHandleOpSharedName sets the optional shared_name attribute to value. +// LRNBeta sets the optional beta attribute to value. // -// value: the name by which this variable is referred to. -// If not specified, defaults to "" -func VarHandleOpSharedName(value string) VarHandleOpAttr { +// value: An exponent. +// If not specified, defaults to 0.5 +func LRNBeta(value float32) LRNAttr { return func(m optionalAttr) { - m["shared_name"] = value + m["beta"] = value } } -// Creates a handle to a Variable resource. +// Local Response Normalization. +// +// The 4-D `input` tensor is treated as a 3-D array of 1-D vectors (along the last +// dimension), and each vector is normalized independently. Within a given vector, +// each component is divided by the weighted, squared sum of inputs within +// `depth_radius`. In detail, +// +// sqr_sum[a, b, c, d] = +// sum(input[a, b, c, d - depth_radius : d + depth_radius + 1] ** 2) +// output = input / (bias + alpha * sqr_sum) ** beta +// +// For details, see [Krizhevsky et al., ImageNet classification with deep +// convolutional neural networks (NIPS 2012)](http://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks). // // Arguments: -// dtype: the type of this variable. Must agree with the dtypes -// of all ops using this variable. -// shape: The (possibly partially specified) shape of this variable. -func VarHandleOp(scope *Scope, dtype tf.DataType, shape tf.Shape, optional ...VarHandleOpAttr) (resource tf.Output) { +// input: 4-D. +func LRN(scope *Scope, input tf.Output, optional ...LRNAttr) (output tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{"dtype": dtype, "shape": shape} + attrs := map[string]interface{}{} for _, a := range optional { a(attrs) } opspec := tf.OpSpec{ - Type: "VarHandleOp", - - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// Elementwise computes the bitwise XOR of `x` and `y`. -// -// The result will have those bits set, that are different in `x` and `y`. The -// computation is performed on the underlying representations of `x` and `y`. -func BitwiseXor(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "BitwiseXor", + Type: "LRN", Input: []tf.Input{ - x, y, + input, }, + Attrs: attrs, } op := scope.AddOperation(opspec) return op.Output(0) } -// Deserialize `SparseTensor` objects. -// -// The input `serialized_sparse` must have the shape `[?, ?, ..., ?, 3]` where -// the last dimension stores serialized `SparseTensor` objects and the other N -// dimensions (N >= 0) correspond to a batch. The ranks of the original -// `SparseTensor` objects must all match. When the final `SparseTensor` is -// created, its rank is the rank of the incoming `SparseTensor` objects plus N; -// the sparse tensors have been concatenated along new dimensions, one for each -// batch. -// -// The output `SparseTensor` object's shape values for the original dimensions -// are the max across the input `SparseTensor` objects' shape values for the -// corresponding dimensions. The new dimensions match the size of the batch. -// -// The input `SparseTensor` objects' indices are assumed ordered in -// standard lexicographic order. If this is not the case, after this -// step run `SparseReorder` to restore index ordering. -// -// For example, if the serialized input is a `[2 x 3]` matrix representing two -// original `SparseTensor` objects: -// -// index = [ 0] -// [10] -// [20] -// values = [1, 2, 3] -// shape = [50] -// -// and -// -// index = [ 2] -// [10] -// values = [4, 5] -// shape = [30] -// -// then the final deserialized `SparseTensor` will be: -// -// index = [0 0] -// [0 10] -// [0 20] -// [1 2] -// [1 10] -// values = [1, 2, 3, 4, 5] -// shape = [2 50] -// -// Arguments: -// serialized_sparse: The serialized `SparseTensor` objects. The last dimension -// must have 3 columns. -// dtype: The `dtype` of the serialized `SparseTensor` objects. -func DeserializeSparse(scope *Scope, serialized_sparse tf.Output, dtype tf.DataType) (sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output) { +// Creates a dataset that zips together `input_datasets`. +func ZipDataset(scope *Scope, input_datasets []tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{"dtype": dtype} + attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes} opspec := tf.OpSpec{ - Type: "DeserializeSparse", + Type: "ZipDataset", Input: []tf.Input{ - serialized_sparse, + tf.OutputList(input_datasets), }, Attrs: attrs, } op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1), op.Output(2) + return op.Output(0) } -// ResourceApplyRMSPropAttr is an optional argument to ResourceApplyRMSProp. -type ResourceApplyRMSPropAttr func(optionalAttr) +// ResourceSparseApplyAdagradAttr is an optional argument to ResourceSparseApplyAdagrad. +type ResourceSparseApplyAdagradAttr func(optionalAttr) -// ResourceApplyRMSPropUseLocking sets the optional use_locking attribute to value. +// ResourceSparseApplyAdagradUseLocking sets the optional use_locking attribute to value. // -// value: If `True`, updating of the var, ms, and mom tensors is protected +// value: If `True`, updating of the var and accum tensors will be protected // by a lock; otherwise the behavior is undefined, but may exhibit less // contention. // If not specified, defaults to false -func ResourceApplyRMSPropUseLocking(value bool) ResourceApplyRMSPropAttr { +func ResourceSparseApplyAdagradUseLocking(value bool) ResourceSparseApplyAdagradAttr { return func(m optionalAttr) { m["use_locking"] = value } } -// Update '*var' according to the RMSProp algorithm. -// -// Note that in dense implementation of this algorithm, ms and mom will -// update even if the grad is zero, but in this sparse implementation, ms -// and mom will not update in iterations during which the grad is zero. -// -// mean_square = decay * mean_square + (1-decay) * gradient ** 2 -// Delta = learning_rate * gradient / sqrt(mean_square + epsilon) +// Update relevant entries in '*var' and '*accum' according to the adagrad scheme. // -// ms <- rho * ms_{t-1} + (1-rho) * grad * grad -// mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon) -// var <- var - mom +// That is for rows we have grad for, we update var and accum as follows: +// accum += grad * grad +// var -= lr * grad * (1 / sqrt(accum)) // // Arguments: // var_: Should be from a Variable(). -// ms: Should be from a Variable(). -// mom: Should be from a Variable(). -// lr: Scaling factor. Must be a scalar. -// rho: Decay rate. Must be a scalar. -// -// epsilon: Ridge term. Must be a scalar. +// accum: Should be from a Variable(). +// lr: Learning rate. Must be a scalar. // grad: The gradient. +// indices: A vector of indices into the first dimension of var and accum. // // Returns the created operation. -func ResourceApplyRMSProp(scope *Scope, var_ tf.Output, ms tf.Output, mom tf.Output, lr tf.Output, rho tf.Output, momentum tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyRMSPropAttr) (o *tf.Operation) { +func ResourceSparseApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyAdagradAttr) (o *tf.Operation) { if scope.Err() != nil { return } @@ -9504,138 +9335,87 @@ func ResourceApplyRMSProp(scope *Scope, var_ tf.Output, ms tf.Output, mom tf.Out a(attrs) } opspec := tf.OpSpec{ - Type: "ResourceApplyRMSProp", + Type: "ResourceSparseApplyAdagrad", Input: []tf.Input{ - var_, ms, mom, lr, rho, momentum, epsilon, grad, + var_, accum, lr, grad, indices, }, Attrs: attrs, } return scope.AddOperation(opspec) } -// ResourceScatterNdUpdateAttr is an optional argument to ResourceScatterNdUpdate. -type ResourceScatterNdUpdateAttr func(optionalAttr) - -// ResourceScatterNdUpdateUseLocking sets the optional use_locking attribute to value. +// 2D real-valued fast Fourier transform. // -// value: An optional bool. Defaults to True. If True, the assignment will -// be protected by a lock; otherwise the behavior is undefined, -// but may exhibit less contention. -// If not specified, defaults to true -func ResourceScatterNdUpdateUseLocking(value bool) ResourceScatterNdUpdateAttr { - return func(m optionalAttr) { - m["use_locking"] = value - } -} - -// Applies sparse `updates` to individual values or slices within a given +// Computes the 2-dimensional discrete Fourier transform of a real-valued signal +// over the inner-most 2 dimensions of `input`. // -// variable according to `indices`. +// Since the DFT of a real signal is Hermitian-symmetric, `RFFT2D` only returns the +// `fft_length / 2 + 1` unique components of the FFT for the inner-most dimension +// of `output`: the zero-frequency term, followed by the `fft_length / 2` +// positive-frequency terms. // -// `ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`. +// Along each axis `RFFT2D` is computed on, if `fft_length` is smaller than the +// corresponding dimension of `input`, the dimension is cropped. If it is larger, +// the dimension is padded with zeros. // -// `indices` must be integer tensor, containing indices into `ref`. -// It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`. +// Arguments: +// input: A float32 tensor. +// fft_length: An int32 tensor of shape [2]. The FFT length for each dimension. // -// The innermost dimension of `indices` (with length `K`) corresponds to -// indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th -// dimension of `ref`. +// Returns A complex64 tensor of the same rank as `input`. The inner-most 2 +// dimensions of `input` are replaced with their 2D Fourier transform. The +// inner-most dimension contains `fft_length / 2 + 1` unique frequency +// components. // -// `updates` is `Tensor` of rank `Q-1+P-K` with shape: -// -// ``` -// [d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]]. -// ``` -// -// For example, say we want to update 4 scattered elements to a rank-1 tensor to -// 8 elements. In Python, that update would look like this: -// -// ```python -// ref = tfe.Variable([1, 2, 3, 4, 5, 6, 7, 8]) -// indices = tf.constant([[4], [3], [1] ,[7]]) -// updates = tf.constant([9, 10, 11, 12]) -// update = tf.scatter_nd_update(ref, indices, updates) -// with tf.Session() as sess: -// print sess.run(update) -// ``` -// -// The resulting update to ref would look like this: -// -// [1, 11, 3, 10, 9, 6, 7, 12] -// -// See @{tf.scatter_nd} for more details about how to make updates to -// slices. -// -// Arguments: -// ref: A resource handle. Must be from a VarHandleOp. -// indices: A Tensor. Must be one of the following types: int32, int64. -// A tensor of indices into ref. -// updates: A Tensor. Must have the same type as ref. A tensor of updated -// values to add to ref. -// -// Returns the created operation. -func ResourceScatterNdUpdate(scope *Scope, ref tf.Output, indices tf.Output, updates tf.Output, optional ...ResourceScatterNdUpdateAttr) (o *tf.Operation) { +// @compatibility(numpy) +// Equivalent to np.fft.rfft2 +// @end_compatibility +func RFFT2D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } opspec := tf.OpSpec{ - Type: "ResourceScatterNdUpdate", + Type: "RFFT2D", Input: []tf.Input{ - ref, indices, updates, + input, fft_length, }, - Attrs: attrs, } - return scope.AddOperation(opspec) + op := scope.AddOperation(opspec) + return op.Output(0) } -// SqueezeAttr is an optional argument to Squeeze. -type SqueezeAttr func(optionalAttr) +// ResizeAreaAttr is an optional argument to ResizeArea. +type ResizeAreaAttr func(optionalAttr) -// SqueezeAxis sets the optional axis attribute to value. -// -// value: If specified, only squeezes the dimensions listed. The dimension -// index starts at 0. It is an error to squeeze a dimension that is not 1. Must -// be in the range `[-rank(input), rank(input))`. -// If not specified, defaults to <> +// ResizeAreaAlignCorners sets the optional align_corners attribute to value. // -// REQUIRES: len(value) >= 0 -func SqueezeAxis(value []int64) SqueezeAttr { +// value: If true, rescale input by (new_height - 1) / (height - 1), which +// exactly aligns the 4 corners of images and resized images. If false, rescale +// by new_height / height. Treat similarly the width dimension. +// If not specified, defaults to false +func ResizeAreaAlignCorners(value bool) ResizeAreaAttr { return func(m optionalAttr) { - m["squeeze_dims"] = value + m["align_corners"] = value } } -// Removes dimensions of size 1 from the shape of a tensor. -// -// Given a tensor `input`, this operation returns a tensor of the same type with -// all dimensions of size 1 removed. If you don't want to remove all size 1 -// dimensions, you can remove specific size 1 dimensions by specifying -// `axis`. -// -// For example: -// -// ``` -// # 't' is a tensor of shape [1, 2, 1, 3, 1, 1] -// shape(squeeze(t)) ==> [2, 3] -// ``` +// Resize `images` to `size` using area interpolation. // -// Or, to remove specific size 1 dimensions: +// Input images can be of different types but output images are always float. // -// ``` -// # 't' is a tensor of shape [1, 2, 1, 3, 1, 1] -// shape(squeeze(t, [2, 4])) ==> [1, 2, 3, 1] -// ``` +// Each output pixel is computed by first transforming the pixel's footprint into +// the input tensor and then averaging the pixels that intersect the footprint. An +// input pixel's contribution to the average is weighted by the fraction of its +// area that intersects the footprint. This is the same as OpenCV's INTER_AREA. // // Arguments: -// input: The `input` to squeeze. +// images: 4-D with shape `[batch, height, width, channels]`. +// size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`. The +// new size for the images. // -// Returns Contains the same data as `input`, but has one or more dimensions of -// size 1 removed. -func Squeeze(scope *Scope, input tf.Output, optional ...SqueezeAttr) (output tf.Output) { +// Returns 4-D with shape +// `[batch, new_height, new_width, channels]`. +func ResizeArea(scope *Scope, images tf.Output, size tf.Output, optional ...ResizeAreaAttr) (resized_images tf.Output) { if scope.Err() != nil { return } @@ -9644,9 +9424,9 @@ func Squeeze(scope *Scope, input tf.Output, optional ...SqueezeAttr) (output tf. a(attrs) } opspec := tf.OpSpec{ - Type: "Squeeze", + Type: "ResizeArea", Input: []tf.Input{ - input, + images, size, }, Attrs: attrs, } @@ -9654,98 +9434,91 @@ func Squeeze(scope *Scope, input tf.Output, optional ...SqueezeAttr) (output tf. return op.Output(0) } -// ResourceApplyAdadeltaAttr is an optional argument to ResourceApplyAdadelta. -type ResourceApplyAdadeltaAttr func(optionalAttr) - -// ResourceApplyAdadeltaUseLocking sets the optional use_locking attribute to value. +// Pads a tensor with zeros. // -// value: If True, updating of the var, accum and update_accum tensors will be protected by -// a lock; otherwise the behavior is undefined, but may exhibit less contention. -// If not specified, defaults to false -func ResourceApplyAdadeltaUseLocking(value bool) ResourceApplyAdadeltaAttr { - return func(m optionalAttr) { - m["use_locking"] = value +// This operation pads a `input` with zeros according to the `paddings` you +// specify. `paddings` is an integer tensor with shape `[Dn, 2]`, where n is the +// rank of `input`. For each dimension D of `input`, `paddings[D, 0]` indicates +// how many zeros to add before the contents of `input` in that dimension, and +// `paddings[D, 1]` indicates how many zeros to add after the contents of `input` +// in that dimension. +// +// The padded size of each dimension D of the output is: +// +// `paddings(D, 0) + input.dim_size(D) + paddings(D, 1)` +// +// For example: +// +// ``` +// # 't' is [[1, 1], [2, 2]] +// # 'paddings' is [[1, 1], [2, 2]] +// # rank of 't' is 2 +// pad(t, paddings) ==> [[0, 0, 0, 0, 0, 0] +// [0, 0, 1, 1, 0, 0] +// [0, 0, 2, 2, 0, 0] +// [0, 0, 0, 0, 0, 0]] +// ``` +func Pad(scope *Scope, input tf.Output, paddings tf.Output) (output tf.Output) { + if scope.Err() != nil { + return } + opspec := tf.OpSpec{ + Type: "Pad", + Input: []tf.Input{ + input, paddings, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) } -// Update '*var' according to the adadelta scheme. -// -// accum = rho() * accum + (1 - rho()) * grad.square(); -// update = (update_accum + epsilon).sqrt() * (accum + epsilon()).rsqrt() * grad; -// update_accum = rho() * update_accum + (1 - rho()) * update.square(); -// var -= update; +// Checks whether a resource handle-based variable has been initialized. // // Arguments: -// var_: Should be from a Variable(). -// accum: Should be from a Variable(). -// accum_update: Should be from a Variable(). -// lr: Scaling factor. Must be a scalar. -// rho: Decay factor. Must be a scalar. -// epsilon: Constant factor. Must be a scalar. -// grad: The gradient. +// resource: the input resource handle. // -// Returns the created operation. -func ResourceApplyAdadelta(scope *Scope, var_ tf.Output, accum tf.Output, accum_update tf.Output, lr tf.Output, rho tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyAdadeltaAttr) (o *tf.Operation) { +// Returns a scalar boolean which is true if the variable has been +// initialized. +func VarIsInitializedOp(scope *Scope, resource tf.Output) (is_initialized tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } opspec := tf.OpSpec{ - Type: "ResourceApplyAdadelta", + Type: "VarIsInitializedOp", Input: []tf.Input{ - var_, accum, accum_update, lr, rho, epsilon, grad, + resource, }, - Attrs: attrs, } - return scope.AddOperation(opspec) + op := scope.AddOperation(opspec) + return op.Output(0) } -// NonMaxSuppressionAttr is an optional argument to NonMaxSuppression. -type NonMaxSuppressionAttr func(optionalAttr) +// StatelessRandomUniformAttr is an optional argument to StatelessRandomUniform. +type StatelessRandomUniformAttr func(optionalAttr) -// NonMaxSuppressionIouThreshold sets the optional iou_threshold attribute to value. +// StatelessRandomUniformDtype sets the optional dtype attribute to value. // -// value: A float representing the threshold for deciding whether boxes -// overlap too much with respect to IOU. -// If not specified, defaults to 0.5 -func NonMaxSuppressionIouThreshold(value float32) NonMaxSuppressionAttr { +// value: The type of the output. +// If not specified, defaults to DT_FLOAT +func StatelessRandomUniformDtype(value tf.DataType) StatelessRandomUniformAttr { return func(m optionalAttr) { - m["iou_threshold"] = value + m["dtype"] = value } } -// Greedily selects a subset of bounding boxes in descending order of score, +// Outputs deterministic pseudorandom random values from a uniform distribution. // -// pruning away boxes that have high intersection-over-union (IOU) overlap -// with previously selected boxes. Bounding boxes are supplied as -// [y1, x1, y2, x2], where (y1, x1) and (y2, x2) are the coordinates of any -// diagonal pair of box corners and the coordinates can be provided as normalized -// (i.e., lying in the interval [0, 1]) or absolute. Note that this algorithm -// is agnostic to where the origin is in the coordinate system. Note that this -// algorithm is invariant to orthogonal transformations and translations -// of the coordinate system; thus translating or reflections of the coordinate -// system result in the same boxes being selected by the algorithm. -// The output of this operation is a set of integers indexing into the input -// collection of bounding boxes representing the selected boxes. The bounding -// box coordinates corresponding to the selected indices can then be obtained -// using the `tf.gather operation`. For example: -// selected_indices = tf.image.non_max_suppression( -// boxes, scores, max_output_size, iou_threshold) -// selected_boxes = tf.gather(boxes, selected_indices) +// The generated values follow a uniform distribution in the range `[0, 1)`. The +// lower bound 0 is included in the range, while the upper bound 1 is excluded. +// +// The outputs are a deterministic function of `shape` and `seed`. // // Arguments: -// boxes: A 2-D float tensor of shape `[num_boxes, 4]`. -// scores: A 1-D float tensor of shape `[num_boxes]` representing a single -// score corresponding to each box (each row of boxes). -// max_output_size: A scalar integer tensor representing the maximum number of -// boxes to be selected by non max suppression. +// shape: The shape of the output tensor. +// seed: 2 seeds (shape [2]). // -// Returns A 1-D integer tensor of shape `[M]` representing the selected -// indices from the boxes tensor, where `M <= max_output_size`. -func NonMaxSuppression(scope *Scope, boxes tf.Output, scores tf.Output, max_output_size tf.Output, optional ...NonMaxSuppressionAttr) (selected_indices tf.Output) { +// Returns Random values with specified shape. +func StatelessRandomUniform(scope *Scope, shape tf.Output, seed tf.Output, optional ...StatelessRandomUniformAttr) (output tf.Output) { if scope.Err() != nil { return } @@ -9754,9 +9527,9 @@ func NonMaxSuppression(scope *Scope, boxes tf.Output, scores tf.Output, max_outp a(attrs) } opspec := tf.OpSpec{ - Type: "NonMaxSuppression", + Type: "StatelessRandomUniform", Input: []tf.Input{ - boxes, scores, max_output_size, + shape, seed, }, Attrs: attrs, } @@ -9764,64 +9537,225 @@ func NonMaxSuppression(scope *Scope, boxes tf.Output, scores tf.Output, max_outp return op.Output(0) } -// Creates a dataset that emits `components` as a tuple of tensors once. -func TensorDataset(scope *Scope, components []tf.Output, output_shapes []tf.Shape) (handle tf.Output) { +// Makes its input available to the next iteration. +// +// Arguments: +// data: The tensor to be made available to the next iteration. +// +// Returns The same tensor as `data`. +func NextIteration(scope *Scope, data tf.Output) (output tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{"output_shapes": output_shapes} opspec := tf.OpSpec{ - Type: "TensorDataset", + Type: "NextIteration", Input: []tf.Input{ - tf.OutputList(components), + data, }, - Attrs: attrs, } op := scope.AddOperation(opspec) return op.Output(0) } -// Component-wise multiplies a SparseTensor by a dense Tensor. -// -// The output locations corresponding to the implicitly zero elements in the sparse -// tensor will be zero (i.e., will not take up storage space), regardless of the -// contents of the dense tensor (even if it's +/-INF and that INF*0 == NaN). +// Output a fact about factorials. +func Fact(scope *Scope) (fact tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "Fact", + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// AngleAttr is an optional argument to Angle. +type AngleAttr func(optionalAttr) + +// AngleTout sets the optional Tout attribute to value. +// If not specified, defaults to DT_FLOAT +func AngleTout(value tf.DataType) AngleAttr { + return func(m optionalAttr) { + m["Tout"] = value + } +} + +// Returns the argument of a complex number. // -// *Limitation*: this Op only broadcasts the dense side to the sparse side, but not -// the other direction. +// Given a tensor `input` of complex numbers, this operation returns a tensor of +// type `float` that is the argument of each element in `input`. All elements in +// `input` must be complex numbers of the form \\(a + bj\\), where *a* +// is the real part and *b* is the imaginary part. +// +// The argument returned by this operation is of the form \\(atan2(b, a)\\). +// +// For example: +// +// ``` +// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j] +// tf.angle(input) ==> [2.0132, 1.056] +// ``` +// +// @compatibility(numpy) +// Equivalent to np.angle. +// @end_compatibility +func Angle(scope *Scope, input tf.Output, optional ...AngleAttr) (output tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "Angle", + Input: []tf.Input{ + input, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// VarHandleOpAttr is an optional argument to VarHandleOp. +type VarHandleOpAttr func(optionalAttr) + +// VarHandleOpContainer sets the optional container attribute to value. +// +// value: the container this variable is placed in. +// If not specified, defaults to "" +func VarHandleOpContainer(value string) VarHandleOpAttr { + return func(m optionalAttr) { + m["container"] = value + } +} + +// VarHandleOpSharedName sets the optional shared_name attribute to value. +// +// value: the name by which this variable is referred to. +// If not specified, defaults to "" +func VarHandleOpSharedName(value string) VarHandleOpAttr { + return func(m optionalAttr) { + m["shared_name"] = value + } +} + +// Creates a handle to a Variable resource. // // Arguments: -// sp_indices: 2-D. `N x R` matrix with the indices of non-empty values in a -// SparseTensor, possibly not in canonical ordering. -// sp_values: 1-D. `N` non-empty values corresponding to `sp_indices`. -// sp_shape: 1-D. Shape of the input SparseTensor. -// dense: `R`-D. The dense Tensor operand. +// dtype: the type of this variable. Must agree with the dtypes +// of all ops using this variable. +// shape: The (possibly partially specified) shape of this variable. +func VarHandleOp(scope *Scope, dtype tf.DataType, shape tf.Shape, optional ...VarHandleOpAttr) (resource tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"dtype": dtype, "shape": shape} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "VarHandleOp", + + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Elementwise computes the bitwise XOR of `x` and `y`. // -// Returns 1-D. The `N` values that are operated on. -func SparseDenseCwiseMul(scope *Scope, sp_indices tf.Output, sp_values tf.Output, sp_shape tf.Output, dense tf.Output) (output tf.Output) { +// The result will have those bits set, that are different in `x` and `y`. The +// computation is performed on the underlying representations of `x` and `y`. +func BitwiseXor(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { if scope.Err() != nil { return } opspec := tf.OpSpec{ - Type: "SparseDenseCwiseMul", + Type: "BitwiseXor", Input: []tf.Input{ - sp_indices, sp_values, sp_shape, dense, + x, y, }, } op := scope.AddOperation(opspec) return op.Output(0) } -// ResourceSparseApplyRMSPropAttr is an optional argument to ResourceSparseApplyRMSProp. -type ResourceSparseApplyRMSPropAttr func(optionalAttr) +// Deserialize `SparseTensor` objects. +// +// The input `serialized_sparse` must have the shape `[?, ?, ..., ?, 3]` where +// the last dimension stores serialized `SparseTensor` objects and the other N +// dimensions (N >= 0) correspond to a batch. The ranks of the original +// `SparseTensor` objects must all match. When the final `SparseTensor` is +// created, its rank is the rank of the incoming `SparseTensor` objects plus N; +// the sparse tensors have been concatenated along new dimensions, one for each +// batch. +// +// The output `SparseTensor` object's shape values for the original dimensions +// are the max across the input `SparseTensor` objects' shape values for the +// corresponding dimensions. The new dimensions match the size of the batch. +// +// The input `SparseTensor` objects' indices are assumed ordered in +// standard lexicographic order. If this is not the case, after this +// step run `SparseReorder` to restore index ordering. +// +// For example, if the serialized input is a `[2 x 3]` matrix representing two +// original `SparseTensor` objects: +// +// index = [ 0] +// [10] +// [20] +// values = [1, 2, 3] +// shape = [50] +// +// and +// +// index = [ 2] +// [10] +// values = [4, 5] +// shape = [30] +// +// then the final deserialized `SparseTensor` will be: +// +// index = [0 0] +// [0 10] +// [0 20] +// [1 2] +// [1 10] +// values = [1, 2, 3, 4, 5] +// shape = [2 50] +// +// Arguments: +// serialized_sparse: The serialized `SparseTensor` objects. The last dimension +// must have 3 columns. +// dtype: The `dtype` of the serialized `SparseTensor` objects. +func DeserializeSparse(scope *Scope, serialized_sparse tf.Output, dtype tf.DataType) (sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"dtype": dtype} + opspec := tf.OpSpec{ + Type: "DeserializeSparse", + Input: []tf.Input{ + serialized_sparse, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0), op.Output(1), op.Output(2) +} -// ResourceSparseApplyRMSPropUseLocking sets the optional use_locking attribute to value. +// ResourceApplyRMSPropAttr is an optional argument to ResourceApplyRMSProp. +type ResourceApplyRMSPropAttr func(optionalAttr) + +// ResourceApplyRMSPropUseLocking sets the optional use_locking attribute to value. // // value: If `True`, updating of the var, ms, and mom tensors is protected // by a lock; otherwise the behavior is undefined, but may exhibit less // contention. // If not specified, defaults to false -func ResourceSparseApplyRMSPropUseLocking(value bool) ResourceSparseApplyRMSPropAttr { +func ResourceApplyRMSPropUseLocking(value bool) ResourceApplyRMSPropAttr { return func(m optionalAttr) { m["use_locking"] = value } @@ -9849,10 +9783,9 @@ func ResourceSparseApplyRMSPropUseLocking(value bool) ResourceSparseApplyRMSProp // // epsilon: Ridge term. Must be a scalar. // grad: The gradient. -// indices: A vector of indices into the first dimension of var, ms and mom. // // Returns the created operation. -func ResourceSparseApplyRMSProp(scope *Scope, var_ tf.Output, ms tf.Output, mom tf.Output, lr tf.Output, rho tf.Output, momentum tf.Output, epsilon tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyRMSPropAttr) (o *tf.Operation) { +func ResourceApplyRMSProp(scope *Scope, var_ tf.Output, ms tf.Output, mom tf.Output, lr tf.Output, rho tf.Output, momentum tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyRMSPropAttr) (o *tf.Operation) { if scope.Err() != nil { return } @@ -9861,168 +9794,77 @@ func ResourceSparseApplyRMSProp(scope *Scope, var_ tf.Output, ms tf.Output, mom a(attrs) } opspec := tf.OpSpec{ - Type: "ResourceSparseApplyRMSProp", + Type: "ResourceApplyRMSProp", Input: []tf.Input{ - var_, ms, mom, lr, rho, momentum, epsilon, grad, indices, + var_, ms, mom, lr, rho, momentum, epsilon, grad, }, Attrs: attrs, } return scope.AddOperation(opspec) } -// Returns the truth value of (x > y) element-wise. -// -// *NOTE*: `Greater` supports broadcasting. More about broadcasting -// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) -func Greater(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "Greater", - Input: []tf.Input{ - x, y, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// SampleDistortedBoundingBoxAttr is an optional argument to SampleDistortedBoundingBox. -type SampleDistortedBoundingBoxAttr func(optionalAttr) +// ResourceScatterNdUpdateAttr is an optional argument to ResourceScatterNdUpdate. +type ResourceScatterNdUpdateAttr func(optionalAttr) -// SampleDistortedBoundingBoxSeed sets the optional seed attribute to value. +// ResourceScatterNdUpdateUseLocking sets the optional use_locking attribute to value. // -// value: If either `seed` or `seed2` are set to non-zero, the random number -// generator is seeded by the given `seed`. Otherwise, it is seeded by a random -// seed. -// If not specified, defaults to 0 -func SampleDistortedBoundingBoxSeed(value int64) SampleDistortedBoundingBoxAttr { +// value: An optional bool. Defaults to True. If True, the assignment will +// be protected by a lock; otherwise the behavior is undefined, +// but may exhibit less contention. +// If not specified, defaults to true +func ResourceScatterNdUpdateUseLocking(value bool) ResourceScatterNdUpdateAttr { return func(m optionalAttr) { - m["seed"] = value + m["use_locking"] = value } } -// SampleDistortedBoundingBoxSeed2 sets the optional seed2 attribute to value. +// Applies sparse `updates` to individual values or slices within a given // -// value: A second seed to avoid seed collision. -// If not specified, defaults to 0 -func SampleDistortedBoundingBoxSeed2(value int64) SampleDistortedBoundingBoxAttr { - return func(m optionalAttr) { - m["seed2"] = value - } -} - -// SampleDistortedBoundingBoxMinObjectCovered sets the optional min_object_covered attribute to value. +// variable according to `indices`. // -// value: The cropped area of the image must contain at least this -// fraction of any bounding box supplied. The value of this parameter should be -// non-negative. In the case of 0, the cropped area does not need to overlap -// any of the bounding boxes supplied. -// If not specified, defaults to 0.1 -func SampleDistortedBoundingBoxMinObjectCovered(value float32) SampleDistortedBoundingBoxAttr { - return func(m optionalAttr) { - m["min_object_covered"] = value - } -} - -// SampleDistortedBoundingBoxAspectRatioRange sets the optional aspect_ratio_range attribute to value. -// -// value: The cropped area of the image must have an aspect ratio = -// width / height within this range. -// If not specified, defaults to -func SampleDistortedBoundingBoxAspectRatioRange(value []float32) SampleDistortedBoundingBoxAttr { - return func(m optionalAttr) { - m["aspect_ratio_range"] = value - } -} - -// SampleDistortedBoundingBoxAreaRange sets the optional area_range attribute to value. -// -// value: The cropped area of the image must contain a fraction of the -// supplied image within in this range. -// If not specified, defaults to -func SampleDistortedBoundingBoxAreaRange(value []float32) SampleDistortedBoundingBoxAttr { - return func(m optionalAttr) { - m["area_range"] = value - } -} - -// SampleDistortedBoundingBoxMaxAttempts sets the optional max_attempts attribute to value. -// -// value: Number of attempts at generating a cropped region of the image -// of the specified constraints. After `max_attempts` failures, return the entire -// image. -// If not specified, defaults to 100 -func SampleDistortedBoundingBoxMaxAttempts(value int64) SampleDistortedBoundingBoxAttr { - return func(m optionalAttr) { - m["max_attempts"] = value - } -} - -// SampleDistortedBoundingBoxUseImageIfNoBoundingBoxes sets the optional use_image_if_no_bounding_boxes attribute to value. +// `ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`. // -// value: Controls behavior if no bounding boxes supplied. -// If true, assume an implicit bounding box covering the whole input. If false, -// raise an error. -// If not specified, defaults to false -func SampleDistortedBoundingBoxUseImageIfNoBoundingBoxes(value bool) SampleDistortedBoundingBoxAttr { - return func(m optionalAttr) { - m["use_image_if_no_bounding_boxes"] = value - } -} - -// Generate a single randomly distorted bounding box for an image. +// `indices` must be integer tensor, containing indices into `ref`. +// It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`. // -// Bounding box annotations are often supplied in addition to ground-truth labels -// in image recognition or object localization tasks. A common technique for -// training such a system is to randomly distort an image while preserving -// its content, i.e. *data augmentation*. This Op outputs a randomly distorted -// localization of an object, i.e. bounding box, given an `image_size`, -// `bounding_boxes` and a series of constraints. +// The innermost dimension of `indices` (with length `K`) corresponds to +// indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th +// dimension of `ref`. // -// The output of this Op is a single bounding box that may be used to crop the -// original image. The output is returned as 3 tensors: `begin`, `size` and -// `bboxes`. The first 2 tensors can be fed directly into `tf.slice` to crop the -// image. The latter may be supplied to `tf.image.draw_bounding_boxes` to visualize -// what the bounding box looks like. +// `updates` is `Tensor` of rank `Q-1+P-K` with shape: // -// Bounding boxes are supplied and returned as `[y_min, x_min, y_max, x_max]`. The -// bounding box coordinates are floats in `[0.0, 1.0]` relative to the width and -// height of the underlying image. +// ``` +// [d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]]. +// ``` // -// For example, +// For example, say we want to update 4 scattered elements to a rank-1 tensor to +// 8 elements. In Python, that update would look like this: // // ```python -// # Generate a single distorted bounding box. -// begin, size, bbox_for_draw = tf.image.sample_distorted_bounding_box( -// tf.shape(image), -// bounding_boxes=bounding_boxes) +// ref = tfe.Variable([1, 2, 3, 4, 5, 6, 7, 8]) +// indices = tf.constant([[4], [3], [1] ,[7]]) +// updates = tf.constant([9, 10, 11, 12]) +// update = tf.scatter_nd_update(ref, indices, updates) +// with tf.Session() as sess: +// print sess.run(update) +// ``` // -// # Draw the bounding box in an image summary. -// image_with_box = tf.image.draw_bounding_boxes(tf.expand_dims(image, 0), -// bbox_for_draw) -// tf.summary.image('images_with_box', image_with_box) +// The resulting update to ref would look like this: // -// # Employ the bounding box to distort the image. -// distorted_image = tf.slice(image, begin, size) -// ``` +// [1, 11, 3, 10, 9, 6, 7, 12] // -// Note that if no bounding box information is available, setting -// `use_image_if_no_bounding_boxes = true` will assume there is a single implicit -// bounding box covering the whole image. If `use_image_if_no_bounding_boxes` is -// false and no bounding boxes are supplied, an error is raised. +// See @{tf.scatter_nd} for more details about how to make updates to +// slices. // // Arguments: -// image_size: 1-D, containing `[height, width, channels]`. -// bounding_boxes: 3-D with shape `[batch, N, 4]` describing the N bounding boxes -// associated with the image. +// ref: A resource handle. Must be from a VarHandleOp. +// indices: A Tensor. Must be one of the following types: int32, int64. +// A tensor of indices into ref. +// updates: A Tensor. Must have the same type as ref. A tensor of updated +// values to add to ref. // -// Returns 1-D, containing `[offset_height, offset_width, 0]`. Provide as input to -// `tf.slice`.1-D, containing `[target_height, target_width, -1]`. Provide as input to -// `tf.slice`.3-D with shape `[1, 1, 4]` containing the distorted bounding box. -// Provide as input to `tf.image.draw_bounding_boxes`. -func SampleDistortedBoundingBox(scope *Scope, image_size tf.Output, bounding_boxes tf.Output, optional ...SampleDistortedBoundingBoxAttr) (begin tf.Output, size tf.Output, bboxes tf.Output) { +// Returns the created operation. +func ResourceScatterNdUpdate(scope *Scope, ref tf.Output, indices tf.Output, updates tf.Output, optional ...ResourceScatterNdUpdateAttr) (o *tf.Operation) { if scope.Err() != nil { return } @@ -10031,37 +9873,68 @@ func SampleDistortedBoundingBox(scope *Scope, image_size tf.Output, bounding_box a(attrs) } opspec := tf.OpSpec{ - Type: "SampleDistortedBoundingBox", + Type: "ResourceScatterNdUpdate", Input: []tf.Input{ - image_size, bounding_boxes, + ref, indices, updates, }, Attrs: attrs, } - op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1), op.Output(2) + return scope.AddOperation(opspec) } -// Converts each string in the input Tensor to its hash mod by a number of buckets. +// SqueezeAttr is an optional argument to Squeeze. +type SqueezeAttr func(optionalAttr) + +// SqueezeAxis sets the optional axis attribute to value. // -// The hash function is deterministic on the content of the string within the -// process and will never change. However, it is not suitable for cryptography. -// This function may be used when CPU time is scarce and inputs are trusted or -// unimportant. There is a risk of adversaries constructing inputs that all hash -// to the same bucket. To prevent this problem, use a strong hash function with -// `tf.string_to_hash_bucket_strong`. +// value: If specified, only squeezes the dimensions listed. The dimension +// index starts at 0. It is an error to squeeze a dimension that is not 1. Must +// be in the range `[-rank(input), rank(input))`. +// If not specified, defaults to <> +// +// REQUIRES: len(value) >= 0 +func SqueezeAxis(value []int64) SqueezeAttr { + return func(m optionalAttr) { + m["squeeze_dims"] = value + } +} + +// Removes dimensions of size 1 from the shape of a tensor. +// +// Given a tensor `input`, this operation returns a tensor of the same type with +// all dimensions of size 1 removed. If you don't want to remove all size 1 +// dimensions, you can remove specific size 1 dimensions by specifying +// `axis`. +// +// For example: +// +// ``` +// # 't' is a tensor of shape [1, 2, 1, 3, 1, 1] +// shape(squeeze(t)) ==> [2, 3] +// ``` +// +// Or, to remove specific size 1 dimensions: +// +// ``` +// # 't' is a tensor of shape [1, 2, 1, 3, 1, 1] +// shape(squeeze(t, [2, 4])) ==> [1, 2, 3, 1] +// ``` // // Arguments: -// input: The strings to assign a hash bucket. -// num_buckets: The number of buckets. +// input: The `input` to squeeze. // -// Returns A Tensor of the same shape as the input `string_tensor`. -func StringToHashBucketFast(scope *Scope, input tf.Output, num_buckets int64) (output tf.Output) { +// Returns Contains the same data as `input`, but has one or more dimensions of +// size 1 removed. +func Squeeze(scope *Scope, input tf.Output, optional ...SqueezeAttr) (output tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{"num_buckets": num_buckets} + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } opspec := tf.OpSpec{ - Type: "StringToHashBucketFast", + Type: "Squeeze", Input: []tf.Input{ input, }, @@ -10071,143 +9944,126 @@ func StringToHashBucketFast(scope *Scope, input tf.Output, num_buckets int64) (o return op.Output(0) } -// TensorArrayGatherV3Attr is an optional argument to TensorArrayGatherV3. -type TensorArrayGatherV3Attr func(optionalAttr) +// ResourceApplyAdadeltaAttr is an optional argument to ResourceApplyAdadelta. +type ResourceApplyAdadeltaAttr func(optionalAttr) -// TensorArrayGatherV3ElementShape sets the optional element_shape attribute to value. +// ResourceApplyAdadeltaUseLocking sets the optional use_locking attribute to value. // -// value: The expected shape of an element, if known. Used to -// validate the shapes of TensorArray elements. If this shape is not -// fully specified, gathering zero-size TensorArrays is an error. -// If not specified, defaults to -func TensorArrayGatherV3ElementShape(value tf.Shape) TensorArrayGatherV3Attr { +// value: If True, updating of the var, accum and update_accum tensors will be protected by +// a lock; otherwise the behavior is undefined, but may exhibit less contention. +// If not specified, defaults to false +func ResourceApplyAdadeltaUseLocking(value bool) ResourceApplyAdadeltaAttr { return func(m optionalAttr) { - m["element_shape"] = value + m["use_locking"] = value } } -// Gather specific elements from the TensorArray into output `value`. +// Update '*var' according to the adadelta scheme. // -// All elements selected by `indices` must have the same shape. +// accum = rho() * accum + (1 - rho()) * grad.square(); +// update = (update_accum + epsilon).sqrt() * (accum + epsilon()).rsqrt() * grad; +// update_accum = rho() * update_accum + (1 - rho()) * update.square(); +// var -= update; // // Arguments: -// handle: The handle to a TensorArray. -// indices: The locations in the TensorArray from which to read tensor elements. -// flow_in: A float scalar that enforces proper chaining of operations. -// dtype: The type of the elem that is returned. +// var_: Should be from a Variable(). +// accum: Should be from a Variable(). +// accum_update: Should be from a Variable(). +// lr: Scaling factor. Must be a scalar. +// rho: Decay factor. Must be a scalar. +// epsilon: Constant factor. Must be a scalar. +// grad: The gradient. // -// Returns All of the elements in the TensorArray, concatenated along a new -// axis (the new dimension 0). -func TensorArrayGatherV3(scope *Scope, handle tf.Output, indices tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayGatherV3Attr) (value tf.Output) { +// Returns the created operation. +func ResourceApplyAdadelta(scope *Scope, var_ tf.Output, accum tf.Output, accum_update tf.Output, lr tf.Output, rho tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyAdadeltaAttr) (o *tf.Operation) { if scope.Err() != nil { return } - attrs := map[string]interface{}{"dtype": dtype} + attrs := map[string]interface{}{} for _, a := range optional { a(attrs) } opspec := tf.OpSpec{ - Type: "TensorArrayGatherV3", + Type: "ResourceApplyAdadelta", Input: []tf.Input{ - handle, indices, flow_in, + var_, accum, accum_update, lr, rho, epsilon, grad, }, Attrs: attrs, } - op := scope.AddOperation(opspec) - return op.Output(0) + return scope.AddOperation(opspec) } -// Returns x / y element-wise for integer types. -// -// Truncation designates that negative numbers will round fractional quantities -// toward zero. I.e. -7 / 5 = -1. This matches C semantics but it is different -// than Python semantics. See `FloorDiv` for a division function that matches -// Python Semantics. +// NonMaxSuppressionAttr is an optional argument to NonMaxSuppression. +type NonMaxSuppressionAttr func(optionalAttr) + +// NonMaxSuppressionIouThreshold sets the optional iou_threshold attribute to value. // -// *NOTE*: `TruncateDiv` supports broadcasting. More about broadcasting -// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) -func TruncateDiv(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { - if scope.Err() != nil { - return +// value: A float representing the threshold for deciding whether boxes +// overlap too much with respect to IOU. +// If not specified, defaults to 0.5 +func NonMaxSuppressionIouThreshold(value float32) NonMaxSuppressionAttr { + return func(m optionalAttr) { + m["iou_threshold"] = value } - opspec := tf.OpSpec{ - Type: "TruncateDiv", - Input: []tf.Input{ - x, y, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) } -// Restores tensors from a V2 checkpoint. -// -// For backward compatibility with the V1 format, this Op currently allows -// restoring from a V1 checkpoint as well: -// - This Op first attempts to find the V2 index file pointed to by "prefix", and -// if found proceed to read it as a V2 checkpoint; -// - Otherwise the V1 read path is invoked. -// Relying on this behavior is not recommended, as the ability to fall back to read -// V1 might be deprecated and eventually removed. -// -// By default, restores the named tensors in full. If the caller wishes to restore -// specific slices of stored tensors, "shape_and_slices" should be non-empty -// strings and correspondingly well-formed. +// Greedily selects a subset of bounding boxes in descending order of score, // -// Callers must ensure all the named tensors are indeed stored in the checkpoint. +// pruning away boxes that have high intersection-over-union (IOU) overlap +// with previously selected boxes. Bounding boxes are supplied as +// [y1, x1, y2, x2], where (y1, x1) and (y2, x2) are the coordinates of any +// diagonal pair of box corners and the coordinates can be provided as normalized +// (i.e., lying in the interval [0, 1]) or absolute. Note that this algorithm +// is agnostic to where the origin is in the coordinate system. Note that this +// algorithm is invariant to orthogonal transformations and translations +// of the coordinate system; thus translating or reflections of the coordinate +// system result in the same boxes being selected by the algorithm. +// The output of this operation is a set of integers indexing into the input +// collection of bounding boxes representing the selected boxes. The bounding +// box coordinates corresponding to the selected indices can then be obtained +// using the `tf.gather operation`. For example: +// selected_indices = tf.image.non_max_suppression( +// boxes, scores, max_output_size, iou_threshold) +// selected_boxes = tf.gather(boxes, selected_indices) // // Arguments: -// prefix: Must have a single element. The prefix of a V2 checkpoint. -// tensor_names: shape {N}. The names of the tensors to be restored. -// shape_and_slices: shape {N}. The slice specs of the tensors to be restored. -// Empty strings indicate that they are non-partitioned tensors. -// dtypes: shape {N}. The list of expected dtype for the tensors. Must match -// those stored in the checkpoint. +// boxes: A 2-D float tensor of shape `[num_boxes, 4]`. +// scores: A 1-D float tensor of shape `[num_boxes]` representing a single +// score corresponding to each box (each row of boxes). +// max_output_size: A scalar integer tensor representing the maximum number of +// boxes to be selected by non max suppression. // -// Returns shape {N}. The restored tensors, whose shapes are read from the -// checkpoint directly. -func RestoreV2(scope *Scope, prefix tf.Output, tensor_names tf.Output, shape_and_slices tf.Output, dtypes []tf.DataType) (tensors []tf.Output) { +// Returns A 1-D integer tensor of shape `[M]` representing the selected +// indices from the boxes tensor, where `M <= max_output_size`. +func NonMaxSuppression(scope *Scope, boxes tf.Output, scores tf.Output, max_output_size tf.Output, optional ...NonMaxSuppressionAttr) (selected_indices tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{"dtypes": dtypes} + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } opspec := tf.OpSpec{ - Type: "RestoreV2", + Type: "NonMaxSuppression", Input: []tf.Input{ - prefix, tensor_names, shape_and_slices, + boxes, scores, max_output_size, }, Attrs: attrs, } op := scope.AddOperation(opspec) - if scope.Err() != nil { - return - } - var idx int - var err error - if tensors, idx, err = makeOutputList(op, idx, "tensors"); err != nil { - scope.UpdateErr("RestoreV2", err) - return - } - return tensors + return op.Output(0) } -// Creates a dataset that skips `count` elements from the `input_dataset`. -// -// Arguments: -// -// count: A scalar representing the number of elements from the `input_dataset` -// that should be skipped. If count is -1, skips everything. -// -// -func SkipDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) { +// Creates a dataset that emits `components` as a tuple of tensors once. +func TensorDataset(scope *Scope, components []tf.Output, output_shapes []tf.Shape) (handle tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes} + attrs := map[string]interface{}{"output_shapes": output_shapes} opspec := tf.OpSpec{ - Type: "SkipDataset", + Type: "TensorDataset", Input: []tf.Input{ - input_dataset, count, + tf.OutputList(components), }, Attrs: attrs, } @@ -10215,235 +10071,248 @@ func SkipDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_ return op.Output(0) } -// Computes the maximum along segments of a tensor. -// -// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of -// segments. -// -// Computes a tensor such that -// \\(output_i = \max_j(data_j)\\) where `max` is over `j` such -// that `segment_ids[j] == i`. +// Component-wise multiplies a SparseTensor by a dense Tensor. // -// If the max is empty for a given segment ID `i`, `output[i] = 0`. +// The output locations corresponding to the implicitly zero elements in the sparse +// tensor will be zero (i.e., will not take up storage space), regardless of the +// contents of the dense tensor (even if it's +/-INF and that INF*0 == NaN). // -//
-// -//
+// *Limitation*: this Op only broadcasts the dense side to the sparse side, but not +// the other direction. // // Arguments: +// sp_indices: 2-D. `N x R` matrix with the indices of non-empty values in a +// SparseTensor, possibly not in canonical ordering. +// sp_values: 1-D. `N` non-empty values corresponding to `sp_indices`. +// sp_shape: 1-D. Shape of the input SparseTensor. +// dense: `R`-D. The dense Tensor operand. // -// segment_ids: A 1-D tensor whose rank is equal to the rank of `data`'s -// first dimension. Values should be sorted and can be repeated. -// -// Returns Has same shape as data, except for dimension 0 which -// has size `k`, the number of segments. -func SegmentMax(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) { +// Returns 1-D. The `N` values that are operated on. +func SparseDenseCwiseMul(scope *Scope, sp_indices tf.Output, sp_values tf.Output, sp_shape tf.Output, dense tf.Output) (output tf.Output) { if scope.Err() != nil { return } opspec := tf.OpSpec{ - Type: "SegmentMax", + Type: "SparseDenseCwiseMul", Input: []tf.Input{ - data, segment_ids, + sp_indices, sp_values, sp_shape, dense, }, } op := scope.AddOperation(opspec) return op.Output(0) } -// Computes hyperbolic tangent of `x` element-wise. -func Tanh(scope *Scope, x tf.Output) (y tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "Tanh", - Input: []tf.Input{ - x, - }, +// ResourceSparseApplyRMSPropAttr is an optional argument to ResourceSparseApplyRMSProp. +type ResourceSparseApplyRMSPropAttr func(optionalAttr) + +// ResourceSparseApplyRMSPropUseLocking sets the optional use_locking attribute to value. +// +// value: If `True`, updating of the var, ms, and mom tensors is protected +// by a lock; otherwise the behavior is undefined, but may exhibit less +// contention. +// If not specified, defaults to false +func ResourceSparseApplyRMSPropUseLocking(value bool) ResourceSparseApplyRMSPropAttr { + return func(m optionalAttr) { + m["use_locking"] = value } - op := scope.AddOperation(opspec) - return op.Output(0) } -// Decode web-safe base64-encoded strings. +// Update '*var' according to the RMSProp algorithm. // -// Input may or may not have padding at the end. See EncodeBase64 for padding. -// Web-safe means that input must use - and _ instead of + and /. +// Note that in dense implementation of this algorithm, ms and mom will +// update even if the grad is zero, but in this sparse implementation, ms +// and mom will not update in iterations during which the grad is zero. +// +// mean_square = decay * mean_square + (1-decay) * gradient ** 2 +// Delta = learning_rate * gradient / sqrt(mean_square + epsilon) +// +// ms <- rho * ms_{t-1} + (1-rho) * grad * grad +// mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon) +// var <- var - mom // // Arguments: -// input: Base64 strings to decode. +// var_: Should be from a Variable(). +// ms: Should be from a Variable(). +// mom: Should be from a Variable(). +// lr: Scaling factor. Must be a scalar. +// rho: Decay rate. Must be a scalar. // -// Returns Decoded strings. -func DecodeBase64(scope *Scope, input tf.Output) (output tf.Output) { +// epsilon: Ridge term. Must be a scalar. +// grad: The gradient. +// indices: A vector of indices into the first dimension of var, ms and mom. +// +// Returns the created operation. +func ResourceSparseApplyRMSProp(scope *Scope, var_ tf.Output, ms tf.Output, mom tf.Output, lr tf.Output, rho tf.Output, momentum tf.Output, epsilon tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyRMSPropAttr) (o *tf.Operation) { if scope.Err() != nil { return } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } opspec := tf.OpSpec{ - Type: "DecodeBase64", + Type: "ResourceSparseApplyRMSProp", Input: []tf.Input{ - input, + var_, ms, mom, lr, rho, momentum, epsilon, grad, indices, }, + Attrs: attrs, } - op := scope.AddOperation(opspec) - return op.Output(0) + return scope.AddOperation(opspec) } -// Store the input tensor in the state of the current session. -// -// Arguments: -// value: The tensor to be stored. +// Returns the truth value of (x > y) element-wise. // -// Returns The handle for the tensor stored in the session state, represented -// as a string. -func GetSessionHandle(scope *Scope, value tf.Output) (handle tf.Output) { +// *NOTE*: `Greater` supports broadcasting. More about broadcasting +// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) +func Greater(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { if scope.Err() != nil { return } opspec := tf.OpSpec{ - Type: "GetSessionHandle", + Type: "Greater", Input: []tf.Input{ - value, + x, y, }, } op := scope.AddOperation(opspec) return op.Output(0) } -// ResourceSparseApplyProximalAdagradAttr is an optional argument to ResourceSparseApplyProximalAdagrad. -type ResourceSparseApplyProximalAdagradAttr func(optionalAttr) +// SampleDistortedBoundingBoxAttr is an optional argument to SampleDistortedBoundingBox. +type SampleDistortedBoundingBoxAttr func(optionalAttr) -// ResourceSparseApplyProximalAdagradUseLocking sets the optional use_locking attribute to value. +// SampleDistortedBoundingBoxSeed sets the optional seed attribute to value. // -// value: If True, updating of the var and accum tensors will be protected by -// a lock; otherwise the behavior is undefined, but may exhibit less contention. -// If not specified, defaults to false -func ResourceSparseApplyProximalAdagradUseLocking(value bool) ResourceSparseApplyProximalAdagradAttr { +// value: If either `seed` or `seed2` are set to non-zero, the random number +// generator is seeded by the given `seed`. Otherwise, it is seeded by a random +// seed. +// If not specified, defaults to 0 +func SampleDistortedBoundingBoxSeed(value int64) SampleDistortedBoundingBoxAttr { return func(m optionalAttr) { - m["use_locking"] = value + m["seed"] = value } } -// Sparse update entries in '*var' and '*accum' according to FOBOS algorithm. -// -// That is for rows we have grad for, we update var and accum as follows: -// accum += grad * grad -// prox_v = var -// prox_v -= lr * grad * (1 / sqrt(accum)) -// var = sign(prox_v)/(1+lr*l2) * max{|prox_v|-lr*l1,0} -// -// Arguments: -// var_: Should be from a Variable(). -// accum: Should be from a Variable(). -// lr: Learning rate. Must be a scalar. -// l1: L1 regularization. Must be a scalar. -// l2: L2 regularization. Must be a scalar. -// grad: The gradient. -// indices: A vector of indices into the first dimension of var and accum. +// SampleDistortedBoundingBoxSeed2 sets the optional seed2 attribute to value. // -// Returns the created operation. -func ResourceSparseApplyProximalAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyProximalAdagradAttr) (o *tf.Operation) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) +// value: A second seed to avoid seed collision. +// If not specified, defaults to 0 +func SampleDistortedBoundingBoxSeed2(value int64) SampleDistortedBoundingBoxAttr { + return func(m optionalAttr) { + m["seed2"] = value } - opspec := tf.OpSpec{ - Type: "ResourceSparseApplyProximalAdagrad", - Input: []tf.Input{ - var_, accum, lr, l1, l2, grad, indices, - }, - Attrs: attrs, - } - return scope.AddOperation(opspec) } -// MaxPool3DGradAttr is an optional argument to MaxPool3DGrad. -type MaxPool3DGradAttr func(optionalAttr) - -// MaxPool3DGradDataFormat sets the optional data_format attribute to value. +// SampleDistortedBoundingBoxMinObjectCovered sets the optional min_object_covered attribute to value. // -// value: The data format of the input and output data. With the -// default format "NDHWC", the data is stored in the order of: -// [batch, in_depth, in_height, in_width, in_channels]. -// Alternatively, the format could be "NCDHW", the data storage order is: -// [batch, in_channels, in_depth, in_height, in_width]. -// If not specified, defaults to "NDHWC" -func MaxPool3DGradDataFormat(value string) MaxPool3DGradAttr { +// value: The cropped area of the image must contain at least this +// fraction of any bounding box supplied. The value of this parameter should be +// non-negative. In the case of 0, the cropped area does not need to overlap +// any of the bounding boxes supplied. +// If not specified, defaults to 0.1 +func SampleDistortedBoundingBoxMinObjectCovered(value float32) SampleDistortedBoundingBoxAttr { return func(m optionalAttr) { - m["data_format"] = value + m["min_object_covered"] = value } } -// Computes gradients of max pooling function. +// SampleDistortedBoundingBoxAspectRatioRange sets the optional aspect_ratio_range attribute to value. // -// Arguments: -// orig_input: The original input tensor. -// orig_output: The original output tensor. -// grad: Output backprop of shape `[batch, depth, rows, cols, channels]`. -// ksize: 1-D tensor of length 5. The size of the window for each dimension of -// the input tensor. Must have `ksize[0] = ksize[4] = 1`. -// strides: 1-D tensor of length 5. The stride of the sliding window for each -// dimension of `input`. Must have `strides[0] = strides[4] = 1`. -// padding: The type of padding algorithm to use. -func MaxPool3DGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPool3DGradAttr) (output tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding} - for _, a := range optional { - a(attrs) +// value: The cropped area of the image must have an aspect ratio = +// width / height within this range. +// If not specified, defaults to +func SampleDistortedBoundingBoxAspectRatioRange(value []float32) SampleDistortedBoundingBoxAttr { + return func(m optionalAttr) { + m["aspect_ratio_range"] = value } - opspec := tf.OpSpec{ - Type: "MaxPool3DGrad", - Input: []tf.Input{ - orig_input, orig_output, grad, - }, - Attrs: attrs, +} + +// SampleDistortedBoundingBoxAreaRange sets the optional area_range attribute to value. +// +// value: The cropped area of the image must contain a fraction of the +// supplied image within in this range. +// If not specified, defaults to +func SampleDistortedBoundingBoxAreaRange(value []float32) SampleDistortedBoundingBoxAttr { + return func(m optionalAttr) { + m["area_range"] = value } - op := scope.AddOperation(opspec) - return op.Output(0) } -// SparseReduceSumAttr is an optional argument to SparseReduceSum. -type SparseReduceSumAttr func(optionalAttr) +// SampleDistortedBoundingBoxMaxAttempts sets the optional max_attempts attribute to value. +// +// value: Number of attempts at generating a cropped region of the image +// of the specified constraints. After `max_attempts` failures, return the entire +// image. +// If not specified, defaults to 100 +func SampleDistortedBoundingBoxMaxAttempts(value int64) SampleDistortedBoundingBoxAttr { + return func(m optionalAttr) { + m["max_attempts"] = value + } +} -// SparseReduceSumKeepDims sets the optional keep_dims attribute to value. +// SampleDistortedBoundingBoxUseImageIfNoBoundingBoxes sets the optional use_image_if_no_bounding_boxes attribute to value. // -// value: If true, retain reduced dimensions with length 1. +// value: Controls behavior if no bounding boxes supplied. +// If true, assume an implicit bounding box covering the whole input. If false, +// raise an error. // If not specified, defaults to false -func SparseReduceSumKeepDims(value bool) SparseReduceSumAttr { +func SampleDistortedBoundingBoxUseImageIfNoBoundingBoxes(value bool) SampleDistortedBoundingBoxAttr { return func(m optionalAttr) { - m["keep_dims"] = value + m["use_image_if_no_bounding_boxes"] = value } } -// Computes the sum of elements across dimensions of a SparseTensor. +// Generate a single randomly distorted bounding box for an image. // -// This Op takes a SparseTensor and is the sparse counterpart to -// `tf.reduce_sum()`. In particular, this Op also returns a dense `Tensor` -// instead of a sparse one. +// Bounding box annotations are often supplied in addition to ground-truth labels +// in image recognition or object localization tasks. A common technique for +// training such a system is to randomly distort an image while preserving +// its content, i.e. *data augmentation*. This Op outputs a randomly distorted +// localization of an object, i.e. bounding box, given an `image_size`, +// `bounding_boxes` and a series of constraints. // -// Reduces `sp_input` along the dimensions given in `reduction_axes`. Unless -// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in -// `reduction_axes`. If `keep_dims` is true, the reduced dimensions are retained -// with length 1. +// The output of this Op is a single bounding box that may be used to crop the +// original image. The output is returned as 3 tensors: `begin`, `size` and +// `bboxes`. The first 2 tensors can be fed directly into `tf.slice` to crop the +// image. The latter may be supplied to `tf.image.draw_bounding_boxes` to visualize +// what the bounding box looks like. // -// If `reduction_axes` has no entries, all dimensions are reduced, and a tensor -// with a single element is returned. Additionally, the axes can be negative, -// which are interpreted according to the indexing rules in Python. +// Bounding boxes are supplied and returned as `[y_min, x_min, y_max, x_max]`. The +// bounding box coordinates are floats in `[0.0, 1.0]` relative to the width and +// height of the underlying image. +// +// For example, +// +// ```python +// # Generate a single distorted bounding box. +// begin, size, bbox_for_draw = tf.image.sample_distorted_bounding_box( +// tf.shape(image), +// bounding_boxes=bounding_boxes) +// +// # Draw the bounding box in an image summary. +// image_with_box = tf.image.draw_bounding_boxes(tf.expand_dims(image, 0), +// bbox_for_draw) +// tf.summary.image('images_with_box', image_with_box) +// +// # Employ the bounding box to distort the image. +// distorted_image = tf.slice(image, begin, size) +// ``` +// +// Note that if no bounding box information is available, setting +// `use_image_if_no_bounding_boxes = true` will assume there is a single implicit +// bounding box covering the whole image. If `use_image_if_no_bounding_boxes` is +// false and no bounding boxes are supplied, an error is raised. // // Arguments: -// input_indices: 2-D. `N x R` matrix with the indices of non-empty values in a -// SparseTensor, possibly not in canonical ordering. -// input_values: 1-D. `N` non-empty values corresponding to `input_indices`. -// input_shape: 1-D. Shape of the input SparseTensor. -// reduction_axes: 1-D. Length-`K` vector containing the reduction axes. +// image_size: 1-D, containing `[height, width, channels]`. +// bounding_boxes: 3-D with shape `[batch, N, 4]` describing the N bounding boxes +// associated with the image. // -// Returns `R-K`-D. The reduced Tensor. -func SparseReduceSum(scope *Scope, input_indices tf.Output, input_values tf.Output, input_shape tf.Output, reduction_axes tf.Output, optional ...SparseReduceSumAttr) (output tf.Output) { +// Returns 1-D, containing `[offset_height, offset_width, 0]`. Provide as input to +// `tf.slice`.1-D, containing `[target_height, target_width, -1]`. Provide as input to +// `tf.slice`.3-D with shape `[1, 1, 4]` containing the distorted bounding box. +// Provide as input to `tf.image.draw_bounding_boxes`. +func SampleDistortedBoundingBox(scope *Scope, image_size tf.Output, bounding_boxes tf.Output, optional ...SampleDistortedBoundingBoxAttr) (begin tf.Output, size tf.Output, bboxes tf.Output) { if scope.Err() != nil { return } @@ -10452,187 +10321,375 @@ func SparseReduceSum(scope *Scope, input_indices tf.Output, input_values tf.Outp a(attrs) } opspec := tf.OpSpec{ - Type: "SparseReduceSum", + Type: "SampleDistortedBoundingBox", Input: []tf.Input{ - input_indices, input_values, input_shape, reduction_axes, + image_size, bounding_boxes, }, Attrs: attrs, } op := scope.AddOperation(opspec) - return op.Output(0) + return op.Output(0), op.Output(1), op.Output(2) } -// Returns element-wise remainder of division. This emulates C semantics in that +// Converts each string in the input Tensor to its hash mod by a number of buckets. // -// the result here is consistent with a truncating divide. E.g. `truncate(x / y) * -// y + truncate_mod(x, y) = x`. +// The hash function is deterministic on the content of the string within the +// process and will never change. However, it is not suitable for cryptography. +// This function may be used when CPU time is scarce and inputs are trusted or +// unimportant. There is a risk of adversaries constructing inputs that all hash +// to the same bucket. To prevent this problem, use a strong hash function with +// `tf.string_to_hash_bucket_strong`. // -// *NOTE*: `TruncateMod` supports broadcasting. More about broadcasting -// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) -func TruncateMod(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { +// Arguments: +// input: The strings to assign a hash bucket. +// num_buckets: The number of buckets. +// +// Returns A Tensor of the same shape as the input `string_tensor`. +func StringToHashBucketFast(scope *Scope, input tf.Output, num_buckets int64) (output tf.Output) { if scope.Err() != nil { return } + attrs := map[string]interface{}{"num_buckets": num_buckets} opspec := tf.OpSpec{ - Type: "TruncateMod", + Type: "StringToHashBucketFast", Input: []tf.Input{ - x, y, + input, }, + Attrs: attrs, } op := scope.AddOperation(opspec) return op.Output(0) } -// Inverse 2D real-valued fast Fourier transform. -// -// Computes the inverse 2-dimensional discrete Fourier transform of a real-valued -// signal over the inner-most 2 dimensions of `input`. +// TensorArrayGatherV3Attr is an optional argument to TensorArrayGatherV3. +type TensorArrayGatherV3Attr func(optionalAttr) + +// TensorArrayGatherV3ElementShape sets the optional element_shape attribute to value. // -// The inner-most 2 dimensions of `input` are assumed to be the result of `RFFT2D`: -// The inner-most dimension contains the `fft_length / 2 + 1` unique components of -// the DFT of a real-valued signal. If `fft_length` is not provided, it is computed -// from the size of the inner-most 2 dimensions of `input`. If the FFT length used -// to compute `input` is odd, it should be provided since it cannot be inferred -// properly. +// value: The expected shape of an element, if known. Used to +// validate the shapes of TensorArray elements. If this shape is not +// fully specified, gathering zero-size TensorArrays is an error. +// If not specified, defaults to +func TensorArrayGatherV3ElementShape(value tf.Shape) TensorArrayGatherV3Attr { + return func(m optionalAttr) { + m["element_shape"] = value + } +} + +// Gather specific elements from the TensorArray into output `value`. // -// Along each axis `IRFFT2D` is computed on, if `fft_length` (or -// `fft_length / 2 + 1` for the inner-most dimension) is smaller than the -// corresponding dimension of `input`, the dimension is cropped. If it is larger, -// the dimension is padded with zeros. +// All elements selected by `indices` must have the same shape. // // Arguments: -// input: A complex64 tensor. -// fft_length: An int32 tensor of shape [2]. The FFT length for each dimension. -// -// Returns A float32 tensor of the same rank as `input`. The inner-most 2 -// dimensions of `input` are replaced with the `fft_length` samples of their -// inverse 2D Fourier transform. +// handle: The handle to a TensorArray. +// indices: The locations in the TensorArray from which to read tensor elements. +// flow_in: A float scalar that enforces proper chaining of operations. +// dtype: The type of the elem that is returned. // -// @compatibility(numpy) -// Equivalent to np.fft.irfft2 -// @end_compatibility -func IRFFT2D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) { +// Returns All of the elements in the TensorArray, concatenated along a new +// axis (the new dimension 0). +func TensorArrayGatherV3(scope *Scope, handle tf.Output, indices tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayGatherV3Attr) (value tf.Output) { if scope.Err() != nil { return } + attrs := map[string]interface{}{"dtype": dtype} + for _, a := range optional { + a(attrs) + } opspec := tf.OpSpec{ - Type: "IRFFT2D", + Type: "TensorArrayGatherV3", Input: []tf.Input{ - input, fft_length, + handle, indices, flow_in, }, + Attrs: attrs, } op := scope.AddOperation(opspec) return op.Output(0) } -// DecodeJpegAttr is an optional argument to DecodeJpeg. -type DecodeJpegAttr func(optionalAttr) - -// DecodeJpegChannels sets the optional channels attribute to value. +// Returns x / y element-wise for integer types. // -// value: Number of color channels for the decoded image. -// If not specified, defaults to 0 -func DecodeJpegChannels(value int64) DecodeJpegAttr { - return func(m optionalAttr) { - m["channels"] = value - } -} - -// DecodeJpegRatio sets the optional ratio attribute to value. +// Truncation designates that negative numbers will round fractional quantities +// toward zero. I.e. -7 / 5 = -1. This matches C semantics but it is different +// than Python semantics. See `FloorDiv` for a division function that matches +// Python Semantics. // -// value: Downscaling ratio. -// If not specified, defaults to 1 -func DecodeJpegRatio(value int64) DecodeJpegAttr { - return func(m optionalAttr) { - m["ratio"] = value +// *NOTE*: `TruncateDiv` supports broadcasting. More about broadcasting +// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) +func TruncateDiv(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { + if scope.Err() != nil { + return } + opspec := tf.OpSpec{ + Type: "TruncateDiv", + Input: []tf.Input{ + x, y, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) } -// DecodeJpegFancyUpscaling sets the optional fancy_upscaling attribute to value. +// Restores tensors from a V2 checkpoint. // -// value: If true use a slower but nicer upscaling of the -// chroma planes (yuv420/422 only). -// If not specified, defaults to true -func DecodeJpegFancyUpscaling(value bool) DecodeJpegAttr { - return func(m optionalAttr) { - m["fancy_upscaling"] = value +// For backward compatibility with the V1 format, this Op currently allows +// restoring from a V1 checkpoint as well: +// - This Op first attempts to find the V2 index file pointed to by "prefix", and +// if found proceed to read it as a V2 checkpoint; +// - Otherwise the V1 read path is invoked. +// Relying on this behavior is not recommended, as the ability to fall back to read +// V1 might be deprecated and eventually removed. +// +// By default, restores the named tensors in full. If the caller wishes to restore +// specific slices of stored tensors, "shape_and_slices" should be non-empty +// strings and correspondingly well-formed. +// +// Callers must ensure all the named tensors are indeed stored in the checkpoint. +// +// Arguments: +// prefix: Must have a single element. The prefix of a V2 checkpoint. +// tensor_names: shape {N}. The names of the tensors to be restored. +// shape_and_slices: shape {N}. The slice specs of the tensors to be restored. +// Empty strings indicate that they are non-partitioned tensors. +// dtypes: shape {N}. The list of expected dtype for the tensors. Must match +// those stored in the checkpoint. +// +// Returns shape {N}. The restored tensors, whose shapes are read from the +// checkpoint directly. +func RestoreV2(scope *Scope, prefix tf.Output, tensor_names tf.Output, shape_and_slices tf.Output, dtypes []tf.DataType) (tensors []tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"dtypes": dtypes} + opspec := tf.OpSpec{ + Type: "RestoreV2", + Input: []tf.Input{ + prefix, tensor_names, shape_and_slices, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + if scope.Err() != nil { + return } + var idx int + var err error + if tensors, idx, err = makeOutputList(op, idx, "tensors"); err != nil { + scope.UpdateErr("RestoreV2", err) + return + } + return tensors } -// DecodeJpegTryRecoverTruncated sets the optional try_recover_truncated attribute to value. +// Creates a dataset that skips `count` elements from the `input_dataset`. // -// value: If true try to recover an image from truncated input. -// If not specified, defaults to false -func DecodeJpegTryRecoverTruncated(value bool) DecodeJpegAttr { - return func(m optionalAttr) { - m["try_recover_truncated"] = value +// Arguments: +// +// count: A scalar representing the number of elements from the `input_dataset` +// that should be skipped. If count is -1, skips everything. +// +// +func SkipDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes} + opspec := tf.OpSpec{ + Type: "SkipDataset", + Input: []tf.Input{ + input_dataset, count, + }, + Attrs: attrs, } + op := scope.AddOperation(opspec) + return op.Output(0) } -// DecodeJpegAcceptableFraction sets the optional acceptable_fraction attribute to value. +// Computes the maximum along segments of a tensor. // -// value: The minimum required fraction of lines before a truncated -// input is accepted. -// If not specified, defaults to 1 -func DecodeJpegAcceptableFraction(value float32) DecodeJpegAttr { - return func(m optionalAttr) { - m["acceptable_fraction"] = value +// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of +// segments. +// +// Computes a tensor such that +// \\(output_i = \max_j(data_j)\\) where `max` is over `j` such +// that `segment_ids[j] == i`. +// +// If the max is empty for a given segment ID `i`, `output[i] = 0`. +// +//
+// +//
+// +// Arguments: +// +// segment_ids: A 1-D tensor whose rank is equal to the rank of `data`'s +// first dimension. Values should be sorted and can be repeated. +// +// Returns Has same shape as data, except for dimension 0 which +// has size `k`, the number of segments. +func SegmentMax(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "SegmentMax", + Input: []tf.Input{ + data, segment_ids, + }, } + op := scope.AddOperation(opspec) + return op.Output(0) } -// DecodeJpegDctMethod sets the optional dct_method attribute to value. -// -// value: string specifying a hint about the algorithm used for -// decompression. Defaults to "" which maps to a system-specific -// default. Currently valid values are ["INTEGER_FAST", -// "INTEGER_ACCURATE"]. The hint may be ignored (e.g., the internal -// jpeg library changes to a version that does not have that specific -// option.) -// If not specified, defaults to "" -func DecodeJpegDctMethod(value string) DecodeJpegAttr { - return func(m optionalAttr) { - m["dct_method"] = value +// Computes hyperbolic tangent of `x` element-wise. +func Tanh(scope *Scope, x tf.Output) (y tf.Output) { + if scope.Err() != nil { + return } + opspec := tf.OpSpec{ + Type: "Tanh", + Input: []tf.Input{ + x, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) } -// Decode a JPEG-encoded image to a uint8 tensor. +// Decode web-safe base64-encoded strings. // -// The attr `channels` indicates the desired number of color channels for the -// decoded image. +// Input may or may not have padding at the end. See EncodeBase64 for padding. +// Web-safe means that input must use - and _ instead of + and /. // -// Accepted values are: +// Arguments: +// input: Base64 strings to decode. // -// * 0: Use the number of channels in the JPEG-encoded image. -// * 1: output a grayscale image. -// * 3: output an RGB image. +// Returns Decoded strings. +func DecodeBase64(scope *Scope, input tf.Output) (output tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "DecodeBase64", + Input: []tf.Input{ + input, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Store the input tensor in the state of the current session. // -// If needed, the JPEG-encoded image is transformed to match the requested number -// of color channels. +// Arguments: +// value: The tensor to be stored. // -// The attr `ratio` allows downscaling the image by an integer factor during -// decoding. Allowed values are: 1, 2, 4, and 8. This is much faster than -// downscaling the image later. +// Returns The handle for the tensor stored in the session state, represented +// as a string. +func GetSessionHandle(scope *Scope, value tf.Output) (handle tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "GetSessionHandle", + Input: []tf.Input{ + value, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// ResourceSparseApplyProximalAdagradAttr is an optional argument to ResourceSparseApplyProximalAdagrad. +type ResourceSparseApplyProximalAdagradAttr func(optionalAttr) + +// ResourceSparseApplyProximalAdagradUseLocking sets the optional use_locking attribute to value. // +// value: If True, updating of the var and accum tensors will be protected by +// a lock; otherwise the behavior is undefined, but may exhibit less contention. +// If not specified, defaults to false +func ResourceSparseApplyProximalAdagradUseLocking(value bool) ResourceSparseApplyProximalAdagradAttr { + return func(m optionalAttr) { + m["use_locking"] = value + } +} + +// Sparse update entries in '*var' and '*accum' according to FOBOS algorithm. // -// This op also supports decoding PNGs and non-animated GIFs since the interface is -// the same, though it is cleaner to use `tf.image.decode_image`. +// That is for rows we have grad for, we update var and accum as follows: +// accum += grad * grad +// prox_v = var +// prox_v -= lr * grad * (1 / sqrt(accum)) +// var = sign(prox_v)/(1+lr*l2) * max{|prox_v|-lr*l1,0} // // Arguments: -// contents: 0-D. The JPEG-encoded image. +// var_: Should be from a Variable(). +// accum: Should be from a Variable(). +// lr: Learning rate. Must be a scalar. +// l1: L1 regularization. Must be a scalar. +// l2: L2 regularization. Must be a scalar. +// grad: The gradient. +// indices: A vector of indices into the first dimension of var and accum. // -// Returns 3-D with shape `[height, width, channels]`.. -func DecodeJpeg(scope *Scope, contents tf.Output, optional ...DecodeJpegAttr) (image tf.Output) { +// Returns the created operation. +func ResourceSparseApplyProximalAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyProximalAdagradAttr) (o *tf.Operation) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "ResourceSparseApplyProximalAdagrad", + Input: []tf.Input{ + var_, accum, lr, l1, l2, grad, indices, + }, + Attrs: attrs, + } + return scope.AddOperation(opspec) +} + +// MaxPool3DGradAttr is an optional argument to MaxPool3DGrad. +type MaxPool3DGradAttr func(optionalAttr) + +// MaxPool3DGradDataFormat sets the optional data_format attribute to value. +// +// value: The data format of the input and output data. With the +// default format "NDHWC", the data is stored in the order of: +// [batch, in_depth, in_height, in_width, in_channels]. +// Alternatively, the format could be "NCDHW", the data storage order is: +// [batch, in_channels, in_depth, in_height, in_width]. +// If not specified, defaults to "NDHWC" +func MaxPool3DGradDataFormat(value string) MaxPool3DGradAttr { + return func(m optionalAttr) { + m["data_format"] = value + } +} + +// Computes gradients of max pooling function. +// +// Arguments: +// orig_input: The original input tensor. +// orig_output: The original output tensor. +// grad: Output backprop of shape `[batch, depth, rows, cols, channels]`. +// ksize: 1-D tensor of length 5. The size of the window for each dimension of +// the input tensor. Must have `ksize[0] = ksize[4] = 1`. +// strides: 1-D tensor of length 5. The stride of the sliding window for each +// dimension of `input`. Must have `strides[0] = strides[4] = 1`. +// padding: The type of padding algorithm to use. +func MaxPool3DGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPool3DGradAttr) (output tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{} + attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding} for _, a := range optional { a(attrs) } opspec := tf.OpSpec{ - Type: "DecodeJpeg", + Type: "MaxPool3DGrad", Input: []tf.Input{ - contents, + orig_input, orig_output, grad, }, Attrs: attrs, } @@ -10640,83 +10697,59 @@ func DecodeJpeg(scope *Scope, contents tf.Output, optional ...DecodeJpegAttr) (i return op.Output(0) } -// Transforms a vector of brain.Example protos (as strings) into typed tensors. +// SparseReduceSumAttr is an optional argument to SparseReduceSum. +type SparseReduceSumAttr func(optionalAttr) + +// SparseReduceSumKeepDims sets the optional keep_dims attribute to value. +// +// value: If true, retain reduced dimensions with length 1. +// If not specified, defaults to false +func SparseReduceSumKeepDims(value bool) SparseReduceSumAttr { + return func(m optionalAttr) { + m["keep_dims"] = value + } +} + +// Computes the sum of elements across dimensions of a SparseTensor. +// +// This Op takes a SparseTensor and is the sparse counterpart to +// `tf.reduce_sum()`. In particular, this Op also returns a dense `Tensor` +// instead of a sparse one. +// +// Reduces `sp_input` along the dimensions given in `reduction_axes`. Unless +// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in +// `reduction_axes`. If `keep_dims` is true, the reduced dimensions are retained +// with length 1. +// +// If `reduction_axes` has no entries, all dimensions are reduced, and a tensor +// with a single element is returned. Additionally, the axes can be negative, +// which are interpreted according to the indexing rules in Python. // // Arguments: -// serialized: A vector containing a batch of binary serialized Example protos. -// names: A vector containing the names of the serialized protos. -// May contain, for example, table key (descriptive) names for the -// corresponding serialized protos. These are purely useful for debugging -// purposes, and the presence of values here has no effect on the output. -// May also be an empty vector if no names are available. -// If non-empty, this vector must be the same length as "serialized". -// sparse_keys: A list of Nsparse string Tensors (scalars). -// The keys expected in the Examples' features associated with sparse values. -// dense_keys: A list of Ndense string Tensors (scalars). -// The keys expected in the Examples' features associated with dense values. -// dense_defaults: A list of Ndense Tensors (some may be empty). -// dense_defaults[j] provides default values -// when the example's feature_map lacks dense_key[j]. If an empty Tensor is -// provided for dense_defaults[j], then the Feature dense_keys[j] is required. -// The input type is inferred from dense_defaults[j], even when it's empty. -// If dense_defaults[j] is not empty, and dense_shapes[j] is fully defined, -// then the shape of dense_defaults[j] must match that of dense_shapes[j]. -// If dense_shapes[j] has an undefined major dimension (variable strides dense -// feature), dense_defaults[j] must contain a single element: -// the padding element. -// sparse_types: A list of Nsparse types; the data types of data in each Feature -// given in sparse_keys. -// Currently the ParseExample supports DT_FLOAT (FloatList), -// DT_INT64 (Int64List), and DT_STRING (BytesList). -// dense_shapes: A list of Ndense shapes; the shapes of data in each Feature -// given in dense_keys. -// The number of elements in the Feature corresponding to dense_key[j] -// must always equal dense_shapes[j].NumEntries(). -// If dense_shapes[j] == (D0, D1, ..., DN) then the shape of output -// Tensor dense_values[j] will be (|serialized|, D0, D1, ..., DN): -// The dense outputs are just the inputs row-stacked by batch. -// This works for dense_shapes[j] = (-1, D1, ..., DN). In this case -// the shape of the output Tensor dense_values[j] will be -// (|serialized|, M, D1, .., DN), where M is the maximum number of blocks -// of elements of length D1 * .... * DN, across all minibatch entries -// in the input. Any minibatch entry with less than M blocks of elements of -// length D1 * ... * DN will be padded with the corresponding default_value -// scalar element along the second dimension. -func ParseExample(scope *Scope, serialized tf.Output, names tf.Output, sparse_keys []tf.Output, dense_keys []tf.Output, dense_defaults []tf.Output, sparse_types []tf.DataType, dense_shapes []tf.Shape) (sparse_indices []tf.Output, sparse_values []tf.Output, sparse_shapes []tf.Output, dense_values []tf.Output) { +// input_indices: 2-D. `N x R` matrix with the indices of non-empty values in a +// SparseTensor, possibly not in canonical ordering. +// input_values: 1-D. `N` non-empty values corresponding to `input_indices`. +// input_shape: 1-D. Shape of the input SparseTensor. +// reduction_axes: 1-D. Length-`K` vector containing the reduction axes. +// +// Returns `R-K`-D. The reduced Tensor. +func SparseReduceSum(scope *Scope, input_indices tf.Output, input_values tf.Output, input_shape tf.Output, reduction_axes tf.Output, optional ...SparseReduceSumAttr) (output tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{"sparse_types": sparse_types, "dense_shapes": dense_shapes} + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } opspec := tf.OpSpec{ - Type: "ParseExample", + Type: "SparseReduceSum", Input: []tf.Input{ - serialized, names, tf.OutputList(sparse_keys), tf.OutputList(dense_keys), tf.OutputList(dense_defaults), + input_indices, input_values, input_shape, reduction_axes, }, Attrs: attrs, } op := scope.AddOperation(opspec) - if scope.Err() != nil { - return - } - var idx int - var err error - if sparse_indices, idx, err = makeOutputList(op, idx, "sparse_indices"); err != nil { - scope.UpdateErr("ParseExample", err) - return - } - if sparse_values, idx, err = makeOutputList(op, idx, "sparse_values"); err != nil { - scope.UpdateErr("ParseExample", err) - return - } - if sparse_shapes, idx, err = makeOutputList(op, idx, "sparse_shapes"); err != nil { - scope.UpdateErr("ParseExample", err) - return - } - if dense_values, idx, err = makeOutputList(op, idx, "dense_values"); err != nil { - scope.UpdateErr("ParseExample", err) - return - } - return sparse_indices, sparse_values, sparse_shapes, dense_values + return op.Output(0) } // VariableShapeAttr is an optional argument to VariableShape. @@ -10759,6 +10792,82 @@ func VariableShape(scope *Scope, input tf.Output, optional ...VariableShapeAttr) return op.Output(0) } +// SparseToSparseSetOperationAttr is an optional argument to SparseToSparseSetOperation. +type SparseToSparseSetOperationAttr func(optionalAttr) + +// SparseToSparseSetOperationValidateIndices sets the optional validate_indices attribute to value. +// If not specified, defaults to true +func SparseToSparseSetOperationValidateIndices(value bool) SparseToSparseSetOperationAttr { + return func(m optionalAttr) { + m["validate_indices"] = value + } +} + +// Applies set operation along last dimension of 2 `SparseTensor` inputs. +// +// See SetOperationOp::SetOperationFromContext for values of `set_operation`. +// +// If `validate_indices` is `True`, `SparseToSparseSetOperation` validates the +// order and range of `set1` and `set2` indices. +// +// Input `set1` is a `SparseTensor` represented by `set1_indices`, `set1_values`, +// and `set1_shape`. For `set1` ranked `n`, 1st `n-1` dimensions must be the same +// as `set2`. Dimension `n` contains values in a set, duplicates are allowed but +// ignored. +// +// Input `set2` is a `SparseTensor` represented by `set2_indices`, `set2_values`, +// and `set2_shape`. For `set2` ranked `n`, 1st `n-1` dimensions must be the same +// as `set1`. Dimension `n` contains values in a set, duplicates are allowed but +// ignored. +// +// If `validate_indices` is `True`, this op validates the order and range of `set1` +// and `set2` indices. +// +// Output `result` is a `SparseTensor` represented by `result_indices`, +// `result_values`, and `result_shape`. For `set1` and `set2` ranked `n`, this +// has rank `n` and the same 1st `n-1` dimensions as `set1` and `set2`. The `nth` +// dimension contains the result of `set_operation` applied to the corresponding +// `[0...n-1]` dimension of `set`. +// +// Arguments: +// set1_indices: 2D `Tensor`, indices of a `SparseTensor`. Must be in row-major +// order. +// set1_values: 1D `Tensor`, values of a `SparseTensor`. Must be in row-major +// order. +// set1_shape: 1D `Tensor`, shape of a `SparseTensor`. `set1_shape[0...n-1]` must +// be the same as `set2_shape[0...n-1]`, `set1_shape[n]` is the +// max set size across `0...n-1` dimensions. +// set2_indices: 2D `Tensor`, indices of a `SparseTensor`. Must be in row-major +// order. +// set2_values: 1D `Tensor`, values of a `SparseTensor`. Must be in row-major +// order. +// set2_shape: 1D `Tensor`, shape of a `SparseTensor`. `set2_shape[0...n-1]` must +// be the same as `set1_shape[0...n-1]`, `set2_shape[n]` is the +// max set size across `0...n-1` dimensions. +// +// +// Returns 2D indices of a `SparseTensor`.1D values of a `SparseTensor`.1D `Tensor` shape of a `SparseTensor`. `result_shape[0...n-1]` is +// the same as the 1st `n-1` dimensions of `set1` and `set2`, `result_shape[n]` +// is the max result set size across all `0...n-1` dimensions. +func SparseToSparseSetOperation(scope *Scope, set1_indices tf.Output, set1_values tf.Output, set1_shape tf.Output, set2_indices tf.Output, set2_values tf.Output, set2_shape tf.Output, set_operation string, optional ...SparseToSparseSetOperationAttr) (result_indices tf.Output, result_values tf.Output, result_shape tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"set_operation": set_operation} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "SparseToSparseSetOperation", + Input: []tf.Input{ + set1_indices, set1_values, set1_shape, set2_indices, set2_values, set2_shape, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0), op.Output(1), op.Output(2) +} + // Computes softmax cross entropy cost and gradients to backpropagate. // // Unlike `SoftmaxCrossEntropyWithLogits`, this operation does not accept @@ -11241,42 +11350,137 @@ func TensorArrayV3IdenticalElementShapes(value bool) TensorArrayV3Attr { // TensorArrayV3TensorArrayName sets the optional tensor_array_name attribute to value. // -// value: Overrides the name used for the temporary tensor_array -// resource. Default value is the name of the 'TensorArray' op (which -// is guaranteed unique). -// If not specified, defaults to "" -func TensorArrayV3TensorArrayName(value string) TensorArrayV3Attr { - return func(m optionalAttr) { - m["tensor_array_name"] = value - } -} - -// An array of Tensors of given size. +// value: Overrides the name used for the temporary tensor_array +// resource. Default value is the name of the 'TensorArray' op (which +// is guaranteed unique). +// If not specified, defaults to "" +func TensorArrayV3TensorArrayName(value string) TensorArrayV3Attr { + return func(m optionalAttr) { + m["tensor_array_name"] = value + } +} + +// An array of Tensors of given size. +// +// Write data via Write and read via Read or Pack. +// +// Arguments: +// size: The size of the array. +// dtype: The type of the elements on the tensor_array. +// +// Returns The handle to the TensorArray.A scalar used to control gradient flow. +func TensorArrayV3(scope *Scope, size tf.Output, dtype tf.DataType, optional ...TensorArrayV3Attr) (handle tf.Output, flow tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"dtype": dtype} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "TensorArrayV3", + Input: []tf.Input{ + size, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0), op.Output(1) +} + +// MatrixSolveLsAttr is an optional argument to MatrixSolveLs. +type MatrixSolveLsAttr func(optionalAttr) + +// MatrixSolveLsFast sets the optional fast attribute to value. +// If not specified, defaults to true +func MatrixSolveLsFast(value bool) MatrixSolveLsAttr { + return func(m optionalAttr) { + m["fast"] = value + } +} + +// Solves one or more linear least-squares problems. +// +// `matrix` is a tensor of shape `[..., M, N]` whose inner-most 2 dimensions +// form real or complex matrices of size `[M, N]`. `Rhs` is a tensor of the same +// type as `matrix` and shape `[..., M, K]`. +// The output is a tensor shape `[..., N, K]` where each output matrix solves +// each of the equations +// `matrix[..., :, :]` * `output[..., :, :]` = `rhs[..., :, :]` +// in the least squares sense. +// +// We use the following notation for (complex) matrix and right-hand sides +// in the batch: +// +// `matrix`=\\(A \in \mathbb{C}^{m \times n}\\), +// `rhs`=\\(B \in \mathbb{C}^{m \times k}\\), +// `output`=\\(X \in \mathbb{C}^{n \times k}\\), +// `l2_regularizer`=\\(\lambda \in \mathbb{R}\\). +// +// If `fast` is `True`, then the solution is computed by solving the normal +// equations using Cholesky decomposition. Specifically, if \\(m \ge n\\) then +// \\(X = (A^H A + \lambda I)^{-1} A^H B\\), which solves the least-squares +// problem \\(X = \mathrm{argmin}_{Z \in \Re^{n \times k} } ||A Z - B||_F^2 + +// \lambda ||Z||_F^2\\). If \\(m \lt n\\) then `output` is computed as +// \\(X = A^H (A A^H + \lambda I)^{-1} B\\), which (for \\(\lambda = 0\\)) is the +// minimum-norm solution to the under-determined linear system, i.e. +// \\(X = \mathrm{argmin}_{Z \in \mathbb{C}^{n \times k} } ||Z||_F^2 \\), +// subject to \\(A Z = B\\). Notice that the fast path is only numerically stable +// when \\(A\\) is numerically full rank and has a condition number +// \\(\mathrm{cond}(A) \lt \frac{1}{\sqrt{\epsilon_{mach} } }\\) or\\(\lambda\\) is +// sufficiently large. // -// Write data via Write and read via Read or Pack. +// If `fast` is `False` an algorithm based on the numerically robust complete +// orthogonal decomposition is used. This computes the minimum-norm +// least-squares solution, even when \\(A\\) is rank deficient. This path is +// typically 6-7 times slower than the fast path. If `fast` is `False` then +// `l2_regularizer` is ignored. // // Arguments: -// size: The size of the array. -// dtype: The type of the elements on the tensor_array. +// matrix: Shape is `[..., M, N]`. +// rhs: Shape is `[..., M, K]`. +// l2_regularizer: Scalar tensor. // -// Returns The handle to the TensorArray.A scalar used to control gradient flow. -func TensorArrayV3(scope *Scope, size tf.Output, dtype tf.DataType, optional ...TensorArrayV3Attr) (handle tf.Output, flow tf.Output) { +// @compatibility(numpy) +// Equivalent to np.linalg.lstsq +// @end_compatibility +// +// Returns Shape is `[..., N, K]`. +func MatrixSolveLs(scope *Scope, matrix tf.Output, rhs tf.Output, l2_regularizer tf.Output, optional ...MatrixSolveLsAttr) (output tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{"dtype": dtype} + attrs := map[string]interface{}{} for _, a := range optional { a(attrs) } opspec := tf.OpSpec{ - Type: "TensorArrayV3", + Type: "MatrixSolveLs", Input: []tf.Input{ - size, + matrix, rhs, l2_regularizer, }, Attrs: attrs, } op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1) + return op.Output(0) +} + +// Elementwise computes the bitwise OR of `x` and `y`. +// +// The result will have those bits set, that are set in `x`, `y` or both. The +// computation is performed on the underlying representations of `x` and `y`. +func BitwiseOr(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "BitwiseOr", + Input: []tf.Input{ + x, y, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) } // MaxPool3DAttr is an optional argument to MaxPool3D. @@ -13490,228 +13694,73 @@ func MaxPoolGradGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, // ``` func Rint(scope *Scope, x tf.Output) (y tf.Output) { if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "Rint", - Input: []tf.Input{ - x, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// OrderedMapUnstageNoKeyAttr is an optional argument to OrderedMapUnstageNoKey. -type OrderedMapUnstageNoKeyAttr func(optionalAttr) - -// OrderedMapUnstageNoKeyCapacity sets the optional capacity attribute to value. -// If not specified, defaults to 0 -// -// REQUIRES: value >= 0 -func OrderedMapUnstageNoKeyCapacity(value int64) OrderedMapUnstageNoKeyAttr { - return func(m optionalAttr) { - m["capacity"] = value - } -} - -// OrderedMapUnstageNoKeyMemoryLimit sets the optional memory_limit attribute to value. -// If not specified, defaults to 0 -// -// REQUIRES: value >= 0 -func OrderedMapUnstageNoKeyMemoryLimit(value int64) OrderedMapUnstageNoKeyAttr { - return func(m optionalAttr) { - m["memory_limit"] = value - } -} - -// OrderedMapUnstageNoKeyContainer sets the optional container attribute to value. -// If not specified, defaults to "" -func OrderedMapUnstageNoKeyContainer(value string) OrderedMapUnstageNoKeyAttr { - return func(m optionalAttr) { - m["container"] = value - } -} - -// OrderedMapUnstageNoKeySharedName sets the optional shared_name attribute to value. -// If not specified, defaults to "" -func OrderedMapUnstageNoKeySharedName(value string) OrderedMapUnstageNoKeyAttr { - return func(m optionalAttr) { - m["shared_name"] = value - } -} - -// Op removes and returns the (key, value) element with the smallest -// -// key from the underlying container. If the underlying container -// does not contain elements, the op will block until it does. -func OrderedMapUnstageNoKey(scope *Scope, indices tf.Output, dtypes []tf.DataType, optional ...OrderedMapUnstageNoKeyAttr) (key tf.Output, values []tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"dtypes": dtypes} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "OrderedMapUnstageNoKey", - Input: []tf.Input{ - indices, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - if scope.Err() != nil { - return - } - var idx int - var err error - key = op.Output(idx) - if values, idx, err = makeOutputList(op, idx, "values"); err != nil { - scope.UpdateErr("OrderedMapUnstageNoKey", err) - return - } - return key, values -} - -// MaxPool3DGradGradAttr is an optional argument to MaxPool3DGradGrad. -type MaxPool3DGradGradAttr func(optionalAttr) - -// MaxPool3DGradGradDataFormat sets the optional data_format attribute to value. -// -// value: The data format of the input and output data. With the -// default format "NDHWC", the data is stored in the order of: -// [batch, in_depth, in_height, in_width, in_channels]. -// Alternatively, the format could be "NCDHW", the data storage order is: -// [batch, in_channels, in_depth, in_height, in_width]. -// If not specified, defaults to "NDHWC" -func MaxPool3DGradGradDataFormat(value string) MaxPool3DGradGradAttr { - return func(m optionalAttr) { - m["data_format"] = value - } -} - -// Computes second-order gradients of the maxpooling function. -// -// Arguments: -// orig_input: The original input tensor. -// orig_output: The original output tensor. -// grad: Output backprop of shape `[batch, depth, rows, cols, channels]`. -// ksize: 1-D tensor of length 5. The size of the window for each dimension of -// the input tensor. Must have `ksize[0] = ksize[4] = 1`. -// strides: 1-D tensor of length 5. The stride of the sliding window for each -// dimension of `input`. Must have `strides[0] = strides[4] = 1`. -// padding: The type of padding algorithm to use. -// -// Returns Gradients of gradients w.r.t. the input to `max_pool`. -func MaxPool3DGradGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPool3DGradGradAttr) (output tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "MaxPool3DGradGrad", - Input: []tf.Input{ - orig_input, orig_output, grad, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// Conv3DBackpropFilterV2Attr is an optional argument to Conv3DBackpropFilterV2. -type Conv3DBackpropFilterV2Attr func(optionalAttr) - -// Conv3DBackpropFilterV2DataFormat sets the optional data_format attribute to value. -// -// value: The data format of the input and output data. With the -// default format "NDHWC", the data is stored in the order of: -// [batch, in_depth, in_height, in_width, in_channels]. -// Alternatively, the format could be "NCDHW", the data storage order is: -// [batch, in_channels, in_depth, in_height, in_width]. -// If not specified, defaults to "NDHWC" -func Conv3DBackpropFilterV2DataFormat(value string) Conv3DBackpropFilterV2Attr { - return func(m optionalAttr) { - m["data_format"] = value - } -} - -// Conv3DBackpropFilterV2Dilations sets the optional dilations attribute to value. -// -// value: 1-D tensor of length 5. The dilation factor for each dimension of -// `input`. If set to k > 1, there will be k-1 skipped cells between each -// filter element on that dimension. The dimension order is determined by the -// value of `data_format`, see above for details. Dilations in the batch and -// depth dimensions must be 1. -// If not specified, defaults to -func Conv3DBackpropFilterV2Dilations(value []int64) Conv3DBackpropFilterV2Attr { - return func(m optionalAttr) { - m["dilations"] = value - } -} - -// Computes the gradients of 3-D convolution with respect to the filter. -// -// Arguments: -// input: Shape `[batch, depth, rows, cols, in_channels]`. -// filter_sizes: An integer vector representing the tensor shape of `filter`, -// where `filter` is a 5-D -// `[filter_depth, filter_height, filter_width, in_channels, out_channels]` -// tensor. -// out_backprop: Backprop signal of shape `[batch, out_depth, out_rows, out_cols, -// out_channels]`. -// strides: 1-D tensor of length 5. The stride of the sliding window for each -// dimension of `input`. Must have `strides[0] = strides[4] = 1`. -// padding: The type of padding algorithm to use. -func Conv3DBackpropFilterV2(scope *Scope, input tf.Output, filter_sizes tf.Output, out_backprop tf.Output, strides []int64, padding string, optional ...Conv3DBackpropFilterV2Attr) (output tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"strides": strides, "padding": padding} - for _, a := range optional { - a(attrs) + return } opspec := tf.OpSpec{ - Type: "Conv3DBackpropFilterV2", + Type: "Rint", Input: []tf.Input{ - input, filter_sizes, out_backprop, + x, }, - Attrs: attrs, } op := scope.AddOperation(opspec) return op.Output(0) } -// Execute a sub graph on a remote processor. -// -// The graph specifications(such as graph itself, input tensors and output names) -// are stored as a serialized protocol buffer of RemoteFusedGraphExecuteInfo -// as serialized_remote_fused_graph_execute_info. -// The specifications will be passed to a dedicated registered -// remote fused graph executor. The executor will send the graph specifications -// to a remote processor and execute that graph. The execution results -// will be passed to consumer nodes as outputs of this node. +// OrderedMapUnstageNoKeyAttr is an optional argument to OrderedMapUnstageNoKey. +type OrderedMapUnstageNoKeyAttr func(optionalAttr) + +// OrderedMapUnstageNoKeyCapacity sets the optional capacity attribute to value. +// If not specified, defaults to 0 // -// Arguments: -// inputs: Arbitrary number of tensors with arbitrary data types +// REQUIRES: value >= 0 +func OrderedMapUnstageNoKeyCapacity(value int64) OrderedMapUnstageNoKeyAttr { + return func(m optionalAttr) { + m["capacity"] = value + } +} + +// OrderedMapUnstageNoKeyMemoryLimit sets the optional memory_limit attribute to value. +// If not specified, defaults to 0 // -// serialized_remote_fused_graph_execute_info: Serialized protocol buffer -// of RemoteFusedGraphExecuteInfo which contains graph specifications. +// REQUIRES: value >= 0 +func OrderedMapUnstageNoKeyMemoryLimit(value int64) OrderedMapUnstageNoKeyAttr { + return func(m optionalAttr) { + m["memory_limit"] = value + } +} + +// OrderedMapUnstageNoKeyContainer sets the optional container attribute to value. +// If not specified, defaults to "" +func OrderedMapUnstageNoKeyContainer(value string) OrderedMapUnstageNoKeyAttr { + return func(m optionalAttr) { + m["container"] = value + } +} + +// OrderedMapUnstageNoKeySharedName sets the optional shared_name attribute to value. +// If not specified, defaults to "" +func OrderedMapUnstageNoKeySharedName(value string) OrderedMapUnstageNoKeyAttr { + return func(m optionalAttr) { + m["shared_name"] = value + } +} + +// Op removes and returns the (key, value) element with the smallest // -// Returns Arbitrary number of tensors with arbitrary data types -func RemoteFusedGraphExecute(scope *Scope, inputs []tf.Output, Toutputs []tf.DataType, serialized_remote_fused_graph_execute_info string) (outputs []tf.Output) { +// key from the underlying container. If the underlying container +// does not contain elements, the op will block until it does. +func OrderedMapUnstageNoKey(scope *Scope, indices tf.Output, dtypes []tf.DataType, optional ...OrderedMapUnstageNoKeyAttr) (key tf.Output, values []tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{"Toutputs": Toutputs, "serialized_remote_fused_graph_execute_info": serialized_remote_fused_graph_execute_info} + attrs := map[string]interface{}{"dtypes": dtypes} + for _, a := range optional { + a(attrs) + } opspec := tf.OpSpec{ - Type: "RemoteFusedGraphExecute", + Type: "OrderedMapUnstageNoKey", Input: []tf.Input{ - tf.OutputList(inputs), + indices, }, Attrs: attrs, } @@ -13721,11 +13770,12 @@ func RemoteFusedGraphExecute(scope *Scope, inputs []tf.Output, Toutputs []tf.Dat } var idx int var err error - if outputs, idx, err = makeOutputList(op, idx, "outputs"); err != nil { - scope.UpdateErr("RemoteFusedGraphExecute", err) + key = op.Output(idx) + if values, idx, err = makeOutputList(op, idx, "values"); err != nil { + scope.UpdateErr("OrderedMapUnstageNoKey", err) return } - return outputs + return key, values } // SerializeManySparseAttr is an optional argument to SerializeManySparse. @@ -14192,14 +14242,192 @@ func MutableDenseHashTableV2(scope *Scope, empty_key tf.Output, value_dtype tf.D if scope.Err() != nil { return } - attrs := map[string]interface{}{"value_dtype": value_dtype} + attrs := map[string]interface{}{"value_dtype": value_dtype} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "MutableDenseHashTableV2", + Input: []tf.Input{ + empty_key, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Returns element-wise remainder of division. This emulates C semantics in that +// +// the result here is consistent with a truncating divide. E.g. `truncate(x / y) * +// y + truncate_mod(x, y) = x`. +// +// *NOTE*: `TruncateMod` supports broadcasting. More about broadcasting +// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) +func TruncateMod(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "TruncateMod", + Input: []tf.Input{ + x, y, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Inverse 2D real-valued fast Fourier transform. +// +// Computes the inverse 2-dimensional discrete Fourier transform of a real-valued +// signal over the inner-most 2 dimensions of `input`. +// +// The inner-most 2 dimensions of `input` are assumed to be the result of `RFFT2D`: +// The inner-most dimension contains the `fft_length / 2 + 1` unique components of +// the DFT of a real-valued signal. If `fft_length` is not provided, it is computed +// from the size of the inner-most 2 dimensions of `input`. If the FFT length used +// to compute `input` is odd, it should be provided since it cannot be inferred +// properly. +// +// Along each axis `IRFFT2D` is computed on, if `fft_length` (or +// `fft_length / 2 + 1` for the inner-most dimension) is smaller than the +// corresponding dimension of `input`, the dimension is cropped. If it is larger, +// the dimension is padded with zeros. +// +// Arguments: +// input: A complex64 tensor. +// fft_length: An int32 tensor of shape [2]. The FFT length for each dimension. +// +// Returns A float32 tensor of the same rank as `input`. The inner-most 2 +// dimensions of `input` are replaced with the `fft_length` samples of their +// inverse 2D Fourier transform. +// +// @compatibility(numpy) +// Equivalent to np.fft.irfft2 +// @end_compatibility +func IRFFT2D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "IRFFT2D", + Input: []tf.Input{ + input, fft_length, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// DecodeJpegAttr is an optional argument to DecodeJpeg. +type DecodeJpegAttr func(optionalAttr) + +// DecodeJpegChannels sets the optional channels attribute to value. +// +// value: Number of color channels for the decoded image. +// If not specified, defaults to 0 +func DecodeJpegChannels(value int64) DecodeJpegAttr { + return func(m optionalAttr) { + m["channels"] = value + } +} + +// DecodeJpegRatio sets the optional ratio attribute to value. +// +// value: Downscaling ratio. +// If not specified, defaults to 1 +func DecodeJpegRatio(value int64) DecodeJpegAttr { + return func(m optionalAttr) { + m["ratio"] = value + } +} + +// DecodeJpegFancyUpscaling sets the optional fancy_upscaling attribute to value. +// +// value: If true use a slower but nicer upscaling of the +// chroma planes (yuv420/422 only). +// If not specified, defaults to true +func DecodeJpegFancyUpscaling(value bool) DecodeJpegAttr { + return func(m optionalAttr) { + m["fancy_upscaling"] = value + } +} + +// DecodeJpegTryRecoverTruncated sets the optional try_recover_truncated attribute to value. +// +// value: If true try to recover an image from truncated input. +// If not specified, defaults to false +func DecodeJpegTryRecoverTruncated(value bool) DecodeJpegAttr { + return func(m optionalAttr) { + m["try_recover_truncated"] = value + } +} + +// DecodeJpegAcceptableFraction sets the optional acceptable_fraction attribute to value. +// +// value: The minimum required fraction of lines before a truncated +// input is accepted. +// If not specified, defaults to 1 +func DecodeJpegAcceptableFraction(value float32) DecodeJpegAttr { + return func(m optionalAttr) { + m["acceptable_fraction"] = value + } +} + +// DecodeJpegDctMethod sets the optional dct_method attribute to value. +// +// value: string specifying a hint about the algorithm used for +// decompression. Defaults to "" which maps to a system-specific +// default. Currently valid values are ["INTEGER_FAST", +// "INTEGER_ACCURATE"]. The hint may be ignored (e.g., the internal +// jpeg library changes to a version that does not have that specific +// option.) +// If not specified, defaults to "" +func DecodeJpegDctMethod(value string) DecodeJpegAttr { + return func(m optionalAttr) { + m["dct_method"] = value + } +} + +// Decode a JPEG-encoded image to a uint8 tensor. +// +// The attr `channels` indicates the desired number of color channels for the +// decoded image. +// +// Accepted values are: +// +// * 0: Use the number of channels in the JPEG-encoded image. +// * 1: output a grayscale image. +// * 3: output an RGB image. +// +// If needed, the JPEG-encoded image is transformed to match the requested number +// of color channels. +// +// The attr `ratio` allows downscaling the image by an integer factor during +// decoding. Allowed values are: 1, 2, 4, and 8. This is much faster than +// downscaling the image later. +// +// +// This op also supports decoding PNGs and non-animated GIFs since the interface is +// the same, though it is cleaner to use `tf.image.decode_image`. +// +// Arguments: +// contents: 0-D. The JPEG-encoded image. +// +// Returns 3-D with shape `[height, width, channels]`.. +func DecodeJpeg(scope *Scope, contents tf.Output, optional ...DecodeJpegAttr) (image tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} for _, a := range optional { a(attrs) } opspec := tf.OpSpec{ - Type: "MutableDenseHashTableV2", + Type: "DecodeJpeg", Input: []tf.Input{ - empty_key, + contents, }, Attrs: attrs, } @@ -14428,6 +14656,29 @@ func SparseSegmentMeanGrad(scope *Scope, grad tf.Output, indices tf.Output, segm return op.Output(0) } +// Returns the set of files matching one or more glob patterns. +// +// Note that this routine only supports wildcard characters in the +// basename portion of the pattern, not in the directory portion. +// +// Arguments: +// pattern: Shell wildcard pattern(s). Scalar or vector of type string. +// +// Returns A vector of matching filenames. +func MatchingFiles(scope *Scope, pattern tf.Output) (filenames tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "MatchingFiles", + Input: []tf.Input{ + pattern, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Returns the truth value of (x >= y) element-wise. // // *NOTE*: `GreaterEqual` supports broadcasting. More about broadcasting @@ -15157,453 +15408,210 @@ func AllCandidateSamplerSeed2(value int64) AllCandidateSamplerAttr { // // The advantages of sampling candidates per-batch are simplicity and the // possibility of efficient dense matrix multiplication. The disadvantage is that -// the sampled candidates must be chosen independently of the context and of the -// true labels. -// -// Arguments: -// true_classes: A batch_size * num_true matrix, in which each row contains the -// IDs of the num_true target_classes in the corresponding original label. -// num_true: Number of true labels per context. -// num_sampled: Number of candidates to produce. -// unique: If unique is true, we sample with rejection, so that all sampled -// candidates in a batch are unique. This requires some approximation to -// estimate the post-rejection sampling probabilities. -// -// Returns A vector of length num_sampled, in which each element is -// the ID of a sampled candidate.A batch_size * num_true matrix, representing -// the number of times each candidate is expected to occur in a batch -// of sampled candidates. If unique=true, then this is a probability.A vector of length num_sampled, for each sampled -// candidate representing the number of times the candidate is expected -// to occur in a batch of sampled candidates. If unique=true, then this is a -// probability. -func AllCandidateSampler(scope *Scope, true_classes tf.Output, num_true int64, num_sampled int64, unique bool, optional ...AllCandidateSamplerAttr) (sampled_candidates tf.Output, true_expected_count tf.Output, sampled_expected_count tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"num_true": num_true, "num_sampled": num_sampled, "unique": unique} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "AllCandidateSampler", - Input: []tf.Input{ - true_classes, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1), op.Output(2) -} - -// Saves the input tensors to disk. -// -// The size of `tensor_names` must match the number of tensors in `data`. `data[i]` -// is written to `filename` with name `tensor_names[i]`. -// -// See also `SaveSlices`. -// -// Arguments: -// filename: Must have a single element. The name of the file to which we write -// the tensor. -// tensor_names: Shape `[N]`. The names of the tensors to be saved. -// data: `N` tensors to save. -// -// Returns the created operation. -func Save(scope *Scope, filename tf.Output, tensor_names tf.Output, data []tf.Output) (o *tf.Operation) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "Save", - Input: []tf.Input{ - filename, tensor_names, tf.OutputList(data), - }, - } - return scope.AddOperation(opspec) -} - -// Returns element-wise remainder of division. When `x < 0` xor `y < 0` is -// -// true, this follows Python semantics in that the result here is consistent -// with a flooring divide. E.g. `floor(x / y) * y + mod(x, y) = x`. -// -// *NOTE*: `FloorMod` supports broadcasting. More about broadcasting -// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) -func FloorMod(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "FloorMod", - Input: []tf.Input{ - x, y, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// SparseTensorDenseMatMulAttr is an optional argument to SparseTensorDenseMatMul. -type SparseTensorDenseMatMulAttr func(optionalAttr) - -// SparseTensorDenseMatMulAdjointA sets the optional adjoint_a attribute to value. -// -// value: Use the adjoint of A in the matrix multiply. If A is complex, this -// is transpose(conj(A)). Otherwise it's transpose(A). -// If not specified, defaults to false -func SparseTensorDenseMatMulAdjointA(value bool) SparseTensorDenseMatMulAttr { - return func(m optionalAttr) { - m["adjoint_a"] = value - } -} - -// SparseTensorDenseMatMulAdjointB sets the optional adjoint_b attribute to value. -// -// value: Use the adjoint of B in the matrix multiply. If B is complex, this -// is transpose(conj(B)). Otherwise it's transpose(B). -// If not specified, defaults to false -func SparseTensorDenseMatMulAdjointB(value bool) SparseTensorDenseMatMulAttr { - return func(m optionalAttr) { - m["adjoint_b"] = value - } -} - -// Multiply SparseTensor (of rank 2) "A" by dense matrix "B". -// -// No validity checking is performed on the indices of A. However, the following -// input format is recommended for optimal behavior: -// -// if adjoint_a == false: -// A should be sorted in lexicographically increasing order. Use SparseReorder -// if you're not sure. -// if adjoint_a == true: -// A should be sorted in order of increasing dimension 1 (i.e., "column major" -// order instead of "row major" order). -// -// Arguments: -// a_indices: 2-D. The `indices` of the `SparseTensor`, size `[nnz, 2]` Matrix. -// a_values: 1-D. The `values` of the `SparseTensor`, size `[nnz]` Vector. -// a_shape: 1-D. The `shape` of the `SparseTensor`, size `[2]` Vector. -// b: 2-D. A dense Matrix. -func SparseTensorDenseMatMul(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b tf.Output, optional ...SparseTensorDenseMatMulAttr) (product tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "SparseTensorDenseMatMul", - Input: []tf.Input{ - a_indices, a_values, a_shape, b, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// Deserialize and concatenate `SparseTensors` from a serialized minibatch. -// -// The input `serialized_sparse` must be a string matrix of shape `[N x 3]` where -// `N` is the minibatch size and the rows correspond to packed outputs of -// `SerializeSparse`. The ranks of the original `SparseTensor` objects -// must all match. When the final `SparseTensor` is created, it has rank one -// higher than the ranks of the incoming `SparseTensor` objects -// (they have been concatenated along a new row dimension). -// -// The output `SparseTensor` object's shape values for all dimensions but the -// first are the max across the input `SparseTensor` objects' shape values -// for the corresponding dimensions. Its first shape value is `N`, the minibatch -// size. -// -// The input `SparseTensor` objects' indices are assumed ordered in -// standard lexicographic order. If this is not the case, after this -// step run `SparseReorder` to restore index ordering. -// -// For example, if the serialized input is a `[2 x 3]` matrix representing two -// original `SparseTensor` objects: -// -// index = [ 0] -// [10] -// [20] -// values = [1, 2, 3] -// shape = [50] -// -// and -// -// index = [ 2] -// [10] -// values = [4, 5] -// shape = [30] -// -// then the final deserialized `SparseTensor` will be: -// -// index = [0 0] -// [0 10] -// [0 20] -// [1 2] -// [1 10] -// values = [1, 2, 3, 4, 5] -// shape = [2 50] -// -// Arguments: -// serialized_sparse: 2-D, The `N` serialized `SparseTensor` objects. -// Must have 3 columns. -// dtype: The `dtype` of the serialized `SparseTensor` objects. -func DeserializeManySparse(scope *Scope, serialized_sparse tf.Output, dtype tf.DataType) (sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"dtype": dtype} - opspec := tf.OpSpec{ - Type: "DeserializeManySparse", - Input: []tf.Input{ - serialized_sparse, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1), op.Output(2) -} - -// StringJoinAttr is an optional argument to StringJoin. -type StringJoinAttr func(optionalAttr) - -// StringJoinSeparator sets the optional separator attribute to value. -// -// value: string, an optional join separator. -// If not specified, defaults to "" -func StringJoinSeparator(value string) StringJoinAttr { - return func(m optionalAttr) { - m["separator"] = value - } -} - -// Joins the strings in the given list of string tensors into one tensor; -// -// with the given separator (default is an empty separator). +// the sampled candidates must be chosen independently of the context and of the +// true labels. // // Arguments: -// inputs: A list of string tensors. The tensors must all have the same shape, -// or be scalars. Scalars may be mixed in; these will be broadcast to the shape -// of non-scalar inputs. -func StringJoin(scope *Scope, inputs []tf.Output, optional ...StringJoinAttr) (output tf.Output) { +// true_classes: A batch_size * num_true matrix, in which each row contains the +// IDs of the num_true target_classes in the corresponding original label. +// num_true: Number of true labels per context. +// num_sampled: Number of candidates to produce. +// unique: If unique is true, we sample with rejection, so that all sampled +// candidates in a batch are unique. This requires some approximation to +// estimate the post-rejection sampling probabilities. +// +// Returns A vector of length num_sampled, in which each element is +// the ID of a sampled candidate.A batch_size * num_true matrix, representing +// the number of times each candidate is expected to occur in a batch +// of sampled candidates. If unique=true, then this is a probability.A vector of length num_sampled, for each sampled +// candidate representing the number of times the candidate is expected +// to occur in a batch of sampled candidates. If unique=true, then this is a +// probability. +func AllCandidateSampler(scope *Scope, true_classes tf.Output, num_true int64, num_sampled int64, unique bool, optional ...AllCandidateSamplerAttr) (sampled_candidates tf.Output, true_expected_count tf.Output, sampled_expected_count tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{} + attrs := map[string]interface{}{"num_true": num_true, "num_sampled": num_sampled, "unique": unique} for _, a := range optional { a(attrs) } opspec := tf.OpSpec{ - Type: "StringJoin", + Type: "AllCandidateSampler", Input: []tf.Input{ - tf.OutputList(inputs), + true_classes, }, Attrs: attrs, } op := scope.AddOperation(opspec) - return op.Output(0) + return op.Output(0), op.Output(1), op.Output(2) } -// Returns immutable tensor from memory region. +// Saves the input tensors to disk. // -// The current implementation memmaps the tensor from a file. +// The size of `tensor_names` must match the number of tensors in `data`. `data[i]` +// is written to `filename` with name `tensor_names[i]`. +// +// See also `SaveSlices`. // // Arguments: -// dtype: Type of the returned tensor. -// shape: Shape of the returned tensor. -// memory_region_name: Name of readonly memory region used by the tensor, see -// NewReadOnlyMemoryRegionFromFile in tensorflow::Env. -func ImmutableConst(scope *Scope, dtype tf.DataType, shape tf.Shape, memory_region_name string) (tensor tf.Output) { +// filename: Must have a single element. The name of the file to which we write +// the tensor. +// tensor_names: Shape `[N]`. The names of the tensors to be saved. +// data: `N` tensors to save. +// +// Returns the created operation. +func Save(scope *Scope, filename tf.Output, tensor_names tf.Output, data []tf.Output) (o *tf.Operation) { if scope.Err() != nil { return } - attrs := map[string]interface{}{"dtype": dtype, "shape": shape, "memory_region_name": memory_region_name} opspec := tf.OpSpec{ - Type: "ImmutableConst", - - Attrs: attrs, + Type: "Save", + Input: []tf.Input{ + filename, tensor_names, tf.OutputList(data), + }, } - op := scope.AddOperation(opspec) - return op.Output(0) + return scope.AddOperation(opspec) } -// Inverse real-valued fast Fourier transform. -// -// Computes the inverse 1-dimensional discrete Fourier transform of a real-valued -// signal over the inner-most dimension of `input`. -// -// The inner-most dimension of `input` is assumed to be the result of `RFFT`: the -// `fft_length / 2 + 1` unique components of the DFT of a real-valued signal. If -// `fft_length` is not provided, it is computed from the size of the inner-most -// dimension of `input` (`fft_length = 2 * (inner - 1)`). If the FFT length used to -// compute `input` is odd, it should be provided since it cannot be inferred -// properly. -// -// Along the axis `IRFFT` is computed on, if `fft_length / 2 + 1` is smaller -// than the corresponding dimension of `input`, the dimension is cropped. If it is -// larger, the dimension is padded with zeros. -// -// Arguments: -// input: A complex64 tensor. -// fft_length: An int32 tensor of shape [1]. The FFT length. +// Returns element-wise remainder of division. When `x < 0` xor `y < 0` is // -// Returns A float32 tensor of the same rank as `input`. The inner-most -// dimension of `input` is replaced with the `fft_length` samples of its inverse -// 1D Fourier transform. +// true, this follows Python semantics in that the result here is consistent +// with a flooring divide. E.g. `floor(x / y) * y + mod(x, y) = x`. // -// @compatibility(numpy) -// Equivalent to np.fft.irfft -// @end_compatibility -func IRFFT(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) { +// *NOTE*: `FloorMod` supports broadcasting. More about broadcasting +// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) +func FloorMod(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { if scope.Err() != nil { return } opspec := tf.OpSpec{ - Type: "IRFFT", + Type: "FloorMod", Input: []tf.Input{ - input, fft_length, + x, y, }, } op := scope.AddOperation(opspec) return op.Output(0) } -// Concatenates a list of `SparseTensor` along the specified dimension. -// -// Concatenation is with respect to the dense versions of these sparse tensors. -// It is assumed that each input is a `SparseTensor` whose elements are ordered -// along increasing dimension number. -// -// All inputs' shapes must match, except for the concat dimension. The -// `indices`, `values`, and `shapes` lists must have the same length. -// -// The output shape is identical to the inputs', except along the concat -// dimension, where it is the sum of the inputs' sizes along that dimension. -// -// The output elements will be resorted to preserve the sort order along -// increasing dimension number. -// -// This op runs in `O(M log M)` time, where `M` is the total number of non-empty -// values across all inputs. This is due to the need for an internal sort in -// order to concatenate efficiently across an arbitrary dimension. -// -// For example, if `concat_dim = 1` and the inputs are -// -// sp_inputs[0]: shape = [2, 3] -// [0, 2]: "a" -// [1, 0]: "b" -// [1, 1]: "c" -// -// sp_inputs[1]: shape = [2, 4] -// [0, 1]: "d" -// [0, 2]: "e" +// SparseTensorDenseMatMulAttr is an optional argument to SparseTensorDenseMatMul. +type SparseTensorDenseMatMulAttr func(optionalAttr) + +// SparseTensorDenseMatMulAdjointA sets the optional adjoint_a attribute to value. // -// then the output will be +// value: Use the adjoint of A in the matrix multiply. If A is complex, this +// is transpose(conj(A)). Otherwise it's transpose(A). +// If not specified, defaults to false +func SparseTensorDenseMatMulAdjointA(value bool) SparseTensorDenseMatMulAttr { + return func(m optionalAttr) { + m["adjoint_a"] = value + } +} + +// SparseTensorDenseMatMulAdjointB sets the optional adjoint_b attribute to value. // -// shape = [2, 7] -// [0, 2]: "a" -// [0, 4]: "d" -// [0, 5]: "e" -// [1, 0]: "b" -// [1, 1]: "c" +// value: Use the adjoint of B in the matrix multiply. If B is complex, this +// is transpose(conj(B)). Otherwise it's transpose(B). +// If not specified, defaults to false +func SparseTensorDenseMatMulAdjointB(value bool) SparseTensorDenseMatMulAttr { + return func(m optionalAttr) { + m["adjoint_b"] = value + } +} + +// Multiply SparseTensor (of rank 2) "A" by dense matrix "B". // -// Graphically this is equivalent to doing +// No validity checking is performed on the indices of A. However, the following +// input format is recommended for optimal behavior: // -// [ a] concat [ d e ] = [ a d e ] -// [b c ] [ ] [b c ] +// if adjoint_a == false: +// A should be sorted in lexicographically increasing order. Use SparseReorder +// if you're not sure. +// if adjoint_a == true: +// A should be sorted in order of increasing dimension 1 (i.e., "column major" +// order instead of "row major" order). // // Arguments: -// indices: 2-D. Indices of each input `SparseTensor`. -// values: 1-D. Non-empty values of each `SparseTensor`. -// shapes: 1-D. Shapes of each `SparseTensor`. -// concat_dim: Dimension to concatenate along. Must be in range [-rank, rank), -// where rank is the number of dimensions in each input `SparseTensor`. -// -// Returns 2-D. Indices of the concatenated `SparseTensor`.1-D. Non-empty values of the concatenated `SparseTensor`.1-D. Shape of the concatenated `SparseTensor`. -func SparseConcat(scope *Scope, indices []tf.Output, values []tf.Output, shapes []tf.Output, concat_dim int64) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) { +// a_indices: 2-D. The `indices` of the `SparseTensor`, size `[nnz, 2]` Matrix. +// a_values: 1-D. The `values` of the `SparseTensor`, size `[nnz]` Vector. +// a_shape: 1-D. The `shape` of the `SparseTensor`, size `[2]` Vector. +// b: 2-D. A dense Matrix. +func SparseTensorDenseMatMul(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b tf.Output, optional ...SparseTensorDenseMatMulAttr) (product tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{"concat_dim": concat_dim} + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } opspec := tf.OpSpec{ - Type: "SparseConcat", + Type: "SparseTensorDenseMatMul", Input: []tf.Input{ - tf.OutputList(indices), tf.OutputList(values), tf.OutputList(shapes), + a_indices, a_values, a_shape, b, }, Attrs: attrs, } op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1), op.Output(2) + return op.Output(0) } -// Generates sparse cross from a list of sparse and dense tensors. -// -// The op takes two lists, one of 2D `SparseTensor` and one of 2D `Tensor`, each -// representing features of one feature column. It outputs a 2D `SparseTensor` with -// the batchwise crosses of these features. -// -// For example, if the inputs are -// -// inputs[0]: SparseTensor with shape = [2, 2] -// [0, 0]: "a" -// [1, 0]: "b" -// [1, 1]: "c" -// -// inputs[1]: SparseTensor with shape = [2, 1] -// [0, 0]: "d" -// [1, 0]: "e" -// -// inputs[2]: Tensor [["f"], ["g"]] +// Deserialize and concatenate `SparseTensors` from a serialized minibatch. // -// then the output will be +// The input `serialized_sparse` must be a string matrix of shape `[N x 3]` where +// `N` is the minibatch size and the rows correspond to packed outputs of +// `SerializeSparse`. The ranks of the original `SparseTensor` objects +// must all match. When the final `SparseTensor` is created, it has rank one +// higher than the ranks of the incoming `SparseTensor` objects +// (they have been concatenated along a new row dimension). // -// shape = [2, 2] -// [0, 0]: "a_X_d_X_f" -// [1, 0]: "b_X_e_X_g" -// [1, 1]: "c_X_e_X_g" +// The output `SparseTensor` object's shape values for all dimensions but the +// first are the max across the input `SparseTensor` objects' shape values +// for the corresponding dimensions. Its first shape value is `N`, the minibatch +// size. // -// if hashed_output=true then the output will be +// The input `SparseTensor` objects' indices are assumed ordered in +// standard lexicographic order. If this is not the case, after this +// step run `SparseReorder` to restore index ordering. // -// shape = [2, 2] -// [0, 0]: FingerprintCat64( -// Fingerprint64("f"), FingerprintCat64( -// Fingerprint64("d"), Fingerprint64("a"))) -// [1, 0]: FingerprintCat64( -// Fingerprint64("g"), FingerprintCat64( -// Fingerprint64("e"), Fingerprint64("b"))) -// [1, 1]: FingerprintCat64( -// Fingerprint64("g"), FingerprintCat64( -// Fingerprint64("e"), Fingerprint64("c"))) +// For example, if the serialized input is a `[2 x 3]` matrix representing two +// original `SparseTensor` objects: // -// Arguments: -// indices: 2-D. Indices of each input `SparseTensor`. -// values: 1-D. values of each `SparseTensor`. -// shapes: 1-D. Shapes of each `SparseTensor`. -// dense_inputs: 2-D. Columns represented by dense `Tensor`. -// hashed_output: If true, returns the hash of the cross instead of the string. -// This will allow us avoiding string manipulations. -// num_buckets: It is used if hashed_output is true. -// output = hashed_value%num_buckets if num_buckets > 0 else hashed_value. -// hash_key: Specify the hash_key that will be used by the `FingerprintCat64` -// function to combine the crosses fingerprints. +// index = [ 0] +// [10] +// [20] +// values = [1, 2, 3] +// shape = [50] // +// and // +// index = [ 2] +// [10] +// values = [4, 5] +// shape = [30] // -// Returns 2-D. Indices of the concatenated `SparseTensor`.1-D. Non-empty values of the concatenated or hashed -// `SparseTensor`.1-D. Shape of the concatenated `SparseTensor`. -func SparseCross(scope *Scope, indices []tf.Output, values []tf.Output, shapes []tf.Output, dense_inputs []tf.Output, hashed_output bool, num_buckets int64, hash_key int64, out_type tf.DataType, internal_type tf.DataType) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) { +// then the final deserialized `SparseTensor` will be: +// +// index = [0 0] +// [0 10] +// [0 20] +// [1 2] +// [1 10] +// values = [1, 2, 3, 4, 5] +// shape = [2 50] +// +// Arguments: +// serialized_sparse: 2-D, The `N` serialized `SparseTensor` objects. +// Must have 3 columns. +// dtype: The `dtype` of the serialized `SparseTensor` objects. +func DeserializeManySparse(scope *Scope, serialized_sparse tf.Output, dtype tf.DataType) (sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{"hashed_output": hashed_output, "num_buckets": num_buckets, "hash_key": hash_key, "out_type": out_type, "internal_type": internal_type} + attrs := map[string]interface{}{"dtype": dtype} opspec := tf.OpSpec{ - Type: "SparseCross", + Type: "DeserializeManySparse", Input: []tf.Input{ - tf.OutputList(indices), tf.OutputList(values), tf.OutputList(shapes), tf.OutputList(dense_inputs), + serialized_sparse, }, Attrs: attrs, } @@ -15611,287 +15619,242 @@ func SparseCross(scope *Scope, indices []tf.Output, values []tf.Output, shapes [ return op.Output(0), op.Output(1), op.Output(2) } -// Concatenates quantized tensors along one dimension. -// -// Arguments: -// concat_dim: 0-D. The dimension along which to concatenate. Must be in the -// range [0, rank(values)). -// values: The `N` Tensors to concatenate. Their ranks and types must match, -// and their sizes must match in all dimensions except `concat_dim`. -// input_mins: The minimum scalar values for each of the input tensors. -// input_maxes: The maximum scalar values for each of the input tensors. +// StringJoinAttr is an optional argument to StringJoin. +type StringJoinAttr func(optionalAttr) + +// StringJoinSeparator sets the optional separator attribute to value. // -// Returns A `Tensor` with the concatenation of values stacked along the -// `concat_dim` dimension. This tensor's shape matches that of `values` except -// in `concat_dim` where it has the sum of the sizes.The float value that the minimum quantized output value represents.The float value that the maximum quantized output value represents. -func QuantizedConcat(scope *Scope, concat_dim tf.Output, values []tf.Output, input_mins []tf.Output, input_maxes []tf.Output) (output tf.Output, output_min tf.Output, output_max tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "QuantizedConcat", - Input: []tf.Input{ - concat_dim, tf.OutputList(values), tf.OutputList(input_mins), tf.OutputList(input_maxes), - }, +// value: string, an optional join separator. +// If not specified, defaults to "" +func StringJoinSeparator(value string) StringJoinAttr { + return func(m optionalAttr) { + m["separator"] = value } - op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1), op.Output(2) } -// Slice a `SparseTensor` based on the `start` and `size`. -// -// For example, if the input is -// -// input_tensor = shape = [2, 7] -// [ a d e ] -// [b c ] -// -// Graphically the output tensors are: -// -// sparse_slice([0, 0], [2, 4]) = shape = [2, 4] -// [ a ] -// [b c ] +// Joins the strings in the given list of string tensors into one tensor; // -// sparse_slice([0, 4], [2, 3]) = shape = [2, 3] -// [ d e ] -// [ ] +// with the given separator (default is an empty separator). // // Arguments: -// indices: 2-D tensor represents the indices of the sparse tensor. -// values: 1-D tensor represents the values of the sparse tensor. -// shape: 1-D. tensor represents the shape of the sparse tensor. -// start: 1-D. tensor represents the start of the slice. -// size: 1-D. tensor represents the size of the slice. -// output indices: A list of 1-D tensors represents the indices of the output -// sparse tensors. -// -// Returns A list of 1-D tensors represents the values of the output sparse -// tensors.A list of 1-D tensors represents the shape of the output sparse -// tensors. -func SparseSlice(scope *Scope, indices tf.Output, values tf.Output, shape tf.Output, start tf.Output, size tf.Output) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) { +// inputs: A list of string tensors. The tensors must all have the same shape, +// or be scalars. Scalars may be mixed in; these will be broadcast to the shape +// of non-scalar inputs. +func StringJoin(scope *Scope, inputs []tf.Output, optional ...StringJoinAttr) (output tf.Output) { if scope.Err() != nil { return } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } opspec := tf.OpSpec{ - Type: "SparseSlice", + Type: "StringJoin", Input: []tf.Input{ - indices, values, shape, start, size, + tf.OutputList(inputs), }, + Attrs: attrs, } op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1), op.Output(2) + return op.Output(0) } -// Adds up a `SparseTensor` and a dense `Tensor`, producing a dense `Tensor`. +// Returns immutable tensor from memory region. // -// This Op does not require `a_indices` be sorted in standard lexicographic order. +// The current implementation memmaps the tensor from a file. // // Arguments: -// a_indices: 2-D. The `indices` of the `SparseTensor`, with shape `[nnz, ndims]`. -// a_values: 1-D. The `values` of the `SparseTensor`, with shape `[nnz]`. -// a_shape: 1-D. The `shape` of the `SparseTensor`, with shape `[ndims]`. -// b: `ndims`-D Tensor. With shape `a_shape`. -func SparseTensorDenseAdd(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b tf.Output) (output tf.Output) { +// dtype: Type of the returned tensor. +// shape: Shape of the returned tensor. +// memory_region_name: Name of readonly memory region used by the tensor, see +// NewReadOnlyMemoryRegionFromFile in tensorflow::Env. +func ImmutableConst(scope *Scope, dtype tf.DataType, shape tf.Shape, memory_region_name string) (tensor tf.Output) { if scope.Err() != nil { return } + attrs := map[string]interface{}{"dtype": dtype, "shape": shape, "memory_region_name": memory_region_name} opspec := tf.OpSpec{ - Type: "SparseTensorDenseAdd", - Input: []tf.Input{ - a_indices, a_values, a_shape, b, - }, + Type: "ImmutableConst", + + Attrs: attrs, } op := scope.AddOperation(opspec) return op.Output(0) } -// Returns the set of files matching one or more glob patterns. +// Inverse real-valued fast Fourier transform. // -// Note that this routine only supports wildcard characters in the -// basename portion of the pattern, not in the directory portion. +// Computes the inverse 1-dimensional discrete Fourier transform of a real-valued +// signal over the inner-most dimension of `input`. +// +// The inner-most dimension of `input` is assumed to be the result of `RFFT`: the +// `fft_length / 2 + 1` unique components of the DFT of a real-valued signal. If +// `fft_length` is not provided, it is computed from the size of the inner-most +// dimension of `input` (`fft_length = 2 * (inner - 1)`). If the FFT length used to +// compute `input` is odd, it should be provided since it cannot be inferred +// properly. +// +// Along the axis `IRFFT` is computed on, if `fft_length / 2 + 1` is smaller +// than the corresponding dimension of `input`, the dimension is cropped. If it is +// larger, the dimension is padded with zeros. // // Arguments: -// pattern: Shell wildcard pattern(s). Scalar or vector of type string. +// input: A complex64 tensor. +// fft_length: An int32 tensor of shape [1]. The FFT length. // -// Returns A vector of matching filenames. -func MatchingFiles(scope *Scope, pattern tf.Output) (filenames tf.Output) { +// Returns A float32 tensor of the same rank as `input`. The inner-most +// dimension of `input` is replaced with the `fft_length` samples of its inverse +// 1D Fourier transform. +// +// @compatibility(numpy) +// Equivalent to np.fft.irfft +// @end_compatibility +func IRFFT(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) { if scope.Err() != nil { return } opspec := tf.OpSpec{ - Type: "MatchingFiles", + Type: "IRFFT", Input: []tf.Input{ - pattern, + input, fft_length, }, } op := scope.AddOperation(opspec) return op.Output(0) } -// MatrixSolveLsAttr is an optional argument to MatrixSolveLs. -type MatrixSolveLsAttr func(optionalAttr) - -// MatrixSolveLsFast sets the optional fast attribute to value. -// If not specified, defaults to true -func MatrixSolveLsFast(value bool) MatrixSolveLsAttr { - return func(m optionalAttr) { - m["fast"] = value - } -} - -// Solves one or more linear least-squares problems. +// Concatenates a list of `SparseTensor` along the specified dimension. // -// `matrix` is a tensor of shape `[..., M, N]` whose inner-most 2 dimensions -// form real or complex matrices of size `[M, N]`. `Rhs` is a tensor of the same -// type as `matrix` and shape `[..., M, K]`. -// The output is a tensor shape `[..., N, K]` where each output matrix solves -// each of the equations -// `matrix[..., :, :]` * `output[..., :, :]` = `rhs[..., :, :]` -// in the least squares sense. +// Concatenation is with respect to the dense versions of these sparse tensors. +// It is assumed that each input is a `SparseTensor` whose elements are ordered +// along increasing dimension number. // -// We use the following notation for (complex) matrix and right-hand sides -// in the batch: +// All inputs' shapes must match, except for the concat dimension. The +// `indices`, `values`, and `shapes` lists must have the same length. // -// `matrix`=\\(A \in \mathbb{C}^{m \times n}\\), -// `rhs`=\\(B \in \mathbb{C}^{m \times k}\\), -// `output`=\\(X \in \mathbb{C}^{n \times k}\\), -// `l2_regularizer`=\\(\lambda \in \mathbb{R}\\). +// The output shape is identical to the inputs', except along the concat +// dimension, where it is the sum of the inputs' sizes along that dimension. // -// If `fast` is `True`, then the solution is computed by solving the normal -// equations using Cholesky decomposition. Specifically, if \\(m \ge n\\) then -// \\(X = (A^H A + \lambda I)^{-1} A^H B\\), which solves the least-squares -// problem \\(X = \mathrm{argmin}_{Z \in \Re^{n \times k} } ||A Z - B||_F^2 + -// \lambda ||Z||_F^2\\). If \\(m \lt n\\) then `output` is computed as -// \\(X = A^H (A A^H + \lambda I)^{-1} B\\), which (for \\(\lambda = 0\\)) is the -// minimum-norm solution to the under-determined linear system, i.e. -// \\(X = \mathrm{argmin}_{Z \in \mathbb{C}^{n \times k} } ||Z||_F^2 \\), -// subject to \\(A Z = B\\). Notice that the fast path is only numerically stable -// when \\(A\\) is numerically full rank and has a condition number -// \\(\mathrm{cond}(A) \lt \frac{1}{\sqrt{\epsilon_{mach} } }\\) or\\(\lambda\\) is -// sufficiently large. +// The output elements will be resorted to preserve the sort order along +// increasing dimension number. // -// If `fast` is `False` an algorithm based on the numerically robust complete -// orthogonal decomposition is used. This computes the minimum-norm -// least-squares solution, even when \\(A\\) is rank deficient. This path is -// typically 6-7 times slower than the fast path. If `fast` is `False` then -// `l2_regularizer` is ignored. +// This op runs in `O(M log M)` time, where `M` is the total number of non-empty +// values across all inputs. This is due to the need for an internal sort in +// order to concatenate efficiently across an arbitrary dimension. // -// Arguments: -// matrix: Shape is `[..., M, N]`. -// rhs: Shape is `[..., M, K]`. -// l2_regularizer: Scalar tensor. +// For example, if `concat_dim = 1` and the inputs are // -// @compatibility(numpy) -// Equivalent to np.linalg.lstsq -// @end_compatibility +// sp_inputs[0]: shape = [2, 3] +// [0, 2]: "a" +// [1, 0]: "b" +// [1, 1]: "c" // -// Returns Shape is `[..., N, K]`. -func MatrixSolveLs(scope *Scope, matrix tf.Output, rhs tf.Output, l2_regularizer tf.Output, optional ...MatrixSolveLsAttr) (output tf.Output) { +// sp_inputs[1]: shape = [2, 4] +// [0, 1]: "d" +// [0, 2]: "e" +// +// then the output will be +// +// shape = [2, 7] +// [0, 2]: "a" +// [0, 4]: "d" +// [0, 5]: "e" +// [1, 0]: "b" +// [1, 1]: "c" +// +// Graphically this is equivalent to doing +// +// [ a] concat [ d e ] = [ a d e ] +// [b c ] [ ] [b c ] +// +// Arguments: +// indices: 2-D. Indices of each input `SparseTensor`. +// values: 1-D. Non-empty values of each `SparseTensor`. +// shapes: 1-D. Shapes of each `SparseTensor`. +// concat_dim: Dimension to concatenate along. Must be in range [-rank, rank), +// where rank is the number of dimensions in each input `SparseTensor`. +// +// Returns 2-D. Indices of the concatenated `SparseTensor`.1-D. Non-empty values of the concatenated `SparseTensor`.1-D. Shape of the concatenated `SparseTensor`. +func SparseConcat(scope *Scope, indices []tf.Output, values []tf.Output, shapes []tf.Output, concat_dim int64) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } + attrs := map[string]interface{}{"concat_dim": concat_dim} opspec := tf.OpSpec{ - Type: "MatrixSolveLs", + Type: "SparseConcat", Input: []tf.Input{ - matrix, rhs, l2_regularizer, + tf.OutputList(indices), tf.OutputList(values), tf.OutputList(shapes), }, Attrs: attrs, } op := scope.AddOperation(opspec) - return op.Output(0) + return op.Output(0), op.Output(1), op.Output(2) } -// Elementwise computes the bitwise OR of `x` and `y`. +// Generates sparse cross from a list of sparse and dense tensors. // -// The result will have those bits set, that are set in `x`, `y` or both. The -// computation is performed on the underlying representations of `x` and `y`. -func BitwiseOr(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "BitwiseOr", - Input: []tf.Input{ - x, y, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// SparseToSparseSetOperationAttr is an optional argument to SparseToSparseSetOperation. -type SparseToSparseSetOperationAttr func(optionalAttr) - -// SparseToSparseSetOperationValidateIndices sets the optional validate_indices attribute to value. -// If not specified, defaults to true -func SparseToSparseSetOperationValidateIndices(value bool) SparseToSparseSetOperationAttr { - return func(m optionalAttr) { - m["validate_indices"] = value - } -} - -// Applies set operation along last dimension of 2 `SparseTensor` inputs. +// The op takes two lists, one of 2D `SparseTensor` and one of 2D `Tensor`, each +// representing features of one feature column. It outputs a 2D `SparseTensor` with +// the batchwise crosses of these features. // -// See SetOperationOp::SetOperationFromContext for values of `set_operation`. +// For example, if the inputs are // -// If `validate_indices` is `True`, `SparseToSparseSetOperation` validates the -// order and range of `set1` and `set2` indices. +// inputs[0]: SparseTensor with shape = [2, 2] +// [0, 0]: "a" +// [1, 0]: "b" +// [1, 1]: "c" // -// Input `set1` is a `SparseTensor` represented by `set1_indices`, `set1_values`, -// and `set1_shape`. For `set1` ranked `n`, 1st `n-1` dimensions must be the same -// as `set2`. Dimension `n` contains values in a set, duplicates are allowed but -// ignored. +// inputs[1]: SparseTensor with shape = [2, 1] +// [0, 0]: "d" +// [1, 0]: "e" // -// Input `set2` is a `SparseTensor` represented by `set2_indices`, `set2_values`, -// and `set2_shape`. For `set2` ranked `n`, 1st `n-1` dimensions must be the same -// as `set1`. Dimension `n` contains values in a set, duplicates are allowed but -// ignored. +// inputs[2]: Tensor [["f"], ["g"]] // -// If `validate_indices` is `True`, this op validates the order and range of `set1` -// and `set2` indices. +// then the output will be // -// Output `result` is a `SparseTensor` represented by `result_indices`, -// `result_values`, and `result_shape`. For `set1` and `set2` ranked `n`, this -// has rank `n` and the same 1st `n-1` dimensions as `set1` and `set2`. The `nth` -// dimension contains the result of `set_operation` applied to the corresponding -// `[0...n-1]` dimension of `set`. +// shape = [2, 2] +// [0, 0]: "a_X_d_X_f" +// [1, 0]: "b_X_e_X_g" +// [1, 1]: "c_X_e_X_g" +// +// if hashed_output=true then the output will be +// +// shape = [2, 2] +// [0, 0]: FingerprintCat64( +// Fingerprint64("f"), FingerprintCat64( +// Fingerprint64("d"), Fingerprint64("a"))) +// [1, 0]: FingerprintCat64( +// Fingerprint64("g"), FingerprintCat64( +// Fingerprint64("e"), Fingerprint64("b"))) +// [1, 1]: FingerprintCat64( +// Fingerprint64("g"), FingerprintCat64( +// Fingerprint64("e"), Fingerprint64("c"))) // // Arguments: -// set1_indices: 2D `Tensor`, indices of a `SparseTensor`. Must be in row-major -// order. -// set1_values: 1D `Tensor`, values of a `SparseTensor`. Must be in row-major -// order. -// set1_shape: 1D `Tensor`, shape of a `SparseTensor`. `set1_shape[0...n-1]` must -// be the same as `set2_shape[0...n-1]`, `set1_shape[n]` is the -// max set size across `0...n-1` dimensions. -// set2_indices: 2D `Tensor`, indices of a `SparseTensor`. Must be in row-major -// order. -// set2_values: 1D `Tensor`, values of a `SparseTensor`. Must be in row-major -// order. -// set2_shape: 1D `Tensor`, shape of a `SparseTensor`. `set2_shape[0...n-1]` must -// be the same as `set1_shape[0...n-1]`, `set2_shape[n]` is the -// max set size across `0...n-1` dimensions. +// indices: 2-D. Indices of each input `SparseTensor`. +// values: 1-D. values of each `SparseTensor`. +// shapes: 1-D. Shapes of each `SparseTensor`. +// dense_inputs: 2-D. Columns represented by dense `Tensor`. +// hashed_output: If true, returns the hash of the cross instead of the string. +// This will allow us avoiding string manipulations. +// num_buckets: It is used if hashed_output is true. +// output = hashed_value%num_buckets if num_buckets > 0 else hashed_value. +// hash_key: Specify the hash_key that will be used by the `FingerprintCat64` +// function to combine the crosses fingerprints. // // -// Returns 2D indices of a `SparseTensor`.1D values of a `SparseTensor`.1D `Tensor` shape of a `SparseTensor`. `result_shape[0...n-1]` is -// the same as the 1st `n-1` dimensions of `set1` and `set2`, `result_shape[n]` -// is the max result set size across all `0...n-1` dimensions. -func SparseToSparseSetOperation(scope *Scope, set1_indices tf.Output, set1_values tf.Output, set1_shape tf.Output, set2_indices tf.Output, set2_values tf.Output, set2_shape tf.Output, set_operation string, optional ...SparseToSparseSetOperationAttr) (result_indices tf.Output, result_values tf.Output, result_shape tf.Output) { +// +// Returns 2-D. Indices of the concatenated `SparseTensor`.1-D. Non-empty values of the concatenated or hashed +// `SparseTensor`.1-D. Shape of the concatenated `SparseTensor`. +func SparseCross(scope *Scope, indices []tf.Output, values []tf.Output, shapes []tf.Output, dense_inputs []tf.Output, hashed_output bool, num_buckets int64, hash_key int64, out_type tf.DataType, internal_type tf.DataType) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{"set_operation": set_operation} - for _, a := range optional { - a(attrs) - } + attrs := map[string]interface{}{"hashed_output": hashed_output, "num_buckets": num_buckets, "hash_key": hash_key, "out_type": out_type, "internal_type": internal_type} opspec := tf.OpSpec{ - Type: "SparseToSparseSetOperation", + Type: "SparseCross", Input: []tf.Input{ - set1_indices, set1_values, set1_shape, set2_indices, set2_values, set2_shape, + tf.OutputList(indices), tf.OutputList(values), tf.OutputList(shapes), tf.OutputList(dense_inputs), }, Attrs: attrs, } @@ -15899,71 +15862,75 @@ func SparseToSparseSetOperation(scope *Scope, set1_indices tf.Output, set1_value return op.Output(0), op.Output(1), op.Output(2) } -// Computes numerical negative value element-wise. +// Concatenates quantized tensors along one dimension. // -// I.e., \\(y = -x\\). -func Neg(scope *Scope, x tf.Output) (y tf.Output) { +// Arguments: +// concat_dim: 0-D. The dimension along which to concatenate. Must be in the +// range [0, rank(values)). +// values: The `N` Tensors to concatenate. Their ranks and types must match, +// and their sizes must match in all dimensions except `concat_dim`. +// input_mins: The minimum scalar values for each of the input tensors. +// input_maxes: The maximum scalar values for each of the input tensors. +// +// Returns A `Tensor` with the concatenation of values stacked along the +// `concat_dim` dimension. This tensor's shape matches that of `values` except +// in `concat_dim` where it has the sum of the sizes.The float value that the minimum quantized output value represents.The float value that the maximum quantized output value represents. +func QuantizedConcat(scope *Scope, concat_dim tf.Output, values []tf.Output, input_mins []tf.Output, input_maxes []tf.Output) (output tf.Output, output_min tf.Output, output_max tf.Output) { if scope.Err() != nil { return } opspec := tf.OpSpec{ - Type: "Neg", + Type: "QuantizedConcat", Input: []tf.Input{ - x, + concat_dim, tf.OutputList(values), tf.OutputList(input_mins), tf.OutputList(input_maxes), }, } op := scope.AddOperation(opspec) - return op.Output(0) -} - -// FakeQuantWithMinMaxVarsAttr is an optional argument to FakeQuantWithMinMaxVars. -type FakeQuantWithMinMaxVarsAttr func(optionalAttr) - -// FakeQuantWithMinMaxVarsNumBits sets the optional num_bits attribute to value. -// If not specified, defaults to 8 -func FakeQuantWithMinMaxVarsNumBits(value int64) FakeQuantWithMinMaxVarsAttr { - return func(m optionalAttr) { - m["num_bits"] = value - } -} - -// FakeQuantWithMinMaxVarsNarrowRange sets the optional narrow_range attribute to value. -// If not specified, defaults to false -func FakeQuantWithMinMaxVarsNarrowRange(value bool) FakeQuantWithMinMaxVarsAttr { - return func(m optionalAttr) { - m["narrow_range"] = value - } + return op.Output(0), op.Output(1), op.Output(2) } -// Fake-quantize the 'inputs' tensor of type float via global float scalars `min` +// Slice a `SparseTensor` based on the `start` and `size`. // -// and `max` to 'outputs' tensor of same shape as `inputs`. +// For example, if the input is // -// `[min; max]` define the clamping range for the `inputs` data. -// `inputs` values are quantized into the quantization range (`[0; 2^num_bits - 1]` -// when `narrow_range` is false and `[1; 2^num_bits - 1]` when it is true) and -// then de-quantized and output as floats in `[min; max]` interval. -// `num_bits` is the bitwidth of the quantization; between 2 and 8, inclusive. +// input_tensor = shape = [2, 7] +// [ a d e ] +// [b c ] // -// This operation has a gradient and thus allows for training `min` and `max` -// values. -func FakeQuantWithMinMaxVars(scope *Scope, inputs tf.Output, min tf.Output, max tf.Output, optional ...FakeQuantWithMinMaxVarsAttr) (outputs tf.Output) { +// Graphically the output tensors are: +// +// sparse_slice([0, 0], [2, 4]) = shape = [2, 4] +// [ a ] +// [b c ] +// +// sparse_slice([0, 4], [2, 3]) = shape = [2, 3] +// [ d e ] +// [ ] +// +// Arguments: +// indices: 2-D tensor represents the indices of the sparse tensor. +// values: 1-D tensor represents the values of the sparse tensor. +// shape: 1-D. tensor represents the shape of the sparse tensor. +// start: 1-D. tensor represents the start of the slice. +// size: 1-D. tensor represents the size of the slice. +// output indices: A list of 1-D tensors represents the indices of the output +// sparse tensors. +// +// Returns A list of 1-D tensors represents the values of the output sparse +// tensors.A list of 1-D tensors represents the shape of the output sparse +// tensors. +func SparseSlice(scope *Scope, indices tf.Output, values tf.Output, shape tf.Output, start tf.Output, size tf.Output) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } opspec := tf.OpSpec{ - Type: "FakeQuantWithMinMaxVars", + Type: "SparseSlice", Input: []tf.Input{ - inputs, min, max, + indices, values, shape, start, size, }, - Attrs: attrs, } op := scope.AddOperation(opspec) - return op.Output(0) + return op.Output(0), op.Output(1), op.Output(2) } // Returns the element-wise min of two SparseTensors. @@ -18018,6 +17985,39 @@ func MatrixBandPart(scope *Scope, input tf.Output, num_lower tf.Output, num_uppe return op.Output(0) } +// Counts the number of occurrences of each value in an integer array. +// +// Outputs a vector with length `size` and the same dtype as `weights`. If +// `weights` are empty, then index `i` stores the number of times the value `i` is +// counted in `arr`. If `weights` are non-empty, then index `i` stores the sum of +// the value in `weights` at each index where the corresponding value in `arr` is +// `i`. +// +// Values in `arr` outside of the range [0, size) are ignored. +// +// Arguments: +// arr: int32 `Tensor`. +// size: non-negative int32 scalar `Tensor`. +// weights: is an int32, int64, float32, or float64 `Tensor` with the same +// shape as `arr`, or a length-0 `Tensor`, in which case it acts as all weights +// equal to 1. +// +// Returns 1D `Tensor` with length equal to `size`. The counts or summed weights for +// each value in the range [0, size). +func Bincount(scope *Scope, arr tf.Output, size tf.Output, weights tf.Output) (bins tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "Bincount", + Input: []tf.Input{ + arr, size, weights, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // CumsumAttr is an optional argument to Cumsum. type CumsumAttr func(optionalAttr) -- GitLab From 902625480b414562e9a4e21e963cacaa4708f9b2 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 19:50:10 -0700 Subject: [PATCH 229/791] Enable a reduce window test case. PiperOrigin-RevId: 192548652 --- tensorflow/compiler/xla/tests/reduce_window_test.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/compiler/xla/tests/reduce_window_test.cc b/tensorflow/compiler/xla/tests/reduce_window_test.cc index 8ef980ebd9..425fef7da7 100644 --- a/tensorflow/compiler/xla/tests/reduce_window_test.cc +++ b/tensorflow/compiler/xla/tests/reduce_window_test.cc @@ -1063,14 +1063,14 @@ struct R2ReduceWindowTestData { /*strides=*/{1, 1}, /*pad_low=*/{0, 130}, /*pad_high=*/{0, 0}, /*layout=*/{1, 0}, /*reducer=*/Reducer::kAdd}, + {/*base_bounds=*/{8, 256}, /*window_bounds=*/{1, 4}, + /*strides=*/{1, 64}, /*pad_low=*/{0, 0}, /*pad_high=*/{0, 0}, + /*layout=*/{1, 0}, /*reducer=*/Reducer::kAdd}, // TODO(b/76025683): These tests fail on TPU. #if defined(XLA_TEST_BACKEND_CPU) || defined(XLA_TEST_BACKEND_GPU) {/*base_bounds=*/{4096, 4096}, /*window_bounds=*/{1, 4}, /*strides=*/{1, 1024}, /*pad_low=*/{0, 0}, /*pad-high=*/{0, 0}, /*layout=*/{1, 0}, /*reducer=*/Reducer::kAdd}, - {/*base_bounds=*/{8, 256}, /*window_bounds=*/{1, 4}, - /*strides=*/{1, 64}, /*pad_low=*/{0, 0}, /*pad_high=*/{0, 0}, - /*layout=*/{1, 0}, /*reducer=*/Reducer::kAdd}, #endif }; -- GitLab From 5b0cb6c724e12e0d66a11d8043c71d1479f70a47 Mon Sep 17 00:00:00 2001 From: James Wexler Date: Wed, 11 Apr 2018 19:58:07 -0700 Subject: [PATCH 230/791] Add closure_js_proto_library build for tf.example protos. PiperOrigin-RevId: 192549109 --- tensorflow/core/BUILD | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index c5ca421ced..55b0040b52 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -70,6 +70,10 @@ package(default_visibility = [ licenses(["notice"]) # Apache 2.0 +load( + "@io_bazel_rules_closure//closure:defs.bzl", + "closure_js_proto_library", +) load( "//tensorflow:tensorflow.bzl", "full_path", @@ -244,6 +248,14 @@ tf_nano_proto_library( deps = [":protos_all_cc"], ) +closure_js_proto_library( + name = "example_js_protos", + srcs = [ + "example/example.proto", + "example/feature.proto", + ], +) + exports_files([ "framework/types.proto", ]) -- GitLab From ac9be81b06e9bf93d8ba5f37983c3dd1163a190e Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 20:08:30 -0700 Subject: [PATCH 231/791] Fix description of DynamicUpdateSlice. PiperOrigin-RevId: 192550101 --- .../docs_src/performance/xla/operation_semantics.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tensorflow/docs_src/performance/xla/operation_semantics.md b/tensorflow/docs_src/performance/xla/operation_semantics.md index 217ab596b7..3963d5faa7 100644 --- a/tensorflow/docs_src/performance/xla/operation_semantics.md +++ b/tensorflow/docs_src/performance/xla/operation_semantics.md @@ -854,12 +854,13 @@ calculation of 'start_indices') is currently implementation-defined. | `operand` | `ComputationDataHandle` | N dimensional array of type T | | `update` | `ComputationDataHandle` | N dimensional array of type T | : : : containing the slice update. : -: : : Each dimension of update shape : +: : : Each dimension of update shape : : : : must be strictly greater than : : : : zero, and start + update must be : -: : : less than operand size for each : -: : : dimension to avoid generating : -: : : out-of-bounds update indices. : +: : : less than or equal to the operand: +: : : size for each dimension to avoid : +: : : generating out-of-bounds update : +: : : indices. : | `start_indices` | `ComputationDataHandle` | Rank 1 array of N integers | : : : containing the starting indices : : : : of the slice for each dimension. : -- GitLab From 28fdb0a6b1714a634ead04602732b1c75212fb94 Mon Sep 17 00:00:00 2001 From: Sami Kama Date: Wed, 11 Apr 2018 20:19:27 -0700 Subject: [PATCH 232/791] Fix double linkage of static variables --- tensorflow/contrib/tensorrt/BUILD | 30 +++++++++++++++++-- .../resources/trt_resource_manager.cc | 8 +++++ .../tensorrt/resources/trt_resource_manager.h | 6 +--- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD index 2f316767b3..2a55a49097 100644 --- a/tensorflow/contrib/tensorrt/BUILD +++ b/tensorflow/contrib/tensorrt/BUILD @@ -27,6 +27,11 @@ load( "if_tensorrt", ) +load( + "//tensorflow/core:platform/default/build_config_root.bzl", + "if_static", +) + tf_cuda_cc_test( name = "tensorrt_test_cc", size = "small", @@ -52,7 +57,7 @@ tf_custom_op_library( "ops/trt_engine_op.cc", ], deps = [ - ":trt_engine_op_kernel", + # ":trt_engine_op_kernel", ":trt_shape_function", "//tensorflow/core:lib_proto_parsing", ] + if_tensorrt([ @@ -183,16 +188,34 @@ tf_py_wrap_cc( copts = tf_copts(), deps = [ ":trt_conversion", + ":trt_engine_op_kernel", "//tensorflow/core:framework_lite", "//util/python:python_headers", ], ) +tf_cuda_library( + name = "trt_resource_manager_impl", + srcs = [ + "resources/trt_resource_manager.cc", + ], + hdrs = [ + "resources/trt_resource_manager.h", + ], + deps = [ + ":trt_logging", + "//tensorflow/core:framework_headers_lib", + "//tensorflow/core:framework_lite", + "//tensorflow/core:lib_proto_parsing", + ] + if_tensorrt([ + "@local_config_tensorrt//:nv_infer", + ]), +) + tf_cuda_library( name = "trt_resources", srcs = [ "resources/trt_int8_calibrator.cc", - "resources/trt_resource_manager.cc", ], hdrs = [ "resources/trt_int8_calibrator.h", @@ -206,6 +229,8 @@ tf_cuda_library( "//tensorflow/core:lib_proto_parsing", ] + if_tensorrt([ "@local_config_tensorrt//:nv_infer", + ]) + if_static([ + ":trt_resource_manager_impl", ]), ) @@ -224,6 +249,7 @@ tf_cuda_library( ":segment", ":trt_logging", ":trt_resources", + ":trt_resource_manager_impl", "//tensorflow/core/grappler:grappler_item", "//tensorflow/core/grappler:utils", "//tensorflow/core:framework", diff --git a/tensorflow/contrib/tensorrt/resources/trt_resource_manager.cc b/tensorflow/contrib/tensorrt/resources/trt_resource_manager.cc index e663eed4dd..b9a5a00366 100644 --- a/tensorflow/contrib/tensorrt/resources/trt_resource_manager.cc +++ b/tensorflow/contrib/tensorrt/resources/trt_resource_manager.cc @@ -19,6 +19,14 @@ limitations under the License. namespace tensorflow { namespace tensorrt { +std::shared_ptr +tensorflow::tensorrt::TRTResourceManager::instance() +{ + static std::shared_ptr instance_( + new tensorflow::tensorrt::TRTResourceManager); + return instance_; +} + std::shared_ptr tensorflow::tensorrt::TRTResourceManager::getManager(const string& op_name) { // mutex is held for lookup only. Most instantiations where mutex will be held diff --git a/tensorflow/contrib/tensorrt/resources/trt_resource_manager.h b/tensorflow/contrib/tensorrt/resources/trt_resource_manager.h index 5f8ad491d3..bc15b51e05 100644 --- a/tensorflow/contrib/tensorrt/resources/trt_resource_manager.h +++ b/tensorflow/contrib/tensorrt/resources/trt_resource_manager.h @@ -29,11 +29,7 @@ class TRTResourceManager { TRTResourceManager() = default; public: - static std::shared_ptr instance() { - static std::shared_ptr instance_( - new TRTResourceManager); - return instance_; - } + static std::shared_ptr instance(); // returns a manager for given op, if it doesn't exists it creates one std::shared_ptr getManager(const string& op_name); -- GitLab From 96aba78b0cdb2b9ad316d3c68a52bc2284ea638c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 22:37:57 -0700 Subject: [PATCH 233/791] Enable an r2 reduce window test case. PiperOrigin-RevId: 192560111 --- tensorflow/compiler/xla/tests/reduce_window_test.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/tensorflow/compiler/xla/tests/reduce_window_test.cc b/tensorflow/compiler/xla/tests/reduce_window_test.cc index 425fef7da7..6a054a5dd3 100644 --- a/tensorflow/compiler/xla/tests/reduce_window_test.cc +++ b/tensorflow/compiler/xla/tests/reduce_window_test.cc @@ -1066,12 +1066,9 @@ struct R2ReduceWindowTestData { {/*base_bounds=*/{8, 256}, /*window_bounds=*/{1, 4}, /*strides=*/{1, 64}, /*pad_low=*/{0, 0}, /*pad_high=*/{0, 0}, /*layout=*/{1, 0}, /*reducer=*/Reducer::kAdd}, -// TODO(b/76025683): These tests fail on TPU. -#if defined(XLA_TEST_BACKEND_CPU) || defined(XLA_TEST_BACKEND_GPU) {/*base_bounds=*/{4096, 4096}, /*window_bounds=*/{1, 4}, /*strides=*/{1, 1024}, /*pad_low=*/{0, 0}, /*pad-high=*/{0, 0}, /*layout=*/{1, 0}, /*reducer=*/Reducer::kAdd}, -#endif }; string R2ReduceWindowTestDataToString( -- GitLab From b79de285e04f995eb0220583d6ed333b33a26d7f Mon Sep 17 00:00:00 2001 From: Hovhannes Harutyunyan Date: Thu, 12 Apr 2018 10:04:40 +0400 Subject: [PATCH 234/791] Remove redefined BroadcastDiv function --- .../internal/reference/reference_ops.h | 41 ------------------- 1 file changed, 41 deletions(-) diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h index 4509db06fd..750737a730 100644 --- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h +++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h @@ -1339,47 +1339,6 @@ inline void BroadcastMul(const uint8* input1_data, const Dims<4>& input1_dims, output_data, output_dims); } -// TODO(jiawen): We can implement BroadcastDiv on buffers of arbitrary -// dimensionality if the runtime code does a single loop over one dimension -// that handles broadcasting as the base case. The code generator would then -// generate max(D1, D2) nested for loops. -template -void BroadcastDiv(const T* input1_data, const Dims<4>& input1_dims, - const T* input2_data, const Dims<4>& input2_dims, - T output_activation_min, T output_activation_max, - T* output_data, const Dims<4>& output_dims) { - gemmlowp::ScopedProfilingLabel label("BroadcastDiv"); - - NdArrayDesc<4> desc1; - NdArrayDesc<4> desc2; - NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2); - - // In Tensorflow, the dimensions are canonically named (batch_number, row, - // col, channel), with extents (batches, height, width, depth), with the - // trailing dimension changing most rapidly (channels has the smallest - // stride, typically 1 element). - // - // In generated C code, we store arrays with the dimensions reversed. The - // first dimension has smallest stride. - // - // We name our variables by their Tensorflow convention, but generate C code - // nesting loops such that the innermost loop has the smallest stride for - // the best cache behavior. - for (int b = 0; b < ArraySize(output_dims, 3); ++b) { - for (int y = 0; y < ArraySize(output_dims, 2); ++y) { - for (int x = 0; x < ArraySize(output_dims, 1); ++x) { - for (int c = 0; c < ArraySize(output_dims, 0); ++c) { - output_data[Offset(output_dims, c, x, y, b)] = - ActivationFunctionWithMinMax( - input1_data[SubscriptToIndex(desc1, c, x, y, b)] / - input2_data[SubscriptToIndex(desc2, c, x, y, b)], - output_activation_min, output_activation_max); - } - } - } - } -} - inline void Div(const float* input1_data, const Dims<4>& input1_dims, const float* input2_data, const Dims<4>& input2_dims, float output_activation_min, float output_activation_max, -- GitLab From 09ab7fc83e3b2b66a2d1ff68ac6ad1b56a61fcd6 Mon Sep 17 00:00:00 2001 From: Hovhannes Harutyunyan Date: Thu, 12 Apr 2018 10:54:41 +0400 Subject: [PATCH 235/791] Fixe merge issue --- .../lite/kernels/internal/reference/reference_ops.h | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h index e8d7da73a2..0fc88b2b8e 100644 --- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h +++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h @@ -1323,18 +1323,6 @@ void BroadcastDiv(const T* input1_data, const Dims<4>& input1_dims, } } -inline void Div(const float* input1_data, const Dims<4>& input1_dims, - const float* input2_data, const Dims<4>& input2_dims, - float output_activation_min, float output_activation_max, - float* output_data, const Dims<4>& output_dims) { - const int flat_size = MatchingFlatSize(input1_dims, input2_dims, output_dims); - for (int i = 0; i < flat_size; ++i) { - output_data[i] = ActivationFunctionWithMinMax( - input1_data[i] / input2_data[i], output_activation_min, - output_activation_max); - } -} - inline void Sub(const float* input1_data, const Dims<4>& input1_dims, const float* input2_data, const Dims<4>& input2_dims, float output_activation_min, float output_activation_max, -- GitLab From e688642372893d9e51be4119342f787560d8e644 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 04:40:42 -0700 Subject: [PATCH 236/791] Make DType, TensorShape, and Dimension "reducable" for pickling purposes. PiperOrigin-RevId: 192591402 --- tensorflow/python/framework/dtypes.py | 3 +++ tensorflow/python/framework/dtypes_test.py | 9 +++++++++ tensorflow/python/framework/tensor_shape.py | 6 ++++++ .../python/framework/tensor_shape_test.py | 18 ++++++++++++++++++ 4 files changed, 36 insertions(+) diff --git a/tensorflow/python/framework/dtypes.py b/tensorflow/python/framework/dtypes.py index a31c424263..51ff5171a3 100644 --- a/tensorflow/python/framework/dtypes.py +++ b/tensorflow/python/framework/dtypes.py @@ -297,6 +297,9 @@ class DType(object): def __hash__(self): return self._type_enum + def __reduce__(self): + return as_dtype, (self.name,) + @property def size(self): if (self._type_enum == types_pb2.DT_VARIANT or diff --git a/tensorflow/python/framework/dtypes_test.py b/tensorflow/python/framework/dtypes_test.py index e49e2fda5d..e55783bb79 100644 --- a/tensorflow/python/framework/dtypes_test.py +++ b/tensorflow/python/framework/dtypes_test.py @@ -295,6 +295,15 @@ class TypesTest(test_util.TensorFlowTestCase): self.assertNotEqual(dtypes.int32, int) self.assertNotEqual(dtypes.float64, 2.1) + def testReduce(self): + for enum in dtypes._TYPE_TO_STRING: + dtype = dtypes.DType(enum) + ctor, args = dtype.__reduce__() + self.assertEquals(ctor, dtypes.as_dtype) + self.assertEquals(args, (dtype.name,)) + reconstructed = ctor(*args) + self.assertEquals(reconstructed, dtype) + if __name__ == "__main__": googletest.main() diff --git a/tensorflow/python/framework/tensor_shape.py b/tensorflow/python/framework/tensor_shape.py index af2a5b1a7e..00f256cd45 100644 --- a/tensorflow/python/framework/tensor_shape.py +++ b/tensorflow/python/framework/tensor_shape.py @@ -456,6 +456,9 @@ class Dimension(object): else: return self._value >= other.value + def __reduce__(self): + return Dimension, (self._value,) + def as_dimension(value): """Converts the given value to a Dimension. @@ -928,6 +931,9 @@ class TensorShape(object): return True return self._dims != other.dims + def __reduce__(self): + return TensorShape, (self._dims,) + def as_shape(shape): """Converts the given object to a TensorShape.""" diff --git a/tensorflow/python/framework/tensor_shape_test.py b/tensorflow/python/framework/tensor_shape_test.py index 4e8ce4d889..498574eded 100644 --- a/tensorflow/python/framework/tensor_shape_test.py +++ b/tensorflow/python/framework/tensor_shape_test.py @@ -192,6 +192,14 @@ class DimensionTest(test_util.TensorFlowTestCase): self.assertEqual(nine % 4, 1) self.assertEqual(4 % nine, 4) + def testReduce(self): + dim = tensor_shape.Dimension(5) + ctor, args = dim.__reduce__() + self.assertEquals(ctor, tensor_shape.Dimension) + self.assertEquals(args, (5,)) + reconstructed = ctor(*args) + self.assertEquals(reconstructed, dim) + class ShapeTest(test_util.TensorFlowTestCase): @@ -417,5 +425,15 @@ class ShapeTest(test_util.TensorFlowTestCase): self.assertAllEqual([2, None, 4], tensor_shape.TensorShape( (2, None, 4)).as_list()) + def testReduce(self): + shape = tensor_shape.TensorShape([2, 3]) + ctor, args = shape.__reduce__() + self.assertEquals(ctor, tensor_shape.TensorShape) + self.assertEquals(args, ([tensor_shape.Dimension(2), + tensor_shape.Dimension(3)],)) + reconstructed = ctor(*args) + self.assertEquals(reconstructed, shape) + + if __name__ == "__main__": googletest.main() -- GitLab From cf542ae4174d954ad21ab255bc0fdb81326e4443 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 06:22:30 -0700 Subject: [PATCH 237/791] Special-case the name scoping for operator methods. TensorFlow disallows top-level name scopes to begin with underscores. Also use the transformer scope information to get to the enclosing function name. PiperOrigin-RevId: 192600256 --- .../autograph/converters/name_scopes.py | 38 ++++++++----- .../autograph/converters/name_scopes_test.py | 55 ++++++++++++++----- 2 files changed, 65 insertions(+), 28 deletions(-) diff --git a/tensorflow/contrib/autograph/converters/name_scopes.py b/tensorflow/contrib/autograph/converters/name_scopes.py index 2a3f474360..280bc4c314 100644 --- a/tensorflow/contrib/autograph/converters/name_scopes.py +++ b/tensorflow/contrib/autograph/converters/name_scopes.py @@ -28,22 +28,34 @@ from tensorflow.contrib.autograph.pyct import transformer class FunctionNameScopeTransformer(transformer.Base): """Wrap a function body with a `name_scope` of the function name.""" - def __init__(self, context): - super(FunctionNameScopeTransformer, self).__init__(context) - self._function_level = 0 + def _name_for_current_scope(self): + innermost = self.enclosing_entities[-1] + if len(self.enclosing_entities) > 1: + parent = self.enclosing_entities[-2] + if isinstance(parent, gast.ClassDef): + # Methods also take the name of their class. + name = '%s/%s' % (parent.name, innermost.name) + else: + name = innermost.name + else: + name = innermost.name + + # Sanitize the name. + # See https://www.tensorflow.org/api_docs/python/tf/Graph#name_scope + # TensorFlow doesn't like leading underscores at the top level. + while name[0] == '_': + name = name[1:] + return name def visit_FunctionDef(self, node): - self._function_level += 1 - try: - self.generic_visit(node) - finally: - self._function_level -= 1 - scope_name = node.name - if self._function_level == 0 and self.context.owner_type is not None: - scope_name = '{}/{}'.format(self.context.owner_type.__name__, scope_name) + self.generic_visit(node) + template = """ + with tf.name_scope(scope_name): + body + """ node.body = templates.replace( - 'with tf.name_scope(scope_name): body', - scope_name=gast.Str(scope_name), + template, + scope_name=gast.Str(self._name_for_current_scope()), body=node.body) return node diff --git a/tensorflow/contrib/autograph/converters/name_scopes_test.py b/tensorflow/contrib/autograph/converters/name_scopes_test.py index 61e5db2af8..2c2b6bbbec 100644 --- a/tensorflow/contrib/autograph/converters/name_scopes_test.py +++ b/tensorflow/contrib/autograph/converters/name_scopes_test.py @@ -38,29 +38,29 @@ class FunctionNameScopeTransformer(converter_test_base.TestCase): node = name_scopes.transform(node, self.ctx) with self.compiled(node, ops.name_scope) as result: - result_op = result.test_fn(constant_op.constant([1, 2, 3])) + result_op = result.test_fn(constant_op.constant(1)) self.assertIn('test_fn/', result_op.op.name) def test_nested_name(self): def test_fn(l): - def body(i): - return i**2 + def inner_fn(i): + return i ** 2 - l += [4] - return body(l) + l += 4 + return inner_fn(l) node = self.parse_and_analyze(test_fn, {}) node = name_scopes.transform(node, self.ctx) with self.compiled(node, ops.name_scope) as result: - result_op = result.test_fn(constant_op.constant([1, 2, 3])) + result_op = result.test_fn(constant_op.constant(1)) first_result_input_name = result_op.op.inputs[0].name second_result_input_name = result_op.op.inputs[1].name self.assertIn('test_fn/', first_result_input_name) - self.assertNotIn('body/', first_result_input_name) - self.assertIn('test_fn/body/', second_result_input_name) + self.assertNotIn('inner_fn', first_result_input_name) + self.assertIn('test_fn/inner_fn/', second_result_input_name) def test_class_name(self): @@ -68,11 +68,11 @@ class FunctionNameScopeTransformer(converter_test_base.TestCase): def test_fn(self, l): - def body(i): - return i**2 + def inner_fn(i): + return i ** 2 - l += [4] - return body(l) + l += 4 + return inner_fn(l) # Note that 'TestClass' was needed in the namespace here. node = self.parse_and_analyze( @@ -80,12 +80,37 @@ class FunctionNameScopeTransformer(converter_test_base.TestCase): node = name_scopes.transform(node, self.ctx) with self.compiled(node, ops.name_scope) as result: - result_op = result.TestClass().test_fn(constant_op.constant([1, 2, 3])) + result_op = result.TestClass().test_fn(constant_op.constant(1)) first_result_input_name = result_op.op.inputs[0].name second_result_input_name = result_op.op.inputs[1].name self.assertIn('TestClass/test_fn/', first_result_input_name) - self.assertNotIn('body/', first_result_input_name) - self.assertIn('TestClass/test_fn/body/', second_result_input_name) + self.assertNotIn('inner_fn', first_result_input_name) + self.assertIn('TestClass/test_fn/inner_fn/', second_result_input_name) + + def test_special_name(self): + + class TestClass(object): + + def __call__(self, l): + + def inner_fn(i): + return i ** 2 + + l += 4 + return inner_fn(l) + + # Note that 'TestClass' was needed in the namespace here. + node = self.parse_and_analyze( + TestClass.__call__, {'TestClass': TestClass}, owner_type=TestClass) + node = name_scopes.transform(node, self.ctx) + + with self.compiled(node, ops.name_scope) as result: + result_op = result.__call__(TestClass(), constant_op.constant(1)) + first_result_input_name = result_op.op.inputs[0].name + second_result_input_name = result_op.op.inputs[1].name + self.assertIn('call__/', first_result_input_name) + self.assertNotIn('inner_fn', first_result_input_name) + self.assertIn('call__/inner_fn/', second_result_input_name) if __name__ == '__main__': -- GitLab From b0978aa81d304a52516362432bc467462b4c7520 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 08:49:47 -0700 Subject: [PATCH 238/791] Updating tests containing graphs with Variables so that they Evaluate the original and optimized graphs and check if the outputs are same. PiperOrigin-RevId: 192616402 --- .../optimizers/constant_folding_test.cc | 127 ++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc index 31abe43846..36625b68b7 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc +++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc @@ -933,6 +933,17 @@ TEST_F(ConstantFoldingTest, ShapeMaterialization) { } } EXPECT_EQ(1, found); + auto v1_t = GenerateRandomTensor(TensorShape({3})); + auto v2_t = GenerateRandomTensor({5, 7}); + auto v3_t = GenerateRandomTensor({11, 13}); + + auto tensors_expected = EvaluateNodes( + item.graph, item.fetch, {{"v1", v1_t}, {"v2", v2_t}, {"v3", v3_t}}); + EXPECT_EQ(1, item.fetch.size()); + auto tensors = EvaluateNodes(output, item.fetch, + {{"v1", v1_t}, {"v2", v2_t}, {"v3", v3_t}}); + EXPECT_EQ(1, item.fetch.size()); + test::ExpectTensorEqual(tensors_expected[0], tensors[0]); } TEST_F(ConstantFoldingTest, ShapeMaterializationEmptyFetch) { @@ -1095,6 +1106,17 @@ TEST_F(ConstantFoldingTest, ShapeMaterializationShapeN_MultipleOutputs) { } } EXPECT_EQ(4, found); + + auto v1_t = GenerateRandomTensor(TensorShape({3, 4})); + auto v2_t = GenerateRandomTensor(TensorShape({4, 6})); + auto tensors_expected = + EvaluateNodes(item.graph, item.fetch, {{"v1", v1_t}, {"v2", v2_t}}); + EXPECT_EQ(2, tensors_expected.size()); + auto tensors = + EvaluateNodes(output, item.fetch, {{"v1", v1_t}, {"v2", v2_t}}); + EXPECT_EQ(2, tensors.size()); + for (int i = 0; i < tensors.size(); i++) + test::ExpectTensorEqual(tensors_expected[i], tensors[i]); } TEST_F(ConstantFoldingTest, SwitchNodesEmptyFetch) { @@ -1234,6 +1256,18 @@ TEST_F(ConstantFoldingTest, SwitchNodes) { } } EXPECT_EQ(2, found); + + auto v_in_t = GenerateRandomTensor(TensorShape({3})); + Tensor v_ctrl_t(DT_BOOL, TensorShape({})); + v_ctrl_t.flat()(0) = true; + auto tensors_expected = EvaluateNodes( + item.graph, item.fetch, {{"v_in", v_in_t}, {"v_ctrl", v_ctrl_t}}); + EXPECT_EQ(2, tensors_expected.size()); + auto tensors = EvaluateNodes(output, item.fetch, + {{"v_in", v_in_t}, {"v_ctrl", v_ctrl_t}}); + EXPECT_EQ(2, tensors.size()); + test::ExpectTensorEqual(tensors_expected[0], tensors[0]); + test::ExpectTensorNear(tensors_expected[1], tensors[1], 1e-5); } TEST_F(ConstantFoldingTest, MergeNodes) { @@ -1374,6 +1408,16 @@ TEST_F(ConstantFoldingTest, SplitRemoval) { AddNode("out", "Add", {"s1", "s2"}, {}, &want); CompareGraphs(want, got); + + auto in1_t = GenerateRandomTensor(TensorShape({2})); + auto in2_t = GenerateRandomTensor(TensorShape({4})); + auto tensors_expected = + EvaluateNodes(item.graph, item.fetch, {{"in1", in1_t}, {"in2", in2_t}}); + EXPECT_EQ(1, tensors_expected.size()); + auto tensors = + EvaluateNodes(got, item.fetch, {{"in1", in1_t}, {"in2", in2_t}}); + EXPECT_EQ(1, tensors.size()); + test::ExpectTensorNear(tensors_expected[0], tensors[0], 1e-5); } TEST_F(ConstantFoldingTest, SplitVRemoval) { @@ -1416,6 +1460,16 @@ TEST_F(ConstantFoldingTest, SplitVRemoval) { AddNode("out", "Add", {"s1", "s2"}, {}, &want); CompareGraphs(want, got); + + auto in1_t = GenerateRandomTensor(TensorShape({2})); + auto in2_t = GenerateRandomTensor(TensorShape({5})); + auto tensors_expected = + EvaluateNodes(item.graph, item.fetch, {{"in1", in1_t}, {"in2", in2_t}}); + EXPECT_EQ(1, tensors_expected.size()); + auto tensors = + EvaluateNodes(got, item.fetch, {{"in1", in1_t}, {"in2", in2_t}}); + EXPECT_EQ(1, tensors.size()); + test::ExpectTensorNear(tensors_expected[0], tensors[0], 1e-5); } TEST_F(ConstantFoldingTest, ShuffleReverseOnScalarRemoval) { @@ -1450,6 +1504,17 @@ TEST_F(ConstantFoldingTest, ShuffleReverseOnScalarRemoval) { AddNode("out2", "Identity", {"s2"}, {}, &want); CompareGraphs(want, got); + + auto in1_t = GenerateRandomTensor(TensorShape({})); + auto in2_t = GenerateRandomTensor(TensorShape({})); + auto tensors_expected = + EvaluateNodes(item.graph, item.fetch, {{"in1", in1_t}, {"in2", in2_t}}); + EXPECT_EQ(2, tensors_expected.size()); + auto tensors = + EvaluateNodes(got, item.fetch, {{"in1", in1_t}, {"in2", in2_t}}); + EXPECT_EQ(2, tensors.size()); + for (int i = 0; i < tensors.size(); i++) + test::ExpectTensorNear(tensors_expected[i], tensors[i], 1e-5); } TEST_F(ConstantFoldingTest, SliceWithSameDimensionRemoval) { @@ -1486,6 +1551,16 @@ TEST_F(ConstantFoldingTest, SliceWithSameDimensionRemoval) { AddNode("out", "Add", {"s1", "s2"}, {}, &want); CompareGraphs(want, got); + + auto in1_t = GenerateRandomTensor(TensorShape({3, 5})); + auto in2_t = GenerateRandomTensor(TensorShape({4, 6})); + auto tensors_expected = + EvaluateNodes(item.graph, item.fetch, {{"in1", in1_t}, {"in2", in2_t}}); + EXPECT_EQ(1, tensors_expected.size()); + auto tensors = + EvaluateNodes(got, item.fetch, {{"in1", in1_t}, {"in2", in2_t}}); + EXPECT_EQ(1, tensors.size()); + test::ExpectTensorNear(tensors_expected[0], tensors[0], 1e-5); } { // size = {-1, -1} tensorflow::Scope scope = tensorflow::Scope::NewRootScope(); @@ -1524,6 +1599,16 @@ TEST_F(ConstantFoldingTest, SliceWithSameDimensionRemoval) { AddNode("out", "Add", {"s1", "s2"}, {}, &want); CompareGraphs(want, got); + + auto in1_t = GenerateRandomTensor(TensorShape({3, 5})); + auto in2_t = GenerateRandomTensor(TensorShape({4, 6})); + auto tensors_expected = + EvaluateNodes(item.graph, item.fetch, {{"in1", in1_t}, {"in2", in2_t}}); + EXPECT_EQ(1, tensors_expected.size()); + auto tensors = + EvaluateNodes(got, item.fetch, {{"in1", in1_t}, {"in2", in2_t}}); + EXPECT_EQ(1, tensors.size()); + test::ExpectTensorNear(tensors_expected[0], tensors[0], 1e-5); } } @@ -1602,6 +1687,16 @@ TEST_F(ConstantFoldingTest, PaddingWithZeroSize) { AddNode("out", "Add", {"p1", "p2"}, {}, &want); CompareGraphs(want, got); + + auto in1_t = GenerateRandomTensor(TensorShape({4, 6})); + auto in2_t = GenerateRandomTensor(TensorShape({2, 2})); + auto tensors_expected = + EvaluateNodes(item.graph, item.fetch, {{"in1", in1_t}, {"in2", in2_t}}); + EXPECT_EQ(1, tensors_expected.size()); + auto tensors = + EvaluateNodes(got, item.fetch, {{"in1", in1_t}, {"in2", in2_t}}); + EXPECT_EQ(1, tensors.size()); + test::ExpectTensorEqual(tensors_expected[0], tensors[0]); } TEST_F(ConstantFoldingTest, SqueezeWithAllDimesionsGreaterThanOne) { @@ -1632,6 +1727,16 @@ TEST_F(ConstantFoldingTest, SqueezeWithAllDimesionsGreaterThanOne) { AddNode("out", "Add", {"s1", "s2"}, {}, &want); CompareGraphs(want, got); + + auto in1_t = GenerateRandomTensor(TensorShape({2, 3})); + auto in2_t = GenerateRandomTensor(TensorShape({1, 2, 3, 1})); + auto tensors_expected = + EvaluateNodes(item.graph, item.fetch, {{"in1", in1_t}, {"in2", in2_t}}); + EXPECT_EQ(1, tensors_expected.size()); + auto tensors = + EvaluateNodes(got, item.fetch, {{"in1", in1_t}, {"in2", in2_t}}); + EXPECT_EQ(1, tensors.size()); + test::ExpectTensorEqual(tensors_expected[0], tensors[0]); } TEST_F(ConstantFoldingTest, NoOpReduction) { @@ -1666,6 +1771,13 @@ TEST_F(ConstantFoldingTest, NoOpReduction) { } } EXPECT_TRUE(found); + + auto v_t = GenerateRandomTensor(TensorShape({3, 5, 7})); + auto tensors_expected = EvaluateNodes(item.graph, item.fetch, {{"v", v_t}}); + EXPECT_EQ(1, tensors_expected.size()); + auto tensors = EvaluateNodes(output, item.fetch, {{"v", v_t}}); + EXPECT_EQ(1, tensors.size()); + test::ExpectTensorNear(tensors_expected[0], tensors[0], 1e-5); } TEST_F(ConstantFoldingTest, NoOpReshape) { @@ -1744,6 +1856,21 @@ TEST_F(ConstantFoldingTest, NoOpReshape) { } } EXPECT_EQ(4, found); + + auto v1_t = GenerateRandomTensor(TensorShape({17})); + auto v2_t = GenerateRandomTensor(TensorShape({17, 1})); + auto v3_t = GenerateRandomTensor(TensorShape({5, 5, 5})); + auto v4_t = GenerateRandomTensor(TensorShape({5, 5, 5})); + auto tensors_expected = + EvaluateNodes(item.graph, item.fetch, + {{"v1", v1_t}, {"v2", v2_t}, {"v3", v3_t}, {"v4", v4_t}}); + EXPECT_EQ(4, tensors_expected.size()); + auto tensors = + EvaluateNodes(output, item.fetch, + {{"v1", v1_t}, {"v2", v2_t}, {"v3", v3_t}, {"v4", v4_t}}); + EXPECT_EQ(4, tensors.size()); + for (int i = 0; i < tensors.size(); i++) + test::ExpectTensorNear(tensors_expected[i], tensors[i], 1e-5); } TEST_F(ConstantFoldingTest, Packing) { -- GitLab From cbea75338433bd36b22742abed13e36bb8cbdc84 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 09:44:16 -0700 Subject: [PATCH 239/791] Fixing dependencies. PiperOrigin-RevId: 192624191 --- tensorflow/python/tools/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/python/tools/BUILD b/tensorflow/python/tools/BUILD index cc2884a4f6..84d20f8e36 100644 --- a/tensorflow/python/tools/BUILD +++ b/tensorflow/python/tools/BUILD @@ -38,6 +38,7 @@ py_library( deps = [ ":saved_model_utils", "//tensorflow/core:protos_all_py", + "//tensorflow/python", # TODO(b/34059704): remove when fixed "//tensorflow/python:client", "//tensorflow/python:framework", "//tensorflow/python:parsing_ops", -- GitLab From 8a247976484173059aedc17bfd8d770b8d1a70e1 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 09:46:34 -0700 Subject: [PATCH 240/791] Collective Ops Part 3 BaseCollectiveExecutor and RingReducer. This change is part of a series of changes introducing infrastructure for collective ops and initial implementations of reduction and broadcast. PiperOrigin-RevId: 192624521 --- tensorflow/core/BUILD | 33 + .../base_collective_executor.cc | 257 ++++++++ .../common_runtime/base_collective_executor.h | 144 +++++ .../common_runtime/collective_executor_mgr.cc | 38 +- tensorflow/core/common_runtime/dma_helper.h | 3 + .../core/common_runtime/ring_reducer.cc | 542 ++++++++++++++++ tensorflow/core/common_runtime/ring_reducer.h | 146 +++++ .../core/common_runtime/ring_reducer_test.cc | 606 ++++++++++++++++++ .../test_collective_executor_mgr.h | 116 ++++ 9 files changed, 1851 insertions(+), 34 deletions(-) create mode 100644 tensorflow/core/common_runtime/base_collective_executor.cc create mode 100644 tensorflow/core/common_runtime/base_collective_executor.h create mode 100644 tensorflow/core/common_runtime/ring_reducer.cc create mode 100644 tensorflow/core/common_runtime/ring_reducer.h create mode 100644 tensorflow/core/common_runtime/ring_reducer_test.cc create mode 100644 tensorflow/core/common_runtime/test_collective_executor_mgr.h diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 55b0040b52..118955219b 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -1064,6 +1064,7 @@ cc_library( hdrs = [ "common_runtime/function_testlib.h", "common_runtime/kernel_benchmark_testlib.h", + "common_runtime/test_collective_executor_mgr.h", "framework/fake_input.h", "framework/function_testlib.h", "framework/shape_inference_testutil.h", @@ -2261,6 +2262,7 @@ tf_cuda_library( CORE_CPU_LIB_HEADERS = CORE_CPU_BASE_HDRS + [ "common_runtime/allocator_retry.h", + "common_runtime/base_collective_executor.h", "common_runtime/bfc_allocator.h", "common_runtime/buf_rendezvous.h", "common_runtime/build_graph_options.h", @@ -2289,6 +2291,7 @@ CORE_CPU_LIB_HEADERS = CORE_CPU_BASE_HDRS + [ "common_runtime/renamed_device.h", "common_runtime/rendezvous_mgr.h", "common_runtime/rendezvous_util.h", + "common_runtime/ring_reducer.h", "common_runtime/scoped_allocator.h", "common_runtime/scoped_allocator_mgr.h", "common_runtime/session_factory.h", @@ -2306,6 +2309,7 @@ tf_cuda_library( srcs = [ "common_runtime/accumulate_n_optimizer.cc", "common_runtime/allocator_retry.cc", + "common_runtime/base_collective_executor.cc", "common_runtime/bfc_allocator.cc", "common_runtime/buf_rendezvous.cc", "common_runtime/build_graph_options.cc", @@ -2336,6 +2340,7 @@ tf_cuda_library( "common_runtime/renamed_device.cc", "common_runtime/rendezvous_mgr.cc", "common_runtime/rendezvous_util.cc", + "common_runtime/ring_reducer.cc", "common_runtime/scoped_allocator.cc", "common_runtime/scoped_allocator_mgr.cc", "common_runtime/session.cc", @@ -3101,6 +3106,34 @@ tf_cc_test( ], ) +tf_cc_tests_gpu( + name = "ring_reducer_test", + size = "medium", + srcs = [ + "common_runtime/ring_reducer_test.cc", + ], + linkstatic = tf_kernel_tests_linkstatic(), + tags = tf_cuda_tests_tags(), + deps = [ + ":all_kernels", + ":core", + ":core_cpu", + ":core_cpu_internal", + ":direct_session_internal", + ":framework", + ":framework_internal", + ":gpu_runtime", + ":lib", + ":lib_internal", + ":ops", + ":protos_all_cc", + ":protos_test_cc", + ":test", + ":test_main", + ":testlib", + ], +) + tf_cc_test_mkl( name = "mkl_runtime_tests", size = "small", diff --git a/tensorflow/core/common_runtime/base_collective_executor.cc b/tensorflow/core/common_runtime/base_collective_executor.cc new file mode 100644 index 0000000000..f6332fabdb --- /dev/null +++ b/tensorflow/core/common_runtime/base_collective_executor.cc @@ -0,0 +1,257 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/core/common_runtime/base_collective_executor.h" + +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" +#include "tensorflow/core/common_runtime/copy_tensor.h" +#include "tensorflow/core/common_runtime/device_mgr.h" +#include "tensorflow/core/common_runtime/dma_helper.h" +#include "tensorflow/core/common_runtime/process_util.h" +#include "tensorflow/core/common_runtime/ring_reducer.h" +#include "tensorflow/core/lib/core/notification.h" +#include "tensorflow/core/lib/strings/str_util.h" + +#define VALUE_IN_DEBUG_STRING false + +namespace tensorflow { +/*static*/ +int64 CollectiveAdapter::AlignedChunkElts(int64 elt_bytes, int64 total_elts, + int64 num_chunks) { + DCHECK_GT(num_chunks, 0); + int64 base_chunk_elts = (total_elts + (num_chunks - 1)) / num_chunks; + if (EIGEN_MAX_ALIGN_BYTES == 0) return base_chunk_elts; + if (EIGEN_MAX_ALIGN_BYTES <= elt_bytes) { + // Tolerate weird small values of EIGEN_MAX_ALIGN_BYTES + DCHECK_EQ(0, elt_bytes % EIGEN_MAX_ALIGN_BYTES); + return base_chunk_elts; + } + // elt_bytes < EIGEN_MAX_ALIGN_BYTES, which + // must be a common multiple of the various atomic data types. + DCHECK_EQ(0, EIGEN_MAX_ALIGN_BYTES % elt_bytes) + << "total_elts=" << total_elts << " num_chunks=" << num_chunks + << " EIGEN_MAX_ALIGN_BYTES=" << EIGEN_MAX_ALIGN_BYTES + << " elt_bytes=" << elt_bytes; + // Round bytes per chunk up to the next multiple of EIGEN_MAX_ALIGN_BYTES. + int64 chunk_bytes = base_chunk_elts * elt_bytes; + int64 diff = + (chunk_bytes < EIGEN_MAX_ALIGN_BYTES) + ? (EIGEN_MAX_ALIGN_BYTES - chunk_bytes) + : (EIGEN_MAX_ALIGN_BYTES - (chunk_bytes % EIGEN_MAX_ALIGN_BYTES)); + CHECK_EQ(0, diff % elt_bytes); + base_chunk_elts += (diff / elt_bytes); + DCHECK_EQ(0, ((base_chunk_elts * elt_bytes) % EIGEN_MAX_ALIGN_BYTES)) + << "total_elts=" << total_elts << " num_chunks=" << num_chunks + << " EIGEN_MAX_ALIGN_BYTES=" << EIGEN_MAX_ALIGN_BYTES + << " base_chunk_elts=" << base_chunk_elts << " elt_bytes=" << elt_bytes; + return base_chunk_elts; +} + +namespace { +template +class CollectiveAdapterImpl : public CollectiveAdapter { + public: + // Takes ownership of output and prepares to properly alias its chunks. + // Ownership is taken because the shape may temporarily change. + CollectiveAdapterImpl(Tensor* output, int64 num_chunks, Allocator* allocator) + : output_(std::move(*output)), + dt_(output_.dtype()), + old_shape_(output_.shape()), + num_chunks_(num_chunks), + allocator_(allocator), + total_elts_(output_.NumElements()), + chunk_elts_(AlignedChunkElts(sizeof(T), total_elts_, num_chunks_)), + data_start_(reinterpret_cast(DMAHelper::base(&output_))), + data_end_(data_start_ + total_elts_) { + CHECK_GT(chunk_elts_, 0); + Flatten(); + } + + ~CollectiveAdapterImpl() override {} + + const Tensor& Value() const override { return output_; } + + // If necessary, flatten output. + void Flatten() { + if (old_shape_.dims() > 1) { + TensorShape new_shape = TensorShape({old_shape_.num_elements()}); + DMAHelper::UnsafeSetShape(&output_, new_shape); + } + } + + void ConsumeFinalValue(Tensor* output) override { + if (old_shape_ != output_.shape()) { + DMAHelper::UnsafeSetShape(&output_, old_shape_); + } + *output = std::move(output_); + } + + // Number of T elements in a particular chunk. + inline int64 ChunkElts(int i) const { + DCHECK_LT(i, num_chunks_); + const T* chunk_start = std::min(data_end_, data_start_ + i * chunk_elts_); + const T* chunk_end = std::min(data_end_, chunk_start + chunk_elts_); + return chunk_end - chunk_start; + } + + int64 ChunkBytes(int i) const override { return sizeof(T) * ChunkElts(i); } + + // Returns a new Tensor that aliases the required chunk. + Tensor ChunkAlias(int i) override { + int64 start = chunk_elts_ * i; + int64 num_elts = ChunkElts(i); + // If this chunk is empty the prior chunk might also be short + // so always take an empty slice from the front of the tensor + // to avoid an illegal offset check failure somewhere. + return (num_elts > 0) ? output_.Slice(start, start + num_elts) + : output_.Slice(0, 0); + } + + Tensor TempChunk(int i) const override { + AllocationAttributes empty; + return Tensor(allocator_, dt_, {ChunkElts(i)}, empty); + } + + string DebugString() const override { + return strings::StrCat( + "base addr ", reinterpret_cast(DMAHelper::base(&output_)), + " num_chunks ", num_chunks_, " total_elts ", total_elts_, " chunk_elts", + chunk_elts_, " value ", + VALUE_IN_DEBUG_STRING ? output_.SummarizeValue(1024) : ""); + } + + string TBounds(const Tensor& t) const override { + int64 base_addr = reinterpret_cast(DMAHelper::base(&t)); + return strings::StrCat("(", base_addr, ", ", (base_addr + t.TotalBytes()), + ")"); + } + + Tensor Scalar(int v) const override { + Tensor t(dt_, TensorShape({})); + t.scalar()() = v; + return t; + } + + Tensor Scalar(Allocator* a) const override { + Tensor t(a, dt_, TensorShape({})); + return t; + } + + Tensor output_; + const DataType dt_; + const TensorShape old_shape_; + const int64 num_chunks_; + Allocator* allocator_; + const int64 total_elts_; + const int64 chunk_elts_; + const T* data_start_; + const T* data_end_; +}; + +} // namespace + +CollectiveAdapter* MakeCollectiveAdapter(Tensor* output, int num_chunks, + Allocator* allocator) { + switch (output->dtype()) { + case DT_FLOAT: + return new CollectiveAdapterImpl(output, num_chunks, allocator); + break; + case DT_DOUBLE: + return new CollectiveAdapterImpl(output, num_chunks, allocator); + break; + case DT_INT32: + return new CollectiveAdapterImpl(output, num_chunks, allocator); + break; + case DT_INT64: + return new CollectiveAdapterImpl(output, num_chunks, allocator); + break; + default: + LOG(FATAL) << "Unsupported type " << output->dtype() + << " to MakeCollectiveAdapter"; + return nullptr; + } +} + +BaseCollectiveExecutor::~BaseCollectiveExecutor() {} + +void BaseCollectiveExecutor::StartAbort(const Status& s) { + LOG(WARNING) << "BaseCollectiveExecutor::StartAbort " << s; + remote_access_->StartAbort(s); +} + +void BaseCollectiveExecutor::ExecuteAsync(OpKernelContext* ctx, + const CollectiveParams& col_params, + const string& exec_key, + StatusCallback done) { + const Tensor* input = &ctx->input(0); + Tensor* output = ctx->mutable_output(0); + string error; + switch (col_params.instance.type) { + case REDUCTION_COLLECTIVE: { + // TODO(tucker): support other reduction algorithms, + // e.g. tree-reduce, hybrid tree/ring, delegate-to-NCCL, etc. + RingReducer* reducer = + CreateReducer(ctx, CtxParams(ctx), col_params, exec_key, step_id_, + input, output, &error); + if (!reducer) { + done(errors::Internal(error)); + return; + } + // Run in an I/O thread, so as not to starve the executor threads. + // TODO(tucker): Instead of forking every per-device Collective + // Op off into its own thread, consider queuing them on a + // fixed-size thread-pool dedicated to running CollectiveOps. + SchedClosure([reducer, done]() { + reducer->Run([reducer, done](const Status& s) { + done(s); + delete reducer; + }); + }); + } break; + case BROADCAST_COLLECTIVE: + done(errors::Internal("Collective Broadcast unimplemented")); + break; + default: + done(errors::Internal("Unimplemented CollectiveType ", + col_params.instance.type)); + } +} + +RingReducer* BaseCollectiveExecutor::CreateReducer( + OpKernelContext* ctx, OpKernelContext::Params* params, + const CollectiveParams& col_params, const string& exec_key, int64 step_id, + const Tensor* input, Tensor* output, string* error) { + switch (col_params.instance.data_type) { + case DT_INT32: + if (col_params.group.device_type == DEVICE_GPU) { + *error = + "Collective Reduce does not support datatype DT_INT32 on " + "DEVICE_GPU"; + return nullptr; + } + TF_FALLTHROUGH_INTENDED; + case DT_FLOAT: + case DT_DOUBLE: + case DT_INT64: + return new RingReducer(this, dev_mgr_, ctx, params, col_params, exec_key, + step_id, input, output); + break; + default: + *error = strings::StrCat("Collective Reduce does not support datatype ", + col_params.instance.data_type); + return nullptr; + } +} + +} // namespace tensorflow diff --git a/tensorflow/core/common_runtime/base_collective_executor.h b/tensorflow/core/common_runtime/base_collective_executor.h new file mode 100644 index 0000000000..58eaf31f71 --- /dev/null +++ b/tensorflow/core/common_runtime/base_collective_executor.h @@ -0,0 +1,144 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_BASE_COLLECTIVE_EXECUTOR_H_ +#define TENSORFLOW_CORE_COMMON_RUNTIME_BASE_COLLECTIVE_EXECUTOR_H_ + +#include +#include "tensorflow/core/common_runtime/buf_rendezvous.h" +#include "tensorflow/core/framework/collective.h" +#include "tensorflow/core/framework/device_attributes.pb.h" + +namespace tensorflow { +class DeviceMgr; +class RingReducer; + +// Helper interface that aliases regular subfields of a Tensor as separate +// Tensors for in-place update. +class CollectiveAdapter { + public: + virtual ~CollectiveAdapter() {} + + // Move the backing tensor to 'output' with its original storage and + // shape. After this call this CollectiveAdapter object should be + // deleted immediately without calling any of its other methods. + virtual void ConsumeFinalValue(Tensor* output) = 0; + + // const access to entire intermediate value for debugging + virtual const Tensor& Value() const = 0; + + // Returns tensor for chunk i which aliases the backing buffer. + virtual Tensor ChunkAlias(int i) = 0; + + // Returns tensor allocated on the same device but with its own + // separate backing buffer. Will have same type and size as + // chunk i. + virtual Tensor TempChunk(int i) const = 0; + + // Bytes in chunk i + virtual int64 ChunkBytes(int i) const = 0; + + // Generate a CPU RAM scalar tensor of the same DataType as the + // backing tensor with the given integer value. + virtual Tensor Scalar(int v) const = 0; + + // Generate a scalar tensor of same DataType and on the same device + // as the backing tensor. + virtual Tensor Scalar(Allocator* a) const = 0; + + // Debugging string describing buffer location + virtual string TBounds(const Tensor& t) const = 0; + + virtual string DebugString() const = 0; + + // Computes the number of elements per alias chunk tensor. + // + // A CHECK in tensor.cc expects that the memory buffer backing a + // Tensor will be aligned according to EIGEN_MAX_ALIGN_BYTES. To + // ensure that all chunk aliasing Tensors maintain this alignment we + // need to pick a chunk size that preserves it. Note than in extreme + // cases (impractical, but possible with very small tensors) one or + // more tail chunks can end up emptby. + static int64 AlignedChunkElts(int64 elt_bytes, int64 total_elts, + int64 num_chunks); +}; + +// Create a CollectiveAdaptor wrapping 'output', specialized to its +// data-type and shape. +CollectiveAdapter* MakeCollectiveAdapter(Tensor* output, int num_chunks, + Allocator* allocator); + +// Default implementation of CollectiveExecutor. Delegates the actual +// work of moving data to a class specialized for the operation type, +// arguments and device+interconnect topology. +class BaseCollectiveExecutor : public CollectiveExecutor { + public: + BaseCollectiveExecutor(CollectiveExecutorMgrInterface* cem, + PerStepCollectiveRemoteAccess* remote_access, + int64 step_id, const DeviceMgr* dev_mgr) + : CollectiveExecutor(cem), + step_id_(step_id), + dev_mgr_(dev_mgr), + remote_access_(remote_access) {} + + ~BaseCollectiveExecutor() override; + + void StartAbort(const Status& s) override; + + void ExecuteAsync(OpKernelContext* ctx, const CollectiveParams& col_params, + const string& exec_key, StatusCallback done) override; + + PerStepCollectiveRemoteAccess* remote_access() override { + return remote_access_.get(); + } + + void RecvFromPeer(const string& peer_device, const string& peer_task, + bool peer_is_local, const string& key, Device* to_device, + DeviceContext* to_device_ctx, + const AllocatorAttributes& to_alloc_attr, Tensor* to_tensor, + const DeviceLocality& client_locality, + const StatusCallback& done) override { + remote_access_->RecvFromPeer(peer_device, peer_task, peer_is_local, key, + to_device, to_device_ctx, to_alloc_attr, + to_tensor, client_locality, done); + } + + void PostToPeer(const string& peer_device, const string& peer_task, + const string& key, Device* from_device, + DeviceContext* from_device_ctx, + const AllocatorAttributes& from_alloc_attr, + const Tensor* from_tensor, + const DeviceLocality& client_locality, + const StatusCallback& done) override { + remote_access_->PostToPeer(peer_device, peer_task, key, from_device, + from_device_ctx, from_alloc_attr, from_tensor, + client_locality, done); + } + + protected: + const int64 step_id_; + const DeviceMgr* dev_mgr_; // Not owned. + std::unique_ptr remote_access_; + + private: + RingReducer* CreateReducer(OpKernelContext* ctx, + OpKernelContext::Params* params, + const CollectiveParams& col_params, + const string& exec_key, int64 step_id, + const Tensor* input, Tensor* output, + string* error); +}; + +} // namespace tensorflow +#endif // TENSORFLOW_CORE_COMMON_RUNTIME_BASE_COLLECTIVE_EXECUTOR_H_ diff --git a/tensorflow/core/common_runtime/collective_executor_mgr.cc b/tensorflow/core/common_runtime/collective_executor_mgr.cc index a5c4946e58..e07829b286 100644 --- a/tensorflow/core/common_runtime/collective_executor_mgr.cc +++ b/tensorflow/core/common_runtime/collective_executor_mgr.cc @@ -14,6 +14,7 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/common_runtime/collective_executor_mgr.h" +#include "tensorflow/core/common_runtime/base_collective_executor.h" #include "tensorflow/core/common_runtime/build_graph_options.h" #include "tensorflow/core/common_runtime/collective_rma_local.h" #include "tensorflow/core/common_runtime/device_mgr.h" @@ -21,39 +22,6 @@ limitations under the License. #include "tensorflow/core/protobuf/config.pb.h" namespace tensorflow { -namespace { -// TODO(tucker): Temporary class just until a real CollectiveExecutor -// implementation is submitted in a later CL. -class DummyCollectiveExecutor : public CollectiveExecutor { - public: - explicit DummyCollectiveExecutor(CollectiveExecutorMgr* ce_mgr) - : CollectiveExecutor(ce_mgr) {} - - ~DummyCollectiveExecutor() override {} - - void RecvFromPeer(const string& peer_device, const string& peer_task, - bool peer_is_local, const string& key, Device* to_device, - DeviceContext* to_device_ctx, - const AllocatorAttributes& to_alloc_attr, Tensor* to_tensor, - const DeviceLocality& client_locality, - const StatusCallback& done) override { - done(errors::Internal("Unimplemented")); - } - - void PostToPeer(const string& peer_device, const string& peer_task, - const string& key, Device* from_device, - DeviceContext* from_device_ctx, - const AllocatorAttributes& from_alloc_attr, - const Tensor* from_tensor, - const DeviceLocality& client_locality, - const StatusCallback& done) override { - done(errors::Internal("Unimplemented")); - } - - private: - TF_DISALLOW_COPY_AND_ASSIGN(DummyCollectiveExecutor); -}; -} // namespace CollectiveExecutorMgr::CollectiveExecutorMgr( const ConfigProto& config, const DeviceMgr* dev_mgr, @@ -77,7 +45,9 @@ CollectiveExecutor* CollectiveExecutorMgr::FindOrCreate(int64 step_id) { if (it != executor_table_.end()) { ce = it->second; } else { - ce = new DummyCollectiveExecutor(this); + CollectiveRemoteAccessLocal* rma = new CollectiveRemoteAccessLocal( + dev_mgr_, dev_resolver_.get(), step_id); + ce = new BaseCollectiveExecutor(this, rma, step_id, dev_mgr_); executor_table_[step_id] = ce; } ce->Ref(); diff --git a/tensorflow/core/common_runtime/dma_helper.h b/tensorflow/core/common_runtime/dma_helper.h index 1cc8b9e723..cdfce1f366 100644 --- a/tensorflow/core/common_runtime/dma_helper.h +++ b/tensorflow/core/common_runtime/dma_helper.h @@ -28,6 +28,9 @@ class DMAHelper { static void* base(Tensor* t) { return t->base(); } static TensorBuffer* buffer(Tensor* t) { return t->buf_; } static const TensorBuffer* buffer(const Tensor* t) { return t->buf_; } + static void UnsafeSetShape(Tensor* t, const TensorShape& s) { + t->set_shape(s); + } }; } // namespace tensorflow diff --git a/tensorflow/core/common_runtime/ring_reducer.cc b/tensorflow/core/common_runtime/ring_reducer.cc new file mode 100644 index 0000000000..79d03a24ce --- /dev/null +++ b/tensorflow/core/common_runtime/ring_reducer.cc @@ -0,0 +1,542 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/core/common_runtime/ring_reducer.h" + +#include "tensorflow/core/common_runtime/collective_rma_local.h" +#include "tensorflow/core/common_runtime/copy_tensor.h" +#include "tensorflow/core/common_runtime/device_mgr.h" +#include "tensorflow/core/common_runtime/dma_helper.h" +#include "tensorflow/core/lib/core/notification.h" +#include "tensorflow/core/lib/strings/str_util.h" +#include "tensorflow/core/platform/env.h" + +// Set true for greater intelligibility of debug mode log messages. +#define READABLE_KEYS false + +namespace tensorflow { +namespace { +// Each CollectiveOp implementation is free to define its own +// BufRendezvous key format. This function produces the key used by +// RingReducer. +string RingReduceBufKey(const string& exec_key, int pass, int section, + int source_rank) { + if (READABLE_KEYS) { + return strings::StrCat("rred(", exec_key, "):pass(", pass, "):section(", + section, "):srcrank(", source_rank, ")"); + } else { + // TODO(tucker): Try out some kind of denser encoding, e.g. 128 bit hash. + return strings::StrCat(exec_key, ":", pass, ":", section, ":", source_rank); + } +} + +} // namespace + +void RingReducer::PCQueue::Enqueue(RingField* rf) { + mutex_lock l(pcq_mu_); + deque_.push_back(rf); + if (waiter_count_ > 0) { + cv_.notify_one(); + } +} + +RingReducer::RingField* RingReducer::PCQueue::Dequeue() { + mutex_lock l(pcq_mu_); + if (deque_.empty()) { + ++waiter_count_; + while (deque_.empty()) { + cv_.wait(l); + } + --waiter_count_; + } + RingField* rf = deque_.front(); + deque_.pop_front(); + return rf; +} + +RingReducer::RingReducer(CollectiveExecutor* col_exec, const DeviceMgr* dev_mgr, + OpKernelContext* ctx, + OpKernelContext::Params* op_params, + const CollectiveParams& col_params, + const string& exec_key, int64 step_id, + const Tensor* input, Tensor* output) + : col_exec_(col_exec), + dev_mgr_(dev_mgr), + ctx_(ctx), + op_params_(op_params), + col_params_(col_params), + exec_key_(exec_key), + input_(input), + output_(output), + rank_(col_params.subdiv_rank[0]), + step_id_(step_id), + group_size_(col_params.group.group_size), + num_subdivs_(static_cast( + col_params.instance.impl_details.subdiv_permutations.size())), + done_(nullptr), + device_(nullptr), + device_name_( + col_params_.instance.device_names[col_params_.default_rank]) { + CHECK_GT(group_size_, 0); + CHECK_GT(num_subdivs_, 0); +} + +string RingReducer::TensorDebugString(Tensor tensor) { + const DeviceBase::GpuDeviceInfo* gpu_device_info = + ctx_->device()->tensorflow_gpu_device_info(); + if (gpu_device_info) { + Tensor cpu_tensor(tensor.dtype(), tensor.shape()); + Notification note; + gpu_device_info->default_context->CopyDeviceTensorToCPU( + &tensor, "" /*tensor_name*/, device_, &cpu_tensor, + [¬e](const Status& s) { + CHECK(s.ok()); + note.Notify(); + }); + note.WaitForNotification(); + return cpu_tensor.SummarizeValue(64); + } else { + return tensor.SummarizeValue(64); + } +} + +void RingReducer::Run(StatusCallback done) { + done_ = std::move(done); + + // Get local execution device. + if (VLOG_IS_ON(1)) { + string buf; + for (int r = 0; r < col_params_.instance.device_names.size(); ++r) { + strings::StrAppend(&buf, "dev ", r, " : ", + col_params_.instance.device_names[r], "\n"); + } + for (int sd = 0; + sd < col_params_.instance.impl_details.subdiv_permutations.size(); + ++sd) { + strings::StrAppend(&buf, "\nsubdiv ", sd, " perm: "); + for (auto x : col_params_.instance.impl_details.subdiv_permutations[sd]) { + strings::StrAppend(&buf, x, ", "); + } + } + VLOG(1) << "RingReducer::Run for device " << device_name_ + << " default_rank " << col_params_.default_rank << "\n" + << buf; + } + CHECK(dev_mgr_); + Status status = dev_mgr_->LookupDevice( + col_params_.instance.device_names[col_params_.default_rank], &device_); + if (!status.ok()) { + LOG(ERROR) << "Failed to find device " + << col_params_.instance.device_names[col_params_.default_rank]; + for (auto d : dev_mgr_->ListDevices()) { + LOG(ERROR) << "Available device " << d->name(); + } + done_(status); + return; + } + CHECK(device_); + device_locality_ = device_->attributes().locality(); + + VLOG(1) << this << " default_rank " << col_params_.default_rank << " cp " + << &col_params_ << ": " << col_params_.ToString(); + + // Start by copying input to output if they're not already the same, i.e. if + // we're not computing in-place on the input tensor. + if ((input_ != output_) && + (DMAHelper::base(input_) != DMAHelper::base(output_))) { + CollectiveRemoteAccessLocal::MemCpyAsync( + ctx_->input_device_context(0), ctx_->op_device_context(), device_, + device_, ctx_->input_alloc_attr(0), ctx_->output_alloc_attr(0), input_, + output_, [this](const Status& s) { + if (!s.ok()) { + done_(s); + } else { + ContinueAfterInputCopy(); + } + }); + } else { + ContinueAfterInputCopy(); + } +} + +void RingReducer::ContinueAfterInputCopy() { + AllocatorAttributes attr = ctx_->output_alloc_attr(0); + ca_.reset(MakeCollectiveAdapter(output_, group_size_ * num_subdivs_, + device_->GetAllocator(attr))); + + if (col_params_.final_op) { + // Create an on-device scalar value from group_size_ that may be needed + // later. + // TODO(tucker): Cache and reuse across invocations? Or maybe the scalar + // can be provided to the kernel in host memory? + Tensor group_size_val = ca_->Scalar(group_size_); + if (col_params_.group.device_type != "CPU") { + group_size_tensor_ = + ca_->Scalar(device_->GetAllocator(ctx_->input_alloc_attr(0))); + DeviceContext* op_dev_ctx = ctx_->op_device_context(); + op_dev_ctx->CopyCPUTensorToDevice(&group_size_val, device_, + &group_size_tensor_, + [this](const Status& s) { + if (!s.ok()) { + StartAbort(s); + } + group_size_tensor_ready_.Notify(); + }); + } else { + group_size_tensor_ = group_size_val; + group_size_tensor_ready_.Notify(); + } + } + Finish(RunAsyncParts()); +} + +void RingReducer::StartAbort(const Status& s) { + // In abort mode we stop issuing additional ProvideBuf + // and ConsumeBuf calls, but we need to wait for all of the + // outstanding callbacks to be invoked before quitting. + bool abort_started = false; + { + mutex_lock l(status_mu_); + if (status_.ok()) { + LOG(ERROR) << "Aborting RingReduce with " << s; + abort_started = true; + status_.Update(s); + } + } + // If this is the initial entry to abort mode then invoke StartAbort + // on the CollectiveExecutor that invoked us. That should start + // cancellation on all of the outstanding CollectiveRemoteAccess + // actions. + if (abort_started) { + col_exec_->StartAbort(s); + } +} + +void RingReducer::Finish(bool ok) { + if (ok) { + // Recover the output from the adaptor. + ca_->ConsumeFinalValue(output_); + } + Status s; + { + mutex_lock l(status_mu_); + s = status_; + } + done_(s); +} + +RingReducer::SubContext::SubContext(OpKernelContext* ctx, + OpKernelContext::Params* params, + OpKernel* op, Tensor* output, Tensor* input) + : sub_params_(*params), + sub_inputs_({output, input}), + sub_input_attr_({ctx->input_alloc_attr(0), ctx->input_alloc_attr(0)}), + sub_input_dc_( + {ctx->input_device_context(0), ctx->input_device_context(0)}) { + sub_params_.op_kernel = op; + sub_params_.inputs = &sub_inputs_; + sub_params_.input_alloc_attrs = &sub_input_attr_; + sub_params_.input_device_contexts = &sub_input_dc_; + sub_params_.eigen_gpu_device = nullptr; + sub_params_.ensure_eigen_gpu_device(); + sub_ctx_ = new OpKernelContext(&sub_params_, 1); +} + +Status RingReducer::ComputeBinOp(Device* device, OpKernel* op, Tensor* output, + Tensor* input) { + // Prepare an OpKernelContext that is identical to that of the original Op + // (i.e. the collective), except for the input output sizes and identities and + // the Op itself. + // TODO(tucker): Is it possible to cache and reuse these objects? They're + // mostly identical inside one device execution. + std::unique_ptr sub_ctx( + new SubContext(ctx_, op_params_, op, output, input)); + device->Compute(op, sub_ctx->sub_ctx_); + return sub_ctx->sub_ctx_->status(); +} + +// At the beginning of the algorithm initialize a RingField struct for +// every independent field of the tensor. +void RingReducer::InitRingField(RingField* rf, int chunk_idx, int subdiv_idx, + int field_idx) { + // Note on field indexing: There are group_size_ devices in the + // instance, implying the same number of chunks per tensor, where a + // chunk is the unit of data transferred in a time step. However, if + // a device can simultaenously send data by 2 or more independent + // channels we can speed up the transfer by subdividing chunks and + // processing multiple subdivisions at once. So the actual number + // of RingFields is group_size_ * num_subdivs_. + DCHECK_EQ(field_idx, (chunk_idx * num_subdivs_) + subdiv_idx); + rf->chunk_idx = chunk_idx; + rf->subdiv_idx = subdiv_idx; + rf->sc_idx = field_idx; + rf->rank = col_params_.subdiv_rank[subdiv_idx]; + rf->second_pass = false; + rf->action = RF_INIT; + // Recv from the device with preceding rank within the subdivision. + int recv_from_rank = (rf->rank + (group_size_ - 1)) % group_size_; + int send_to_rank = (rf->rank + 1) % group_size_; + rf->recv_dev_idx = col_params_.instance.impl_details + .subdiv_permutations[subdiv_idx][recv_from_rank]; + int send_dev_idx = col_params_.instance.impl_details + .subdiv_permutations[subdiv_idx][send_to_rank]; + rf->recv_is_remote = !col_params_.task.is_local[rf->recv_dev_idx]; + rf->send_is_remote = !col_params_.task.is_local[send_dev_idx]; + if (ca_->ChunkBytes(rf->sc_idx) > 0) { + // In pass 0 we skip Recv when rank = chunk_idx + rf->do_recv = (rf->chunk_idx != rf->rank); + // In pass 0 we skip Send when rank = chunk_idx-1 + rf->do_send = + (rf->rank != ((rf->chunk_idx + (group_size_ - 1)) % group_size_)); + } + rf->is_final = + (rf->rank == ((rf->chunk_idx + (group_size_ - 1)) % group_size_)); + if (rf->do_send || rf->do_recv) { + rf->chunk = ca_->ChunkAlias(rf->sc_idx); + CHECK(rf->chunk.IsAligned()) << rf->DebugString(); + } + if (rf->do_recv) { + rf->tmp_chunk = ca_->TempChunk(rf->sc_idx); + CHECK(rf->tmp_chunk.IsAligned()) << rf->DebugString(); + } + VLOG(2) << this << " InitRingField " << rf->DebugString() << " chunk " + << ca_->TBounds(rf->chunk); +} + +// When a RingField transitions from first to second recompute the +// do_send and do_recv values. +void RingReducer::AdvanceToSecondPass(RingField* rf) { + VLOG(3) << "IncrRingField old value " << rf->DebugString(); + CHECK(!rf->second_pass); + rf->second_pass = true; + rf->action = RF_INIT; + if (ca_->ChunkBytes(rf->sc_idx) > 0) { + // In pass 1 the send/no-send boundary moves down 1 place. + rf->do_recv = + (rf->rank != ((rf->chunk_idx + (group_size_ - 1)) % group_size_)); + rf->do_send = + (rf->rank != ((rf->chunk_idx + (group_size_ - 2)) % group_size_)); + } + rf->is_final = + (rf->rank == ((rf->chunk_idx + (group_size_ - 2)) % group_size_)); + VLOG(3) << "IncrRingField new value " << rf->DebugString(); +} + +string RingReducer::RingField::DebugString() const { + string rv = strings::StrCat("RingField rank=", rank, " chunk_idx=", chunk_idx, + " subdiv=", subdiv_idx, " sc_idx=", sc_idx, + " action=", action); + strings::StrAppend(&rv, " pass=", second_pass); + strings::StrAppend(&rv, " do_send=", do_send, " do_recv=", do_recv, + " is_final=", is_final, " recv_is_remote=", recv_is_remote, + " recv_dev_idx=", recv_dev_idx, " sc_idx=", sc_idx); + return rv; +} + +void RingReducer::DispatchSend(RingField* rf, const StatusCallback& done) { + CHECK(rf->do_send); + string send_buf_key = + RingReduceBufKey(exec_key_, rf->second_pass, rf->sc_idx, rf->rank); + VLOG(3) << "DispatchSend rank=" << col_params_.default_rank << " send key " + << send_buf_key << " chunk " << ca_->TBounds(rf->chunk) << " sc_idx " + << rf->sc_idx; + int send_to_rank = (rf->rank + 1) % group_size_; + int send_to_dev_idx = col_params_.instance.impl_details + .subdiv_permutations[rf->subdiv_idx][send_to_rank]; + col_exec_->PostToPeer(col_params_.instance.device_names[send_to_dev_idx], + col_params_.instance.task_names[send_to_dev_idx], + send_buf_key, device_, ctx_->op_device_context(), + ctx_->output_alloc_attr(0), &rf->chunk, + device_locality_, done); +} + +void RingReducer::DispatchRecv(RingField* rf, const StatusCallback& done) { + CHECK(rf->do_recv); + string recv_buf_key = + RingReduceBufKey(exec_key_, rf->second_pass, rf->sc_idx, + (rf->rank + (group_size_ - 1)) % group_size_); + VLOG(3) << "DispatchRecv rank=" << col_params_.default_rank << " recv key " + << recv_buf_key << " chunk " << ca_->TBounds(rf->chunk) << " into " + << ((col_params_.merge_op != nullptr) ? "tmp_chunk" : "chunk"); + Tensor* dst_tensor = (!rf->second_pass && (col_params_.merge_op != nullptr)) + ? &rf->tmp_chunk + : &rf->chunk; + col_exec_->RecvFromPeer(col_params_.instance.device_names[rf->recv_dev_idx], + col_params_.instance.task_names[rf->recv_dev_idx], + col_params_.task.is_local[rf->recv_dev_idx], + recv_buf_key, device_, ctx_->op_device_context(), + ctx_->output_alloc_attr(0), dst_tensor, + device_locality_, done); +} + +string RingReducer::FieldState() { + string s = strings::StrCat("RingReducer ", + strings::Hex(reinterpret_cast(this)), + " exec ", exec_key_, " step_id=", step_id_, + " state of all ", rfv_.size(), " fields:"); + for (int i = 0; i < rfv_.size(); ++i) { + s.append("\n"); + s.append(rfv_[i].DebugString()); + } + return s; +} + +bool RingReducer::RunAsyncParts() { + // This function orchestrates RingReduce actions on behalf of a + // single device. It is entered by a blockable thread that + // loops within it until all actions assigned to that device + // complete. Hence function local variables are accessible only by that + // one thread and do not require an explicit mutex. + rfv_.clear(); + rfv_.resize(group_size_ * num_subdivs_); + PCQueue ready_queue; + int field_done_count = 0; + int send_pending_count = 0; + int recv_pending_count = 0; + std::atomic aborted(false); + field_done_count = 0; + send_pending_count = 0; + recv_pending_count = 0; + for (int chunk_idx = 0; chunk_idx < group_size_; ++chunk_idx) { + for (int subdiv_idx = 0; subdiv_idx < num_subdivs_; ++subdiv_idx) { + int rf_index = (chunk_idx * num_subdivs_) + subdiv_idx; + InitRingField(&rfv_[rf_index], chunk_idx, subdiv_idx, rf_index); + ready_queue.Enqueue(&rfv_[rf_index]); + } + } + + // Loop until all RingFields have advanced to completion. + while (field_done_count < rfv_.size()) { + VLOG(4) << FieldState(); + // Wait for a RingField to appear in the ready_queue. + RingField* rf = ready_queue.Dequeue(); + // Advance the RingField to its next action and execute, repeating + // until either an async action has been started or the RingField + // is done. + bool dispatched = false; // true if async action was initiated + do { + if (aborted) break; + switch (rf->action) { + case RF_INIT: + if (rf->do_recv) { + rf->action = RF_RECV; + auto requeue = [this, rf, &ready_queue, &aborted](Status s) { + if (!s.ok()) { + aborted = true; + StartAbort(s); + } + ready_queue.Enqueue(rf); + }; + DispatchRecv(rf, requeue); + dispatched = true; + ++recv_pending_count; + } else { + rf->action = RF_SEND_READY; + } + break; + case RF_RECV: + CHECK_GT(recv_pending_count, 0); + --recv_pending_count; + if (!rf->second_pass) { + rf->action = RF_REDUCE; + Status s = ComputeBinOp(device_, col_params_.merge_op.get(), + &rf->chunk, &rf->tmp_chunk); + if (!s.ok()) { + aborted = true; + StartAbort(s); + } + } else { + rf->action = RF_SEND_READY; + } + break; + case RF_REDUCE: + if (!rf->second_pass && col_params_.final_op.get() && rf->is_final) { + rf->action = RF_FINALIZE; + group_size_tensor_ready_.WaitForNotification(); + Status s = ComputeBinOp(device_, col_params_.final_op.get(), + &rf->chunk, &group_size_tensor_); + if (!s.ok()) { + aborted = true; + StartAbort(s); + } + } else { + rf->action = RF_SEND_READY; + } + break; + case RF_FINALIZE: + rf->action = RF_DONE; + break; + case RF_SEND_READY: + if (rf->do_send) { + rf->action = RF_SEND; + auto send_complete = [this, rf, &ready_queue, &aborted](Status s) { + if (!s.ok()) { + aborted = true; + StartAbort(s); + } + ready_queue.Enqueue(rf); + }; + DispatchSend(rf, send_complete); + dispatched = true; + ++send_pending_count; + } else { + rf->action = RF_DONE; + } + break; + case RF_SEND: + CHECK_GT(send_pending_count, 0); + --send_pending_count; + rf->action = RF_DONE; + break; + case RF_DONE: + break; + } + if (rf->action == RF_DONE) { + if (rf->second_pass) { + ++field_done_count; + break; // from do while(!dispatched) + } else { + AdvanceToSecondPass(rf); + } + } + } while (!dispatched); + if (aborted) break; + } // while (field_done_count < number of fields) + + if (aborted) { + // All of the pending data actions should be aborted; field the + // callbacks and clear the queue before quitting. + while ((send_pending_count > 0) || (recv_pending_count > 0)) { + RingField* rf = ready_queue.Dequeue(); + switch (rf->action) { + case RF_RECV: + --recv_pending_count; + break; + case RF_SEND: + --send_pending_count; + break; + default: {} // Ignore any other actions + } + } + } + + CHECK_EQ(send_pending_count, 0); + CHECK_EQ(recv_pending_count, 0); + + VLOG(2) << this << " rank=" << rank_ << " finish;" + << " final value " << TensorDebugString(ca_->Value()); + return !aborted; +} + +} // namespace tensorflow diff --git a/tensorflow/core/common_runtime/ring_reducer.h b/tensorflow/core/common_runtime/ring_reducer.h new file mode 100644 index 0000000000..8fde18dc1c --- /dev/null +++ b/tensorflow/core/common_runtime/ring_reducer.h @@ -0,0 +1,146 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_RING_REDUCER_H_ +#define TENSORFLOW_CORE_COMMON_RUNTIME_RING_REDUCER_H_ + +#include + +#include "tensorflow/core/common_runtime/base_collective_executor.h" +#include "tensorflow/core/framework/collective.h" +#include "tensorflow/core/framework/device_attributes.pb.h" + +namespace tensorflow { +class DeviceMgr; + +// Ring-algorithm implementation of collective all-reduce. +class RingReducer { + public: + RingReducer(CollectiveExecutor* col_exec, const DeviceMgr* dev_mgr, + OpKernelContext* ctx, OpKernelContext::Params* op_params, + const CollectiveParams& col_params, const string& exec_key, + int64 step_id, const Tensor* input, Tensor* output); + + virtual ~RingReducer() {} + + void Run(StatusCallback done); + + private: + // Called when a bad status is received that implies we should terminate + // execution and return a bad status. + void StartAbort(const Status& s); + void ContinueAfterInputCopy(); + void Finish(bool ok); + Status ComputeBinOp(Device* device, OpKernel* op, Tensor* output, + Tensor* input); + bool RunAsyncParts(); + + // Used for executing a sub-operation, e.g. a merge_op instance, with + // an OpKernelContext based on the one passed into this Op. + class SubContext { + public: + OpKernelContext::Params sub_params_; + gtl::InlinedVector sub_inputs_; + gtl::InlinedVector sub_input_attr_; + gtl::InlinedVector sub_input_dc_; + // Used only for Binary and Unary Ops for which we require + // the calculation to be in-place on the first input. + int forward_from_ = 0; + OpKernelContext* sub_ctx_; + SubContext(OpKernelContext* ctx, OpKernelContext::Params* params, + OpKernel* op, Tensor* output, Tensor* input); + ~SubContext() { delete sub_ctx_; } + }; + + // Current status of a RingField + enum RingFieldAction { + RF_INIT = 0, // Just initialized for a pass + RF_RECV, // Recv pending + RF_REDUCE, // Reduce pending + RF_FINALIZE, // FinalOp pending + RF_SEND_READY, // Ready to send + RF_SEND, // Send pending + RF_DONE, // No more work + }; + + // Tracks progress of actions on a single subfield of the entire tensor. + struct RingField { + int16 chunk_idx; // major division index + int16 subdiv_idx; // minor division index + int16 sc_idx; // subchunk index + int16 rank; // rank within subdiv permutation + int16 recv_dev_idx; // dev from which value should be recv'd + RingFieldAction action; + bool second_pass; + bool recv_is_remote = false; + bool send_is_remote = false; + bool do_send = false; // is the value sent in this pass? + bool do_recv = false; // is the value recv'd in this pass? + bool is_final = false; // is the last field in the pass for this rank + Tensor chunk; // alias to field values + Tensor tmp_chunk; + Status status; + string DebugString() const; + }; + void AdvanceToSecondPass(RingField* rf); + void InitRingField(RingField* rf, int chunk_idx, int subdiv_idx, + int field_idx); + void DispatchSend(RingField* rf, const StatusCallback& done); + void DispatchRecv(RingField* rf, const StatusCallback& done); + + // For constructing log messages for debugging. + string FieldState(); + string TensorDebugString(Tensor tensor); + + // Producer/Consumer Queue of RingField structs. + class PCQueue { + public: + void Enqueue(RingField* rf); + RingField* Dequeue(); + + private: + mutex pcq_mu_; + condition_variable cv_; + int waiter_count_ GUARDED_BY(pcq_mu_) = 0; + std::deque deque_ GUARDED_BY(pcq_mu_); + }; + + CollectiveExecutor* col_exec_; // Not owned + const DeviceMgr* dev_mgr_; // Not owned + OpKernelContext* ctx_; // Not owned + OpKernelContext::Params* op_params_; // Not owned + const CollectiveParams& col_params_; + const string exec_key_; + const Tensor* input_; // Not owned + Tensor* output_; // Not owned + const int rank_; + const int64 step_id_; + const int group_size_; + const int num_subdivs_; + Tensor group_size_tensor_; + Notification group_size_tensor_ready_; + std::unique_ptr ca_; + StatusCallback done_; + Device* device_; // The device for which this instance labors + const string device_name_; + DeviceLocality device_locality_; + + mutex status_mu_; + Status status_ GUARDED_BY(status_mu_); + + std::vector rfv_; +}; + +} // namespace tensorflow +#endif // TENSORFLOW_CORE_COMMON_RUNTIME_RING_REDUCER_H_ diff --git a/tensorflow/core/common_runtime/ring_reducer_test.cc b/tensorflow/core/common_runtime/ring_reducer_test.cc new file mode 100644 index 0000000000..e4387a074a --- /dev/null +++ b/tensorflow/core/common_runtime/ring_reducer_test.cc @@ -0,0 +1,606 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/core/common_runtime/ring_reducer.h" + +#include +#include "tensorflow/core/common_runtime/base_collective_executor.h" +#include "tensorflow/core/common_runtime/collective_rma_local.h" +#include "tensorflow/core/common_runtime/device.h" +#include "tensorflow/core/common_runtime/device_mgr.h" +#include "tensorflow/core/common_runtime/device_resolver_local.h" +#include "tensorflow/core/common_runtime/dma_helper.h" +#include "tensorflow/core/common_runtime/process_util.h" +#include "tensorflow/core/common_runtime/test_collective_executor_mgr.h" +#include "tensorflow/core/common_runtime/threadpool_device.h" +#include "tensorflow/core/framework/collective.h" +#include "tensorflow/core/framework/fake_input.h" +#include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/framework/node_def_builder.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/lib/core/notification.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/public/session_options.h" +#include "tensorflow/core/public/version.h" + +namespace tensorflow { +namespace { + +// Wraps CollectiveRemoteAccessLocal with the ability to return an +// error status to the N'th action. +class FailTestRMA : public CollectiveRemoteAccessLocal { + public: + FailTestRMA(const DeviceMgr* dev_mgr, DeviceResolverInterface* dev_resolver, + int64 step_id, int fail_after) + : CollectiveRemoteAccessLocal(dev_mgr, dev_resolver, step_id), + fail_after_(fail_after) {} + + bool MaybeFail(const StatusCallback& done) { + bool fail_now = false; + { + mutex_lock l(mu_); + if (fail_after_ > 0) { + fail_now = (--fail_after_ == 0); + } + } + if (fail_now) { + done(errors::Internal("Deliberate failure")); + return true; + } + return false; + } + + void RecvFromPeer(const string& peer_device, const string& peer_task, + bool peer_is_local, const string& key, Device* to_device, + DeviceContext* to_device_ctx, + const AllocatorAttributes& to_alloc_attr, Tensor* to_tensor, + const DeviceLocality& client_locality, + const StatusCallback& done) override { + if (MaybeFail(done)) return; + CollectiveRemoteAccessLocal::RecvFromPeer( + peer_device, peer_task, peer_is_local, key, to_device, to_device_ctx, + to_alloc_attr, to_tensor, client_locality, done); + } + + void PostToPeer(const string& peer_device, const string& peer_task, + const string& key, Device* from_device, + DeviceContext* from_device_ctx, + const AllocatorAttributes& from_alloc_attr, + const Tensor* from_tensor, + const DeviceLocality& client_locality, + const StatusCallback& done) override { + if (MaybeFail(done)) return; + CollectiveRemoteAccessLocal::PostToPeer( + peer_device, peer_task, key, from_device, from_device_ctx, + from_alloc_attr, from_tensor, client_locality, done); + } + + mutex mu_; + int fail_after_ GUARDED_BY(mu_); +}; + +std::unique_ptr GetKernel(const NodeDef& node, + const DeviceType& device_type, + DeviceBase* device) { + Status status; + std::unique_ptr k = CreateOpKernel( + device_type, device, device->GetAllocator(AllocatorAttributes()), node, + TF_GRAPH_DEF_VERSION, &status); + if (!status.ok()) { + LOG(FATAL) << status; + } + return k; +} + +std::unique_ptr GetAdd(DataType dtype, const DeviceType& device_type, + DeviceBase* device) { + NodeDef node_def; + NodeDefBuilder builder("add_node", "Add"); + TF_CHECK_OK(builder.Attr("T", dtype) + .Input(FakeInput(dtype)) + .Input(FakeInput(dtype)) + .Finalize(&node_def)); + return GetKernel(node_def, device_type, device); +} + +std::unique_ptr GetDiv(DataType dtype, const DeviceType& device_type, + DeviceBase* device) { + NodeDef node_def; + NodeDefBuilder builder("add_node", "Div"); + TF_CHECK_OK(builder.Attr("T", dtype) + .Input(FakeInput(dtype)) + .Input(FakeInput(dtype)) + .Finalize(&node_def)); + return GetKernel(node_def, device_type, device); +} + +static int64 kStepId = 123; + +class RingReducerTest : public ::testing::Test { + protected: + RingReducerTest() : device_type_(DEVICE_CPU) {} + + void SetUp() override { +#if GOOGLE_CUDA + auto device_factory = DeviceFactory::GetFactory("GPU"); + CHECK(device_factory); + SessionOptions options; + Status s = device_factory->CreateDevices( + options, "/job:worker/replica:0/task:0", &gpu_devices_); + CHECK(s.ok()); +#endif + } + + ~RingReducerTest() override { + stop_ = true; + for (auto i : instances_) { + delete i; + } + if (col_exec_) col_exec_->Unref(); + } + + void Init(int num_workers, int num_devices, DataType dtype, + const DeviceType& device_type, int num_subdivs, int fail_after) { + device_type_ = device_type; + std::vector local_devices; + SessionOptions sess_opts; + sess_opts.env = Env::Default(); + Bytes mem_limit(4 << 20); + DeviceLocality dev_locality; + for (int wi = 0; wi < num_workers; ++wi) { + for (int di = 0; di < num_devices; ++di) { + if (device_type == DEVICE_CPU) { + string dev_name = + strings::StrCat("/job:worker/replica:0/task:", wi, "/cpu:", di); + local_devices.push_back(new ThreadPoolDevice( + sess_opts, dev_name, mem_limit, dev_locality, cpu_allocator())); + } else if (device_type == DEVICE_GPU && !gpu_devices_.empty()) { + int dev_idx = (wi * num_devices) + di; + if (dev_idx >= static_cast(gpu_devices_.size())) { + LOG(INFO) << "dev_mgr has access to limited GPUs, reusing for more " + "than one ring node."; + } else { + local_devices.push_back(gpu_devices_[dev_idx]); + } + } else { + LOG(FATAL) << "Unsupported device_type " << device_type; + } + } + } + if (!dev_mgr_ || device_type == DEVICE_CPU) { + LOG(ERROR) << "resetting dev_mgr for " << local_devices.size() + << " devices: "; + dev_mgr_.reset(new DeviceMgr(local_devices)); + } + dev_resolver_.reset(new DeviceResolverLocal(dev_mgr_.get())); + rma_ = new FailTestRMA(dev_mgr_.get(), dev_resolver_.get(), kStepId, + fail_after); + col_exec_ = new BaseCollectiveExecutor(&col_exec_mgr_, rma_, kStepId, + dev_mgr_.get()); + col_params_.name = "test_collective"; + static const int kGroupKey = 5; + col_params_.group.group_key = kGroupKey; + col_params_.group.device_type = device_type; + col_params_.group.group_size = num_workers * num_devices; + static const int kInstanceKey = 17; + col_params_.instance.instance_key = kInstanceKey; + col_params_.instance.impl_details.subdiv_offsets.clear(); + col_params_.instance.type = REDUCTION_COLLECTIVE; + col_params_.instance.data_type = dtype; + col_params_.instance.impl_details.subdiv_permutations.resize(num_subdivs); + col_params_.subdiv_rank.resize(num_subdivs); + int subdiv_stride = num_devices / num_subdivs; + for (int sdi = 0; sdi < num_subdivs; ++sdi) { + col_params_.instance.impl_details.subdiv_offsets.push_back(sdi * + subdiv_stride); + col_params_.subdiv_rank[sdi] = sdi * subdiv_stride; + } + + // Set up a local device ring order that's not just 0,1,2... + std::vector local_ring_order; + for (int di = 0; di < num_devices; ++di) { + local_ring_order.push_back(di); + } + for (int di = 0; di < num_devices; ++di) { + bool is_odd = ((di % 2) == 1); + int other = (di + (is_odd ? 7 : 3)) % num_devices; + if (di == other) continue; + iter_swap(local_ring_order.begin() + di, + local_ring_order.begin() + other); + } + string lro_buf; + for (auto d : local_ring_order) strings::StrAppend(&lro_buf, d, ", "); + VLOG(1) << "local_ring_order " << lro_buf; + + // Set up all of the fake device contexts. + for (int wi = 0; wi < num_workers; ++wi) { + for (int di = 0; di < num_devices; ++di) { + string task_name = strings::StrCat("/job:worker/replica:0/task:", wi); + string dev_name = strings::StrCat(task_name, "/cpu:", di); + if (device_type == DEVICE_GPU) { + dev_name = + strings::StrCat(task_name, "/gpu:", di % gpu_devices_.size()); + } + col_params_.instance.device_names.push_back(dev_name); + col_params_.instance.task_names.push_back(task_name); + // Normally each device would set is_local to its own perspective but + // this test runs in a single process so is_local is always true. + col_params_.task.is_local.push_back(true); + for (int sdi = 0; sdi < num_subdivs; ++sdi) { + int rotated_di = + (di + col_params_.instance.impl_details.subdiv_offsets[sdi]) % + num_devices; + col_params_.instance.impl_details.subdiv_permutations[sdi].push_back( + wi * num_devices + local_ring_order[rotated_di]); + } + } + } + for (int wi = 0; wi < num_workers; ++wi) { + for (int di = 0; di < num_devices; ++di) { + int rank = wi * num_devices + di; + instances_.push_back(new DeviceInstance( + rank, col_params_.instance.device_names[rank], device_type_, this)); + } + } + } + + void Reduce() { + std::atomic done(0); + for (auto di : instances_) { + SchedClosure([di, &done] { + di->DoReduce(); + ++done; + }); + } + while (done < static_cast(instances_.size())) { + if (stop_) break; + Env::Default()->SleepForMicroseconds(1000); + } + } + + template + void RunTest(DataType dtype, const DeviceType& device_type, int num_workers, + int num_devices, int num_subdivs, int tensor_len, + int fail_after) { + Init(num_workers, num_devices, dtype, device_type, num_subdivs, fail_after); + std::vector expected(tensor_len, 0.0); + for (int di = 0; di < static_cast(instances_.size()); ++di) { + DeviceInstance* instance = instances_[di]; + instance->InitTensor( + dtype, TensorShape({tensor_len}), [&expected, dtype, di](Tensor* t) { + for (size_t i = 0; i < t->NumElements(); ++i) { + // The cast is necessary to prevent clang-tidy from insisting + // that a faster non-open source function be substituted. + float value = pow(10, static_cast(di)) * i; + if (dtype == DT_INT32 || dtype == DT_INT64) { + value = di * 10 + i; + } + t->flat()(i) = static_cast(value); + expected[i] += value; + } + }); + } + Reduce(); + if (fail_after > 0) { + // Confirm that every device terminated with the expected error status. + for (int di = 0; di < static_cast(instances_.size()); ++di) { + EXPECT_EQ("Deliberate failure", + instances_[di]->status_.error_message()); + } + } else { + // Confirm that every device computed the same correct reduction value. + for (int i = 0; i < tensor_len; ++i) { + expected[i] /= (num_workers * num_devices); + } + for (int di = 0; di < static_cast(instances_.size()); ++di) { + TF_EXPECT_OK(instances_[di]->status_); + Tensor* inst = &instances_[di]->tensor_; + CHECK(inst); + Tensor actual(dtype, TensorShape({tensor_len})); + if (device_type_ == DEVICE_CPU) { + CHECK(actual.CopyFrom(*inst, inst->shape())); + VLOG(1) << "actual " << actual.SummarizeValue(100); + } else if (device_type_ == DEVICE_GPU) { + Notification note; + Device* dev = instances_[di]->device_; + auto* dev_info = dev->tensorflow_gpu_device_info(); + CHECK(dev_info); + dev_info->default_context->CopyDeviceTensorToCPU( + inst, "" /*tensor_name*/, dev, &actual, [¬e](const Status& s) { + CHECK(s.ok()); + note.Notify(); + }); + note.WaitForNotification(); + } + + for (int i = 0; i < tensor_len; ++i) { + switch (dtype) { + case DT_FLOAT: + EXPECT_FLOAT_EQ(expected[i], actual.template flat()(i)) + << "Mismatch at device " << di << " index " << i; + break; + case DT_DOUBLE: + EXPECT_DOUBLE_EQ(expected[i], actual.template flat()(i)) + << "Mismatch at device " << di << " index " << i; + break; + case DT_INT32: + case DT_INT64: + EXPECT_EQ(expected[i], actual.template flat()(i)) + << "Mismatch at device " << di << " index " << i; + break; + default: + LOG(FATAL) << "unimplemented"; + } + } + } + } + } + + std::unique_ptr GetCollectiveReduce(const CollectiveParams& params, + Tensor* input, + const DeviceType& device_type, + DeviceBase* device) { + mutex_lock l(mu_); + NodeDef node_def; + NodeDefBuilder builder( + strings::StrCat("collective_reduce_", reduce_counter_++), + "CollectiveReduce"); + TF_CHECK_OK( + builder.Attr("T", params.instance.data_type) + .Attr("merge_op", "Add") + .Attr("final_op", "Id") + .Attr("group_size", params.group.group_size) + .Attr("group_key", params.group.group_key) + .Attr("instance_key", params.instance.instance_key) + .Attr("subdiv_offsets", params.instance.impl_details.subdiv_offsets) + .Input(FakeInput(params.instance.data_type)) + .Finalize(&node_def)); + return GetKernel(node_def, device_type, device); + } + + class DeviceInstance { + public: + DeviceInstance(int rank, const string& dev_name, + const DeviceType& device_type, RingReducerTest* parent) + : parent_(parent), + dev_name_(dev_name), + device_type_(device_type), + rank_(rank) { + TF_CHECK_OK(parent_->dev_mgr_->LookupDevice(dev_name, &device_)) + << "Couldn't find device " << dev_name + << " existing devices: " << parent_->dev_mgr_->DebugString(); + col_params_.name = parent_->col_params_.name; + col_params_.group.group_key = parent_->col_params_.group.group_key; + col_params_.group.device_type = parent_->col_params_.group.device_type; + col_params_.group.group_size = parent_->col_params_.group.group_size; + col_params_.instance = parent->col_params_.instance; + col_params_.task.is_local = parent_->col_params_.task.is_local; + col_params_.subdiv_rank = parent_->col_params_.subdiv_rank; + + int num_subdivs = static_cast(col_params_.subdiv_rank.size()); + int group_size = col_params_.group.group_size; + CHECK_EQ(group_size, + static_cast(col_params_.instance.device_names.size())); + // Id of this device is at rank position in first subdiv perm. + int my_device_id = + col_params_.instance.impl_details.subdiv_permutations[0][rank]; + col_params_.default_rank = my_device_id; + // Set rank for all other subdivs by finding that device_id. + for (int sdi = 0; sdi < num_subdivs; ++sdi) { + for (int r = 0; r < static_cast(col_params_.instance.impl_details + .subdiv_permutations[sdi] + .size()); + ++r) { + if (my_device_id == + col_params_.instance.impl_details.subdiv_permutations[sdi][r]) { + col_params_.subdiv_rank[sdi] = r; + break; + } + } + } + } + + void InitTensor(DataType dtype, const TensorShape& shape, + const std::function& init_f) { + tensor_ = + Tensor(device_->GetAllocator(AllocatorAttributes()), dtype, shape); + if (device_type_ == DEVICE_CPU) { + init_f(&tensor_); + } else if (device_type_ == DEVICE_GPU) { + Tensor cpu_tensor(dtype, shape); + init_f(&cpu_tensor); + auto* dev_info = device_->tensorflow_gpu_device_info(); + CHECK(dev_info); + Notification note; + dev_info->default_context->CopyCPUTensorToDevice( + &cpu_tensor, device_, &tensor_, [¬e](const Status& s) { + CHECK(s.ok()); + note.Notify(); + }); + note.WaitForNotification(); + } else { + LOG(FATAL) << "Unsupported device_type " << device_type_; + } + } + + void DoReduce() { + col_params_.merge_op = + GetAdd(col_params_.instance.data_type, device_type_, device_); + col_params_.final_op = + GetDiv(col_params_.instance.data_type, device_type_, device_); + + // Prepare an OpKernelContext. + OpKernelContext::Params op_params; + op_params.step_id = kStepId; + op_params.device = device_; + gtl::InlinedVector inputs; + inputs.push_back(TensorValue(&tensor_)); + op_params.inputs = &inputs; + gtl::InlinedVector input_aa( + {AllocatorAttributes()}); + op_params.input_alloc_attrs = &input_aa; + gtl::InlinedVector input_dc; + DeviceContext* dev_ctx = nullptr; + auto* dev_info = device_->tensorflow_gpu_device_info(); + if (dev_info) { + dev_ctx = dev_info->default_context; + dev_ctx->Ref(); + } else { + dev_ctx = new DeviceContext; + } + input_dc.push_back(dev_ctx); + op_params.input_device_contexts = &input_dc; + op_params.op_device_context = dev_ctx; + int forward_from = 0; + op_params.forward_from_array = &forward_from; + AllocatorAttributes generic_alloc_attr; + op_params.output_attr_array = &generic_alloc_attr; + std::unique_ptr op = parent_->GetCollectiveReduce( + col_params_, &tensor_, DEVICE_CPU, device_); + op_params.op_kernel = op.get(); + OpKernelContext ctx(&op_params, 1); + + // We never actually execute the kernel, so we need to do the + // output allocation that it would do, ourselves. + Tensor* output_tensor_ptr = nullptr; + TF_CHECK_OK(ctx.forward_input_or_allocate_output({0}, 0, tensor_.shape(), + &output_tensor_ptr)); + CHECK_EQ(output_tensor_ptr, ctx.mutable_output(0)); + + // Prepare a RingReducer instance. + string exec_key = + strings::StrCat(col_params_.instance.instance_key, ":0:0"); + RingReducer rr(parent_->col_exec_, parent_->dev_mgr_.get(), &ctx, + &op_params, col_params_, exec_key, kStepId, &tensor_, + &tensor_); + + // Start execution in a threadpool then wait for completion. + Notification notification; + SchedClosure([this, ¬ification, &rr]() { + rr.Run([this, ¬ification](Status s) { + status_ = s; + notification.Notify(); + }); + }); + notification.WaitForNotification(); + CHECK(tensor_.CopyFrom(*ctx.mutable_output(0), tensor_.shape())); + + dev_ctx->Unref(); + } + + const Tensor& tensor() { return tensor_; } + + RingReducerTest* parent_; + string dev_name_; + DeviceType device_type_; + int rank_; + Tensor tensor_; + Device* device_; + CollectiveParams col_params_; + std::unique_ptr ca_; + std::unique_ptr ctx_; + Status status_; + }; + + bool stop_ = false; + DeviceType device_type_; + TestCollectiveExecutorMgr col_exec_mgr_; + CollectiveExecutor* col_exec_; + CollectiveRemoteAccessLocal* rma_; + std::unique_ptr dev_resolver_; + std::vector instances_; + CollectiveParams col_params_; + std::vector gpu_devices_; + std::unique_ptr dev_mgr_; + mutex mu_; + int32 reduce_counter_ GUARDED_BY(mu_) = 0; +}; + +#define DEF_TEST(B, T, W, D, S, L, A) \ + TEST_F(RingReducerTest, \ + DaTy##B##_DevTy##T##_Wkr##W##_Dev##D##_Sdiv##S##_Len##L##_Abrt##A) { \ + DataType dtype = DT_##B; \ + switch (dtype) { \ + case DT_FLOAT: { \ + RunTest(dtype, DEVICE_##T, W, D, S, L, A); \ + } break; \ + case DT_DOUBLE: { \ + RunTest(dtype, DEVICE_##T, W, D, S, L, A); \ + } break; \ + case DT_INT32: { \ + RunTest(dtype, DEVICE_##T, W, D, S, L, A); \ + } break; \ + case DT_INT64: { \ + RunTest(dtype, DEVICE_##T, W, D, S, L, A); \ + } break; \ + default: \ + LOG(FATAL) << "Unimplemented"; \ + } \ + } + +#ifndef GOOGLE_CUDA +// Success tests +DEF_TEST(FLOAT, CPU, 1, 2, 1, 1, 0) +DEF_TEST(FLOAT, CPU, 1, 2, 1, 2, 0) +DEF_TEST(FLOAT, CPU, 1, 2, 1, 8, 0) +DEF_TEST(FLOAT, CPU, 1, 2, 1, 16, 0) +DEF_TEST(FLOAT, CPU, 1, 2, 1, 1001, 0) +DEF_TEST(FLOAT, CPU, 2, 4, 1, 128, 0) +DEF_TEST(FLOAT, CPU, 2, 8, 1, 1001, 0) +DEF_TEST(FLOAT, CPU, 2, 8, 1, 4096, 0) +DEF_TEST(FLOAT, CPU, 2, 8, 1, 9408, 0) +DEF_TEST(FLOAT, CPU, 2, 8, 3, 4095, 0) +DEF_TEST(FLOAT, CPU, 2, 8, 3, 1045991, 0) +DEF_TEST(FLOAT, CPU, 4, 4, 4, 1045991, 0) +DEF_TEST(DOUBLE, CPU, 1, 2, 1, 1001, 0) +DEF_TEST(DOUBLE, CPU, 2, 8, 3, 4095, 0) +DEF_TEST(INT32, CPU, 1, 2, 1, 1001, 0) +DEF_TEST(INT32, CPU, 2, 8, 3, 4095, 0) +DEF_TEST(INT64, CPU, 1, 2, 1, 1001, 0) +DEF_TEST(INT64, CPU, 2, 8, 3, 4095, 0) + +// Failure tests +DEF_TEST(FLOAT, CPU, 2, 8, 1, 9408, 7) +DEF_TEST(FLOAT, CPU, 2, 8, 2, 9408, 11) +#endif + +#ifdef GOOGLE_CUDA +// GPU tests. So long as the device names are all in a single tasks we +// bypass inter-worker routing code and can fake multiple GPUs with a single +// GPU, from the perspective of the RingReducer logic. So these tests +// are all single-worker. +DEF_TEST(FLOAT, GPU, 1, 2, 1, 1, 0) +DEF_TEST(FLOAT, GPU, 1, 2, 1, 2, 0) +DEF_TEST(FLOAT, GPU, 1, 2, 1, 8, 0) +DEF_TEST(FLOAT, GPU, 1, 2, 1, 16, 0) +DEF_TEST(FLOAT, GPU, 1, 2, 1, 1001, 0) +DEF_TEST(FLOAT, GPU, 1, 8, 1, 1001, 0) +DEF_TEST(FLOAT, GPU, 1, 8, 1, 4096, 0) +DEF_TEST(FLOAT, GPU, 1, 8, 3, 4095, 0) +DEF_TEST(FLOAT, GPU, 1, 8, 3, 1045991, 0) +DEF_TEST(FLOAT, GPU, 1, 4, 4, 1045991, 0) +DEF_TEST(DOUBLE, GPU, 1, 2, 1, 1001, 0) +// INT32 values are never on the GPU. +// DEF_TEST(INT32, GPU, 1, 2, 1, 1001, 0) +DEF_TEST(INT64, GPU, 1, 2, 1, 1001, 0) + +// Failure tests +DEF_TEST(FLOAT, GPU, 1, 8, 1, 9408, 2) +DEF_TEST(FLOAT, GPU, 1, 8, 2, 9408, 5) +#endif + +} // namespace +} // namespace tensorflow diff --git a/tensorflow/core/common_runtime/test_collective_executor_mgr.h b/tensorflow/core/common_runtime/test_collective_executor_mgr.h new file mode 100644 index 0000000000..d0d4f24b11 --- /dev/null +++ b/tensorflow/core/common_runtime/test_collective_executor_mgr.h @@ -0,0 +1,116 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_TEST_COLLECTIVE_EXECUTOR_MGR_H_ +#define TENSORFLOW_CORE_COMMON_RUNTIME_TEST_COLLECTIVE_EXECUTOR_MGR_H_ + +#include "tensorflow/core/framework/collective.h" +#include "tensorflow/core/lib/gtl/flatmap.h" + +namespace tensorflow { + +// Mock objects that can't actually execute a Collective, but satisfy +// general infrastructure expectations within tests that don't require +// full functionality. + +class TestCollectiveExecutor : public CollectiveExecutor { + public: + explicit TestCollectiveExecutor(CollectiveExecutorMgrInterface* cem) + : CollectiveExecutor(cem) {} + void RecvFromPeer(const string& peer_device, const string& peer_task, + bool peer_is_local, const string& key, Device* to_device, + DeviceContext* to_device_ctx, + const AllocatorAttributes& to_alloc_attr, Tensor* to_tensor, + const DeviceLocality& client_locality, //??? + const StatusCallback& done) override { + done(errors::Internal("Unimplemented")); + } + + void PostToPeer(const string& peer_device, const string& peer_task, + const string& key, Device* from_device, + DeviceContext* from_device_ctx, + const AllocatorAttributes& from_alloc_attr, + const Tensor* from_tensor, + const DeviceLocality& client_locality, + const StatusCallback& done) override { + done(errors::Internal("Unimplemented")); + } +}; + +class TestCollectiveExecutorMgr : public CollectiveExecutorMgrInterface { + public: + TestCollectiveExecutorMgr() {} + + ~TestCollectiveExecutorMgr() override { + for (auto& iter : table_) { + iter.second->Unref(); + } + } + + CollectiveExecutor* FindOrCreate(int64 step_id) override { + mutex_lock l(mu_); + CollectiveExecutor* ce = nullptr; + auto iter = table_.find(step_id); + if (iter != table_.end()) { + ce = iter->second; + } else { + ce = new TestCollectiveExecutor(this); + table_[step_id] = ce; + } + ce->Ref(); + return ce; + } + + void Cleanup(int64 step_id) override { + mutex_lock l(mu_); + auto iter = table_.find(step_id); + if (iter != table_.end()) { + iter->second->Unref(); + table_.erase(iter); + } + } + + ParamResolverInterface* GetParamResolver() const override { + LOG(FATAL); + return nullptr; + } + + DeviceResolverInterface* GetDeviceResolver() const override { + LOG(FATAL); + return nullptr; + } + + void GetStepSequenceAsync(const GetStepSequenceRequest* request, + GetStepSequenceResponse* response, + const StatusCallback& done) override { + done(errors::Internal("unimplemented")); + } + + void RefreshStepIdSequenceAsync(int64 graph_key, + const StatusCallback& done) override { + done(errors::Internal("unimplemented")); + } + + int64 NextStepId(int64 graph_key) override { + return CollectiveExecutor::kInvalidId; + } + + void RetireStepId(int64 graph_key, int64 step_id) override {} + + mutex mu_; + gtl::FlatMap table_ GUARDED_BY(mu_); +}; + +} // namespace tensorflow +#endif // TENSORFLOW_CORE_COMMON_RUNTIME_TEST_COLLECTIVE_EXECUTOR_MGR_H_ -- GitLab From ffbf77de81d0b7b4b169c92d0d9fbbdef5b8842a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 10:14:02 -0700 Subject: [PATCH 241/791] Introduced tool to run an HLO module in replicated fashion, by infeeding random data and outfeeding the data generated at each step. The arguments of the computation can be either read from the session module, or randomly generated. The tool uses the raw transfer manager API to infeed and outfeed the data. PiperOrigin-RevId: 192628605 --- tensorflow/compiler/xla/service/BUILD | 2 + tensorflow/compiler/xla/service/hlo_runner.cc | 189 ++++++++++++++---- tensorflow/compiler/xla/service/hlo_runner.h | 66 +++++- tensorflow/compiler/xla/shape_util.h | 5 + tensorflow/compiler/xla/tests/test_utils.cc | 4 +- tensorflow/compiler/xla/tests/test_utils.h | 3 +- 6 files changed, 221 insertions(+), 48 deletions(-) diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index db91e80407..65203fa2a0 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -2535,6 +2535,7 @@ cc_library( srcs = ["hlo_runner.cc"], hdrs = ["hlo_runner.h"], deps = [ + ":computation_placer", ":executable", ":hlo", ":transfer_manager", @@ -2551,6 +2552,7 @@ cc_library( "//tensorflow/core:lib", "//tensorflow/core:stream_executor_no_cuda", "//third_party/eigen3", + "@com_google_absl//absl/memory", ], ) diff --git a/tensorflow/compiler/xla/service/hlo_runner.cc b/tensorflow/compiler/xla/service/hlo_runner.cc index ec7d8210a7..2e834a79d9 100644 --- a/tensorflow/compiler/xla/service/hlo_runner.cc +++ b/tensorflow/compiler/xla/service/hlo_runner.cc @@ -16,21 +16,16 @@ limitations under the License. #include "tensorflow/compiler/xla/service/hlo_runner.h" -#include #include #include +#include "absl/memory/memory.h" #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/compiler/xla/layout_util.h" #include "tensorflow/compiler/xla/ptr_util.h" -#include "tensorflow/compiler/xla/service/backend.h" -#include "tensorflow/compiler/xla/service/executable.h" -#include "tensorflow/compiler/xla/service/hlo_computation.h" #include "tensorflow/compiler/xla/service/transfer_manager.h" #include "tensorflow/compiler/xla/shape_util.h" -#include "tensorflow/compiler/xla/statusor.h" #include "tensorflow/compiler/xla/tools/parser/hlo_parser.h" -#include "tensorflow/compiler/xla/types.h" #include "tensorflow/core/common_runtime/eigen_thread_pool.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/types.h" @@ -91,15 +86,6 @@ HloRunner::ReadModuleFromHloTextFile(const std::string& filename, return tools::Parse(hlo_string, config); } -// Define this in .cc file to avoid having to include eigen or forward declare -// these types in the header. -struct HloRunner::EigenThreadPoolWrapper { - std::unique_ptr pool; - std::unique_ptr device; -}; - -HloRunner::HloRunner() {} - HloRunner::HloRunner(se::Platform* platform) { BackendOptions backend_options; backend_options.set_platform(platform); @@ -113,32 +99,14 @@ StatusOr> HloRunner::Execute( std::unique_ptr module, const tensorflow::gtl::ArraySlice arguments, bool run_hlo_passes) { - if (run_hlo_passes) { - TF_ASSIGN_OR_RETURN( - module, backend().compiler()->RunHloPasses( - std::move(module), backend().default_stream_executor(), - /*device_allocator=*/nullptr)); - } - TF_ASSIGN_OR_RETURN( - std::unique_ptr executable, - backend().compiler()->RunBackend(std::move(module), - backend().default_stream_executor(), - /*device_allocator=*/nullptr)); - + TF_ASSIGN_OR_RETURN(std::unique_ptr executable, + CreateExecutable(std::move(module), run_hlo_passes)); se::Stream stream(backend().default_stream_executor()); stream.Init(); - ExecutableRunOptions run_options; - run_options.set_device_ordinal(backend().default_device_ordinal()); - run_options.set_stream(&stream); - run_options.set_allocator(backend().memory_allocator()); - run_options.set_inter_op_thread_pool(backend().inter_op_thread_pool()); - run_options.set_intra_op_thread_pool( - backend().eigen_intra_op_thread_pool_device()); - - ServiceExecutableRunOptions service_run_options( - run_options, backend().StreamBorrower(), - backend().inter_op_thread_pool()); + ServiceExecutableRunOptions service_run_options(GetServiceRunOptionsForDevice( + backend().default_device_ordinal(), &stream, nullptr)); + const ExecutableRunOptions& run_options = service_run_options.run_options(); // Copy arguments to device. std::vector> argument_buffers; @@ -178,10 +146,153 @@ StatusOr> HloRunner::Execute( return result_literal; } +StatusOr>> HloRunner::ExecuteReplicated( + std::unique_ptr module, + const ReplicatedExecuteOptions& options) { + TF_ASSIGN_OR_RETURN( + std::unique_ptr executable, + CreateExecutable(std::move(module), options.run_hlo_passes)); + TF_ASSIGN_OR_RETURN( + DeviceAssignment device_assignment, + backend().computation_placer()->AssignDevices(options.num_replicas, 1)); + std::vector> streams; + std::vector service_run_options; + std::vector> argument_buffers; + // Plus one so we can safely get &argument_buffer_ptrs[0] in case there are + // no arguments. + std::vector argument_buffer_ptrs( + options.num_replicas * options.arguments.size() + 1); + std::vector> + argument_buffer_slices; + int64 index = 0; + for (int64 i = 0; i < options.num_replicas; ++i) { + int64 device = device_assignment(i, 0); + TF_ASSIGN_OR_RETURN(se::StreamExecutor * executor, + backend().stream_executor(device)); + streams.push_back(absl::make_unique(executor)); + streams.back()->Init(); + service_run_options.emplace_back(GetServiceRunOptionsForDevice( + device, streams.back().get(), &device_assignment)); + + // Copy arguments to device. + for (const Literal* argument : options.arguments) { + TF_ASSIGN_OR_RETURN( + std::unique_ptr argument_buffer, + backend().transfer_manager()->AllocateScopedShapedBuffer( + argument->shape(), backend().memory_allocator(), device)); + TF_RETURN_IF_ERROR(backend().transfer_manager()->TransferLiteralToDevice( + executor, *argument, *argument_buffer)); + argument_buffers.push_back(std::move(argument_buffer)); + argument_buffer_ptrs[index++] = argument_buffers.back().get(); + } + argument_buffer_slices.emplace_back( + &argument_buffer_ptrs[index - options.arguments.size()], + options.arguments.size()); + } + + std::unique_ptr pool; + int64 num_threads = (options.infeed != nullptr) ? options.num_replicas : 0; + if (ShapeUtil::IsInitialized(options.outfeed_shape)) { + num_threads += options.num_replicas; + } + if (num_threads > 0) { + pool = absl::make_unique( + tensorflow::Env::Default(), "infeed_outfeed", + /*num_threads=*/num_threads); + } + if (options.infeed != nullptr) { + for (int64 i = 0; i < options.num_replicas; ++i) { + int64 device = device_assignment(i, 0); + pool->Schedule([this, device, &options]() { + se::StreamExecutor* executor = + backend().stream_executor(device).ValueOrDie(); + VLOG(1) << "Starting infeed on device " << device; + for (int64 step = 1; + options.infeed_steps < 0 || step <= options.infeed_steps; ++step) { + TF_CHECK_OK(backend().transfer_manager()->TransferLiteralToInfeed( + executor, *options.infeed)); + if (step % 100 == 0) { + VLOG(1) << "Infeed step " << step; + } + } + }); + } + } + if (ShapeUtil::IsInitialized(options.outfeed_shape)) { + for (int64 i = 0; i < options.num_replicas; ++i) { + int64 device = device_assignment(i, 0); + pool->Schedule([this, device, &options]() { + se::StreamExecutor* executor = + backend().stream_executor(device).ValueOrDie(); + VLOG(1) << "Starting outfeed on device " << device; + for (int64 step = 1; + options.infeed_steps < 0 || step <= options.infeed_steps; ++step) { + auto literal = absl::make_unique(); + TF_CHECK_OK(backend().transfer_manager()->TransferLiteralFromOutfeed( + executor, options.outfeed_shape, literal.get())); + if (options.outfeed_values != nullptr) { + options.outfeed_values->push_back(std::move(literal)); + } + if (step % 100 == 0) { + VLOG(1) << "Outfeed step " << step; + } + } + }); + } + } + + LOG(INFO) << "Replicated execution started"; + TF_ASSIGN_OR_RETURN(std::vector> results, + executable->ExecuteOnStreams(service_run_options, + argument_buffer_slices)); + LOG(INFO) << "Replicated execution terminated"; + + std::vector> exec_results; + for (int64 i = 0; i < options.num_replicas; ++i) { + TF_ASSIGN_OR_RETURN(std::unique_ptr result, + ScopedShapedBuffer::MakeScoped( + results[i].get(), backend().memory_allocator())); + TF_ASSIGN_OR_RETURN(std::unique_ptr literal, + backend().transfer_manager()->TransferLiteralFromDevice( + streams[i]->parent(), *result)); + exec_results.push_back(std::move(literal)); + } + return std::move(exec_results); +} + +StatusOr> HloRunner::CreateExecutable( + std::unique_ptr module, bool run_hlo_passes) { + if (run_hlo_passes) { + TF_ASSIGN_OR_RETURN( + module, backend().compiler()->RunHloPasses( + std::move(module), backend().default_stream_executor(), + backend().memory_allocator())); + } + return backend().compiler()->RunBackend(std::move(module), + backend().default_stream_executor(), + backend().memory_allocator()); +} + +ServiceExecutableRunOptions HloRunner::GetServiceRunOptionsForDevice( + int64 device, se::Stream* stream, DeviceAssignment* device_assignment) { + ExecutableRunOptions run_options; + run_options.set_device_ordinal(device); + run_options.set_stream(stream); + run_options.set_allocator(backend().memory_allocator()); + run_options.set_inter_op_thread_pool(backend().inter_op_thread_pool()); + run_options.set_intra_op_thread_pool( + backend().eigen_intra_op_thread_pool_device()); + if (device_assignment != nullptr) { + run_options.set_device_assignment(device_assignment); + } + return ServiceExecutableRunOptions(run_options, backend().StreamBorrower(), + backend().inter_op_thread_pool()); +} + Backend& HloRunner::backend() { if (!backend_) { backend_ = Backend::CreateDefaultBackend().ConsumeValueOrDie(); - VLOG(1) << "executing on platform " << backend().platform()->Name(); + VLOG(1) << "Executing on platform " << backend().platform()->Name(); } return *backend_; } diff --git a/tensorflow/compiler/xla/service/hlo_runner.h b/tensorflow/compiler/xla/service/hlo_runner.h index 06ce22a5b9..f54fb44766 100644 --- a/tensorflow/compiler/xla/service/hlo_runner.h +++ b/tensorflow/compiler/xla/service/hlo_runner.h @@ -16,12 +16,16 @@ limitations under the License. #ifndef TENSORFLOW_COMPILER_XLA_SERVICE_HLO_RUNNER_H_ #define TENSORFLOW_COMPILER_XLA_SERVICE_HLO_RUNNER_H_ +#include #include +#include #include #include #include "tensorflow/compiler/xla/service/backend.h" #include "tensorflow/compiler/xla/service/compiler.h" +#include "tensorflow/compiler/xla/service/computation_placer.h" +#include "tensorflow/compiler/xla/service/executable.h" #include "tensorflow/compiler/xla/service/hlo_computation.h" #include "tensorflow/compiler/xla/service/hlo_module.h" #include "tensorflow/compiler/xla/status_macros.h" @@ -40,9 +44,43 @@ namespace xla { // file), or parsed from a hlo textual IR string. class HloRunner { public: - HloRunner(); - - HloRunner(::perftools::gputools::Platform* platform); + // The options used to configure a ExecuteReplicated() call. + struct ReplicatedExecuteOptions { + // The number of devices the HLO module should be replicated onto. + int64 num_replicas = 1; + + // The arguments to be fed to each replica. Since this is used for a + // replicated execution, all the arguments are the same for all replicas. + std::vector arguments; + + // If the HLO module being run has an infeed instruction, this will be the + // data which will be fed to it, for as many as infeed_steps steps. + const Literal* infeed = nullptr; + + // The number of times the infeed literal should be fed to the HLO module. + // For a clean exit, this should match the iterations-per-loop parameter + // used when generating the HLO module proto (that is usually the main + // while bounary counter). A value higher then iterations-per-loop would + // lead to infeed threads feeding to a gone computation, while a lower + // value would trigger a stuck ExecuteReplicated() call (the computation + // will be trying to infeed data which will never come). + int64 infeed_steps = -1; + + // The shape of the outfeed operation. If empty, the HLO module does not + // generate any outfeed. + Shape outfeed_shape; + + // A pointer to a vector where the outfeed values will be stored. If + // nullptr, the values will be read and discarded. + std::vector>* outfeed_values = nullptr; + + // Whether the HLO passes should be run on the input module. Usually + // saved modules are coming from after the HLO pass pipeline, so triggering + // another run will likely cause errors. + bool run_hlo_passes = false; + }; + + explicit HloRunner(::perftools::gputools::Platform* platform); ~HloRunner(); @@ -86,6 +124,13 @@ class HloRunner { return Execute(std::move(module), argument_pointers, run_hlo_passes); } + // Executes a given HLO module into a set of replicas, and returns a map + // with the replica number as key, and the corresponding returned literal as + // value. + StatusOr>> ExecuteReplicated( + std::unique_ptr module, + const ReplicatedExecuteOptions& options); + // If backend is not created in the constructor, creates and returns the // default backend. If creation fails, crashes the program. // @@ -94,9 +139,18 @@ class HloRunner { Backend& backend(); private: - struct EigenThreadPoolWrapper; - - std::unique_ptr thread_pool_wrapper_; + // Creates an executable object given an HLO module. If run_hlo_passes is + // true, the HLO passes will be run before. + StatusOr> CreateExecutable( + std::unique_ptr module, bool run_hlo_passes); + + // Creates a ServiceExecutableRunOptions object to configure a run on device, + // using the provided stream object. If device_assignment is not nullptr, it + // will be used to configure the replication parameters. Replicated executions + // should pass the device_assignment parameter. + ServiceExecutableRunOptions GetServiceRunOptionsForDevice( + int64 device, ::perftools::gputools::Stream* stream, + DeviceAssignment* device_assignment); std::unique_ptr backend_; }; diff --git a/tensorflow/compiler/xla/shape_util.h b/tensorflow/compiler/xla/shape_util.h index 1375f981a8..6d228eff46 100644 --- a/tensorflow/compiler/xla/shape_util.h +++ b/tensorflow/compiler/xla/shape_util.h @@ -319,6 +319,11 @@ class ShapeUtil { // Returns an empty tuple shape. Can be used to indicate side-effects. static Shape MakeNil() { return MakeTupleShape({}); } + // Checks whether the shape is initialized. + static bool IsInitialized(const Shape& shape) { + return shape.element_type() != PRIMITIVE_TYPE_INVALID; + } + // Constructs a new shape with the given element type and sequence of // dimensions. static Shape MakeShape(PrimitiveType element_type, diff --git a/tensorflow/compiler/xla/tests/test_utils.cc b/tensorflow/compiler/xla/tests/test_utils.cc index e30d115fae..cda1989fad 100644 --- a/tensorflow/compiler/xla/tests/test_utils.cc +++ b/tensorflow/compiler/xla/tests/test_utils.cc @@ -340,8 +340,8 @@ StatusOr>> MakeFakeArguments( } Status VerifyHloModule(const perftools::gputools::Platform& platform, - HloModule* const module) { - return HloVerifier().Run(module).status(); + HloModule* const module, bool allow_mixed_precision) { + return HloVerifier(allow_mixed_precision).Run(module).status(); } } // namespace xla diff --git a/tensorflow/compiler/xla/tests/test_utils.h b/tensorflow/compiler/xla/tests/test_utils.h index 0fb024ffb0..b5ab779574 100644 --- a/tensorflow/compiler/xla/tests/test_utils.h +++ b/tensorflow/compiler/xla/tests/test_utils.h @@ -69,7 +69,8 @@ StatusOr>> MakeFakeArguments( // Check that a given module satisfies various constraints before trying to // execute it. Status VerifyHloModule(const perftools::gputools::Platform& platform, - HloModule* const module); + HloModule* const module, + bool allow_mixed_precision = false); } // namespace xla -- GitLab From 844b8cae970d835850a75f8063324224b2de0df0 Mon Sep 17 00:00:00 2001 From: Eugene Brevdo Date: Thu, 12 Apr 2018 10:35:41 -0700 Subject: [PATCH 242/791] [TF] Add TensorListPushBackBatch. Also modify code to ensure aliased forwarding happens whenever possible with DT_VARIANT objects in ResourceVariables and in the new op. PiperOrigin-RevId: 192632202 --- .../base_api/api_def_TensorListGetItem.pbtxt | 3 + .../api_def_TensorListPushBackBatch.pbtxt | 3 + .../base_api/api_def_TensorListSetItem.pbtxt | 3 + tensorflow/core/kernels/list_kernels.cc | 16 +++ tensorflow/core/kernels/list_kernels.cu.cc | 15 +++ tensorflow/core/kernels/list_kernels.h | 121 ++++++++++++++++++ .../core/kernels/resource_variable_ops.cc | 7 +- tensorflow/core/ops/list_ops.cc | 44 +++++++ .../python/kernel_tests/list_ops_test.py | 42 ++++++ 9 files changed, 252 insertions(+), 2 deletions(-) create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorListGetItem.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorListPushBackBatch.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorListSetItem.pbtxt diff --git a/tensorflow/core/api_def/base_api/api_def_TensorListGetItem.pbtxt b/tensorflow/core/api_def/base_api/api_def_TensorListGetItem.pbtxt new file mode 100644 index 0000000000..2c47208fa0 --- /dev/null +++ b/tensorflow/core/api_def/base_api/api_def_TensorListGetItem.pbtxt @@ -0,0 +1,3 @@ +op { + graph_op_name: "TensorListGetItem" +} diff --git a/tensorflow/core/api_def/base_api/api_def_TensorListPushBackBatch.pbtxt b/tensorflow/core/api_def/base_api/api_def_TensorListPushBackBatch.pbtxt new file mode 100644 index 0000000000..1f33d49260 --- /dev/null +++ b/tensorflow/core/api_def/base_api/api_def_TensorListPushBackBatch.pbtxt @@ -0,0 +1,3 @@ +op { + graph_op_name: "TensorListPushBackBatch" +} diff --git a/tensorflow/core/api_def/base_api/api_def_TensorListSetItem.pbtxt b/tensorflow/core/api_def/base_api/api_def_TensorListSetItem.pbtxt new file mode 100644 index 0000000000..002e2a9bd3 --- /dev/null +++ b/tensorflow/core/api_def/base_api/api_def_TensorListSetItem.pbtxt @@ -0,0 +1,3 @@ +op { + graph_op_name: "TensorListSetItem" +} diff --git a/tensorflow/core/kernels/list_kernels.cc b/tensorflow/core/kernels/list_kernels.cc index 9e7786f25e..d1e481d7cc 100644 --- a/tensorflow/core/kernels/list_kernels.cc +++ b/tensorflow/core/kernels/list_kernels.cc @@ -475,6 +475,22 @@ REGISTER_KERNEL_BUILDER( #endif // GOOGLE_CUDA +#define REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(T) \ + REGISTER_KERNEL_BUILDER(Name("TensorListPushBackBatch") \ + .TypeConstraint("element_dtype") \ + .Device(DEVICE_CPU), \ + TensorListPushBackBatch) + +TF_CALL_ALL_TYPES(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU); +REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(quint8); +REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(qint8); +REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(quint16); +REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(qint16); +REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(qint32); +REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(bfloat16); + +#undef REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU + #define REGISTER_TENSOR_LIST_STACK_CPU(T) \ REGISTER_KERNEL_BUILDER(Name("TensorListStack") \ .TypeConstraint("element_dtype") \ diff --git a/tensorflow/core/kernels/list_kernels.cu.cc b/tensorflow/core/kernels/list_kernels.cu.cc index 935f892dd0..0ea9362cbe 100644 --- a/tensorflow/core/kernels/list_kernels.cu.cc +++ b/tensorflow/core/kernels/list_kernels.cu.cc @@ -51,6 +51,21 @@ REGISTER_TENSOR_LIST_STACK_GPU(bool); #undef REGISTER_TENSOR_LIST_STACK_GPU +#define REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU(T) \ + REGISTER_KERNEL_BUILDER(Name("TensorListPushBackBatch") \ + .TypeConstraint("element_dtype") \ + .Device(DEVICE_GPU), \ + TensorListPushBackBatch) + +TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU); +REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU(bfloat16); +TF_CALL_complex64(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU); +TF_CALL_complex128(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU); +TF_CALL_int64(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU); +REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU(bool); + +#undef REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU + #define REGISTER_TENSOR_LIST_FROM_TENSOR_GPU(T) \ REGISTER_KERNEL_BUILDER(Name("TensorListFromTensor") \ .TypeConstraint("element_dtype") \ diff --git a/tensorflow/core/kernels/list_kernels.h b/tensorflow/core/kernels/list_kernels.h index f3bbf3b6e3..42871c6113 100644 --- a/tensorflow/core/kernels/list_kernels.h +++ b/tensorflow/core/kernels/list_kernels.h @@ -34,6 +34,8 @@ limitations under the License. namespace tensorflow { +typedef Eigen::ThreadPoolDevice CPUDevice; + // Variant compatible type for a list of tensors. This is mutable but instances // should never be mutated after stored in a variant tensor. struct TensorList { @@ -146,6 +148,10 @@ class TensorListFromTensor : public OpKernel { TensorList output_list; const Tensor& t = c->input(0); output_list.element_dtype = t.dtype(); + OP_REQUIRES(c, TensorShapeUtils::IsVectorOrHigher(t.shape()), + errors::InvalidArgument( + "Tensor must be at least a vector, but saw shape: ", + t.shape().DebugString())); TensorShape output_shape(t.shape()); output_shape.RemoveDim(0); OP_REQUIRES(c, element_shape.IsCompatibleWith(output_shape), @@ -267,6 +273,121 @@ Status TensorListZerosLike(OpKernelContext* c, const TensorList& x, return Status::OK(); } +template +class TensorListPushBackBatch : public OpKernel { + public: + explicit TensorListPushBackBatch(OpKernelConstruction* c) : OpKernel(c) { + OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_)); + } + + ~TensorListPushBackBatch() override {} + + void Compute(OpKernelContext* c) override { + const Tensor& input = c->input(1); + OP_REQUIRES(c, element_dtype_ == input.dtype(), + errors::InvalidArgument("Invalid data types; list elements ", + DataTypeString(element_dtype_), + " but tried to append ", + DataTypeString(input.dtype()))); + OP_REQUIRES(c, TensorShapeUtils::IsVectorOrHigher(input.shape()), + errors::InvalidArgument( + "Expected tensor to be at least a vector, but saw shape: ", + input.shape().DebugString())); + + const TensorShape& tls_shape = c->input(0).shape(); + + // For purposes of input forwarding, we want the least restrictive + // AllocatorAttributes possible. If we need to allocate later, + // we'll request the DT_VARIANT be allocated on host. + AllocatorAttributes attr; + + std::unique_ptr tls_alias = c->forward_input( + 0 /*input_index*/, 0 /*output_index*/, DT_VARIANT, tls_shape, + DEVICE_MEMORY /* input is always on DEVICE_MEMORY */, attr); + + const Tensor& tls = tls_alias ? *tls_alias : c->input(0); + + OP_REQUIRES(c, tls.dtype() == DT_VARIANT, + errors::InvalidArgument( + "Expected input_handles dtype to be Variant, but saw: ", + DataTypeString(tls.dtype()))); + OP_REQUIRES(c, TensorShapeUtils::IsVector(tls_shape), + errors::InvalidArgument( + "Expected input_handles to be a vector, but saw shape: ", + tls_shape.DebugString())); + const int64 batch_size = tls.NumElements(); + OP_REQUIRES(c, input.dim_size(0) == batch_size, + errors::InvalidArgument( + "Expected tensor.shape[0] == input_handles.size, but saw ", + input.dim_size(0), " vs. ", batch_size)); + auto tls_t = tls.vec(); + + TensorShape input_element_shape = input.shape(); + input_element_shape.RemoveDim(0); + std::vector tl_batch; + for (int64 b = 0; b < batch_size; ++b) { + const TensorList* l = tls_t(b).get(); + OP_REQUIRES(c, l != nullptr, + errors::InvalidArgument("Input handle at index ", b, + " is not a list. Saw: '", + tls_t(b).DebugString(), "'")); + OP_REQUIRES( + c, l->element_shape.IsCompatibleWith(input_element_shape), + errors::InvalidArgument( + "Tried to append a tensor with incompatible shape to a " + "list at index ", + b, ". Op element shape: ", input_element_shape.DebugString(), + " list shape: ", l->element_shape.DebugString())); + OP_REQUIRES(c, element_dtype_ == l->element_dtype, + errors::InvalidArgument( + "Invalid data type at index ", b, "; op elements ", + DataTypeString(element_dtype_), " but list elements ", + DataTypeString(l->element_dtype))); + tl_batch.push_back(l); + } + + Tensor* result; + + if (tls_alias) { + result = tls_alias.get(); + c->set_output(0, *result); + } else { + // DT_VARIANT tensors always allocated on host. + AllocatorAttributes attr; + attr.set_on_host(true); + OP_REQUIRES_OK( + c, c->allocate_output(0, TensorShape{batch_size}, &result, attr)); + } + + if (batch_size == 0) { + return; + } + + auto input_t = input.flat_outer_dims(); + auto result_t = result->vec(); + + for (int64 b = 0; b < batch_size; ++b) { + if (!tls_alias) { + result_t(b) = *tl_batch[b]; + } + TensorList* output = result_t(b).get(); + DCHECK(output != nullptr); + Tensor* frame; + PersistentTensor tmp; + OP_REQUIRES_OK(c, c->allocate_persistent( + element_dtype_, input_element_shape, &tmp, &frame)); + if (input_element_shape.num_elements() > 0) { + auto frame_t = frame->flat(); + frame_t.device(c->eigen_device()) = input_t.template chip<0>(b); + } + output->tensors.push_back(std::move(*frame)); + } + } + + private: + DataType element_dtype_; +}; + } // namespace tensorflow #endif // TENSORFLOW_CORE_KERNELS_LIST_KERNELS_H_ diff --git a/tensorflow/core/kernels/resource_variable_ops.cc b/tensorflow/core/kernels/resource_variable_ops.cc index 72504200cc..916869fb56 100644 --- a/tensorflow/core/kernels/resource_variable_ops.cc +++ b/tensorflow/core/kernels/resource_variable_ops.cc @@ -306,8 +306,9 @@ class AssignVariableOp : public OpKernel { DataTypeString(variable->tensor()->dtype()), " got ", DataTypeString(DT_VARIANT))); + // For purposes of forwarding DT_VARIANT, we want the least + // restrictive attr; we already know the input is on host. AllocatorAttributes attr; - attr.set_on_host(true); // Copying is unnecessary if we are the last user of the value // tensor, we can just adopt the input tensor's buffer instead. @@ -320,7 +321,7 @@ class AssignVariableOp : public OpKernel { std::unique_ptr input_alias = context->forward_input( 1, OpKernelContext::Params::kNoReservation /*output_index*/, DT_VARIANT, value.shape(), - std::is_same::value ? HOST_MEMORY : DEVICE_MEMORY, + DEVICE_MEMORY /* HOST_MEMORY is only reserved for special cases */, attr); mutex_lock ml(*variable->mu()); @@ -337,6 +338,8 @@ class AssignVariableOp : public OpKernel { !variable->tensor()->shape().IsSameSize(value.shape())) { PersistentTensor unused; Tensor* tmp; + // Allocation of DT_VARIANT is always on host. + attr.set_on_host(true); OP_REQUIRES_OK(context, context->allocate_persistent(DT_VARIANT, value.shape(), &unused, &tmp, attr)); diff --git a/tensorflow/core/ops/list_ops.cc b/tensorflow/core/ops/list_ops.cc index c151055ee6..7af70110b7 100644 --- a/tensorflow/core/ops/list_ops.cc +++ b/tensorflow/core/ops/list_ops.cc @@ -71,6 +71,50 @@ REGISTER_OP("TensorListPushBack") return Status::OK(); }); +REGISTER_OP("TensorListPushBackBatch") + .Input("input_handles: variant") + .Input("tensor: element_dtype") + .Output("output_handles: variant") + .Attr("element_dtype: type") + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle input_handles; + TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 1, &input_handles)); + + shape_inference::ShapeHandle tensor; + TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(1), 1, &tensor)); + + TF_RETURN_IF_ERROR( + c->MergePrefix(tensor, input_handles, &tensor, &input_handles)); + + c->set_output(0, input_handles); + + DataType t; + TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t)); + shape_inference::ShapeHandle s = c->UnknownShape(); + + auto* handle_data = c->input_handle_shapes_and_types(0); + if (handle_data != nullptr && handle_data->size() != 1) { + return errors::InvalidArgument( + "Trying to push to list with wrong variant data."); + } + if (handle_data != nullptr) { + const shape_inference::ShapeAndType& list_shape_type = + (*handle_data)[0]; + if (list_shape_type.dtype != t) { + return errors::InvalidArgument( + "Trying to push to list with wrong element dtype. List has type ", + DataTypeString(list_shape_type.dtype), + " but trying to push element with type ", DataTypeString(t)); + } + shape_inference::ShapeHandle ignored; + TF_RETURN_IF_ERROR(c->Merge(s, list_shape_type.shape, &ignored)); + s = list_shape_type.shape; + } + c->set_output_handle_shapes_and_types( + 0, std::vector{{s, t}}); + return Status::OK(); + }); + REGISTER_OP("TensorListLength") .Input("input_handle: variant") .Output("length: int32") diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py index 6173a1def3..2084599760 100644 --- a/tensorflow/python/kernel_tests/list_ops_test.py +++ b/tensorflow/python/kernel_tests/list_ops_test.py @@ -318,6 +318,48 @@ class ListOpsTest(test_util.TensorFlowTestCase): [[1.0, 2.0]] * 4) self.assertAllEqual(self.evaluate(updated_v_stacked), expected) + @test_util.run_in_graph_and_eager_modes() + def testPushBackBatch(self): + c = constant_op.constant([1.0, 2.0], dtype=dtypes.float32) + l0 = list_ops.tensor_list_from_tensor(c, element_shape=scalar_shape()) + l1 = list_ops.tensor_list_from_tensor([-1.0], element_shape=scalar_shape()) + l_batch = array_ops.stack([l0, l1]) + l_push = list_ops.tensor_list_push_back_batch(l_batch, [3.0, 4.0]) + l_unstack = array_ops.unstack(l_push) + l0_ret = list_ops.tensor_list_stack(l_unstack[0], dtypes.float32) + l1_ret = list_ops.tensor_list_stack(l_unstack[1], dtypes.float32) + self.assertAllClose([1.0, 2.0, 3.0], self.evaluate(l0_ret)) + self.assertAllClose([-1.0, 4.0], self.evaluate(l1_ret)) + + with ops.control_dependencies([l_push]): + l_unstack_orig = array_ops.unstack(l_batch) + l0_orig_ret = list_ops.tensor_list_stack(l_unstack_orig[0], + dtypes.float32) + l1_orig_ret = list_ops.tensor_list_stack(l_unstack_orig[1], + dtypes.float32) + + # Check that without aliasing, push_back_batch still works; and + # that it doesn't modify the input. + l0_r_v, l1_r_v, l0_orig_v, l1_orig_v = self.evaluate( + (l0_ret, l1_ret, l0_orig_ret, l1_orig_ret)) + self.assertAllClose([1.0, 2.0, 3.0], l0_r_v) + self.assertAllClose([-1.0, 4.0], l1_r_v) + self.assertAllClose([1.0, 2.0], l0_orig_v) + self.assertAllClose([-1.0], l1_orig_v) + + # Pushing back mismatched shapes fails. + with self.assertRaises((errors.InvalidArgumentError, ValueError)): + self.evaluate(list_ops.tensor_list_push_back_batch(l_batch, [])) + + with self.assertRaisesRegexp(errors.InvalidArgumentError, + "incompatible shape to a list at index 0"): + self.evaluate( + list_ops.tensor_list_push_back_batch(l_batch, [[3.0], [4.0]])) + + with self.assertRaisesRegexp(errors.InvalidArgumentError, + "Invalid data type at index 0"): + self.evaluate(list_ops.tensor_list_push_back_batch(l_batch, [3, 4])) + if __name__ == "__main__": test.main() -- GitLab From 151c31ce75f4370fd3749f3b07ac8297d3b2e203 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 10:47:26 -0700 Subject: [PATCH 243/791] Make default weights initializer in `base_layers.Layer` suitable for their dtype. PiperOrigin-RevId: 192634133 --- .../keras/_impl/keras/engine/base_layer.py | 20 ++++++++++++++++--- tensorflow/python/layers/base_test.py | 6 ++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/keras/_impl/keras/engine/base_layer.py b/tensorflow/python/keras/_impl/keras/engine/base_layer.py index 3b3af7d092..6c68d25127 100644 --- a/tensorflow/python/keras/_impl/keras/engine/base_layer.py +++ b/tensorflow/python/keras/_impl/keras/engine/base_layer.py @@ -473,16 +473,30 @@ class Layer(checkpointable.CheckpointableBase): Raises: RuntimeError: If called with partioned variable regularization and eager execution is enabled. + ValueError: When giving unsupported dtype and no initializer. """ if dtype is None: dtype = self.dtype or backend.floatx() + else: + dtype = dtypes.as_dtype(dtype) initializer = initializers.get(initializer) - if initializer is None: - # Default TensorFlow initializer. - initializer = initializers.glorot_uniform() regularizer = regularizers.get(regularizer) constraint = constraints.get(constraint) + # Initialize variable when no initializer provided + if initializer is None: + # If dtype is DT_FLOAT, provide a uniform unit scaling initializer + if dtype.is_floating: + initializer = initializers.glorot_uniform() + # If dtype is DT_INT/DT_UINT, provide a default value `zero` + # If dtype is DT_BOOL, provide a default value `FALSE` + elif dtype.is_integer or dtype.is_unsigned or dtype.is_bool: + initializer = initializers.zeros() + # NOTES:Do we need to support for handling DT_STRING and DT_COMPLEX here? + else: + raise ValueError('An initializer for variable %s of type %s is required' + ' for layer %s' % (name, dtype.base_dtype, self.name)) + variable = self._add_variable_with_custom_getter( name=name, shape=shape, diff --git a/tensorflow/python/layers/base_test.py b/tensorflow/python/layers/base_test.py index c05c675263..f08b552840 100644 --- a/tensorflow/python/layers/base_test.py +++ b/tensorflow/python/layers/base_test.py @@ -52,6 +52,12 @@ class BaseLayerTest(test.TestCase): layer = base_layers.Layer(name='my_layer', trainable=False) self.assertEqual(layer.trainable, False) + @test_util.run_in_graph_and_eager_modes() + def testInt64Layer(self): + layer = base_layers.Layer(name='my_layer', dtype='int64') + layer.add_variable('my_var', [2, 2]) + self.assertEqual(layer.name, 'my_layer') + @test_util.run_in_graph_and_eager_modes() def testAddWeight(self): layer = base_layers.Layer(name='my_layer') -- GitLab From dc2d1c297a1e577151249d953a003357b4962b26 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 11:04:55 -0700 Subject: [PATCH 244/791] Fix shape inference for outside_compilation clusters that include cycles. PiperOrigin-RevId: 192637289 --- tensorflow/compiler/jit/BUILD | 8 ++ .../jit/encapsulate_subgraphs_pass.cc | 103 +++++++++++++++--- .../compiler/jit/shape_inference_helpers.cc | 66 +++++++++++ .../compiler/jit/shape_inference_helpers.h | 65 +++++++++++ 4 files changed, 228 insertions(+), 14 deletions(-) create mode 100644 tensorflow/compiler/jit/shape_inference_helpers.cc create mode 100644 tensorflow/compiler/jit/shape_inference_helpers.h diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD index 4cefc08645..6edeb7047f 100644 --- a/tensorflow/compiler/jit/BUILD +++ b/tensorflow/compiler/jit/BUILD @@ -183,6 +183,13 @@ cc_library( ], ) +cc_library( + name = "shape_inference_helpers", + srcs = ["shape_inference_helpers.cc"], + hdrs = ["shape_inference_helpers.h"], + deps = ["//tensorflow/core:graph"], +) + # Internal targets below this point. cc_library( @@ -293,6 +300,7 @@ cc_library( deps = [ ":common", ":graph_to_functiondef", + ":shape_inference_helpers", ":union_find", "//tensorflow/compiler/jit/graphcycles", "//tensorflow/compiler/jit/kernels:parallel_check_op", diff --git a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc index b04b333141..9465385b58 100644 --- a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc +++ b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc @@ -25,6 +25,7 @@ limitations under the License. #include "tensorflow/compiler/jit/graph_to_functiondef.h" #include "tensorflow/compiler/jit/legacy_flags/encapsulate_subgraphs_pass_flags.h" #include "tensorflow/compiler/jit/mark_for_compilation_pass.h" +#include "tensorflow/compiler/jit/shape_inference_helpers.h" #include "tensorflow/compiler/tf2xla/const_analysis.h" #include "tensorflow/compiler/tf2xla/dump_graph.h" #include "tensorflow/compiler/xla/status_macros.h" @@ -36,6 +37,7 @@ limitations under the License. #include "tensorflow/core/framework/node_def_builder.h" #include "tensorflow/core/framework/node_def_util.h" #include "tensorflow/core/graph/algorithm.h" +#include "tensorflow/core/graph/control_flow.h" #include "tensorflow/core/graph/graph.h" #include "tensorflow/core/graph/graph_def_builder.h" #include "tensorflow/core/graph/tensor_id.h" @@ -576,7 +578,8 @@ class Encapsulator { // satisfied, e.g., because send_node depends on a node that doesn't have a // registered shape inference function. Status DoStaticShapeInferenceForOutsideCompilationSend( - const Graph& graph_in, const ShapeRefiner& shape_refiner, + const Graph& graph_in, const BackEdgeHelper& back_edge_helper, + const ShapeRefiner& shape_refiner, const std::unordered_set& recv_at_host_nodes, Node* send_node, FunctionLibraryDefinition* library, std::vector* static_shape_out, @@ -599,7 +602,7 @@ class Encapsulator { // to nodes in pruned_graph. Status MakeGraphForOutsideCompilationSends( const Graph& graph, std::unique_ptr* pruned_graph, - ShapeRefiner* shape_refiner, + BackEdgeHelper* back_edge_helper, ShapeRefiner* shape_refiner, std::unordered_map* node_images, FunctionLibraryDefinition* library); @@ -1712,9 +1715,13 @@ namespace { // matter because it will only be used subsequently for shape inference. (It // would be possible to add a switch statement over data_type to create a value // for the constant, but that would entail maintaining the logic as new types -// are added, and is not necessary.) -Node* AddDummyShapedNode(DataType data_type, const TensorShapeProto& shape, - Graph* graph_out) { +// are added, and is not necessary.) If the node being replaced was within a +// control flow frame, adds appropriate Enter nodes so that the use of the Const +// is well-formed. +Node* AddDummyShapedNode(const Node* src_node, int src_port, + const std::vector& control_flow_info, + const TensorShapeProto& shape, Graph* graph_out) { + DataType data_type = src_node->output_type(src_port); TensorProto dummy_proto; dummy_proto.set_dtype(data_type); *dummy_proto.mutable_tensor_shape() = shape; @@ -1725,7 +1732,23 @@ Node* AddDummyShapedNode(DataType data_type, const TensorShapeProto& shape, NodeBuilder node_builder(options.GetNameForOp("KnownShape"), "Const", options.op_registry()); node_builder.Attr("dtype", data_type).Attr("value", dummy_proto); - return options.FinalizeBuilder(&node_builder); + Node* node = options.FinalizeBuilder(&node_builder); + // Add any Enter nodes required to bring the constant to the correct control + // flow frame. + while (!control_flow_info[src_node->id()].frame_name.empty()) { + NodeBuilder enter_builder(options.GetNameForOp("Enter"), "Enter", + options.op_registry()); + enter_builder.Attr("frame_name", + control_flow_info[src_node->id()].frame_name); + enter_builder.Attr("is_constant", true); + enter_builder.Input(node, 0); + Node* enter_node = options.FinalizeBuilder(&enter_builder); + // Adopt the new Enter node as the value in the current frame. + node = enter_node; + // Recurse to the parent frame to see if more Enter nodes need to be added. + src_node = control_flow_info[src_node->id()].parent_frame; + } + return node; } // Adds a copy of node_in to graph_out and adds the mapping to @@ -1767,17 +1790,30 @@ Status CopyShapeInferenceNodeToGraph( } } } + // Work around the fact that Enter nodes refuse to propagate shape information + // unless they are marked loop invariant. Since we are never going to execute + // this graph, marking them all loop invariant is fine. + if (node_out->type_string() == "Enter") { + node_out->ClearAttr("is_constant"); + node_out->AddAttr("is_constant", true); + } return Status::OK(); } } // namespace Status Encapsulator::DoStaticShapeInferenceForOutsideCompilationSend( - const Graph& graph_in, const ShapeRefiner& shape_refiner, + const Graph& graph_in, const BackEdgeHelper& back_edge_helper, + const ShapeRefiner& shape_refiner, const std::unordered_set& recv_at_host_nodes, Node* send_node, FunctionLibraryDefinition* library, std::vector* static_shape_out, std::unique_ptr* graph_out) { + // Get the control flow structure of the input graph so we can build + // well-formed output graphs. + std::vector control_flow_info; + TF_RETURN_IF_ERROR(BuildControlFlowInfo(&graph_in, &control_flow_info)); + // Maps from nodes in graph_in to nodes in graph_out. // // When an edge has fully defined shape the source node in graph_in is @@ -1802,7 +1838,6 @@ Status Encapsulator::DoStaticShapeInferenceForOutsideCompilationSend( // We don't use the standard ReverseDFS because we want to cut off traversal // whenever we find an output with fully defined shape. - // TODO(misard) make this work properly in the presence of control flow. struct Work { Node* node; bool leave; // Are we entering or leaving node? @@ -1840,8 +1875,9 @@ Status Encapsulator::DoStaticShapeInferenceForOutsideCompilationSend( TensorShapeProto proto; context->ShapeHandleToProto(shape, &proto); if (dummy_node_images.find(src_node) == dummy_node_images.end()) { - dummy_node_images[src_node] = AddDummyShapedNode( - src_node->output_type(src_port), proto, graph_out->get()); + dummy_node_images[src_node] = + AddDummyShapedNode(src_node, src_port, control_flow_info, + proto, graph_out->get()); } // The final input to the send node is the dynamic key, which we // don't include in the static shapes. @@ -1889,6 +1925,38 @@ Status Encapsulator::DoStaticShapeInferenceForOutsideCompilationSend( } } + for (const auto edge : back_edge_helper.RemovedEdges()) { + if (copied_node_images.find(edge.dst) != copied_node_images.end()) { + // The destination of this back edge was added to the inference graph, so + // fix it up. + Node* dst = copied_node_images[edge.dst]; + if (dst->type_string() != "Merge") { + return errors::InvalidArgument( + "outside_compilation cluster contains a back-edge to node ", + dst->name(), " of type ", dst->type_string(), + ". The analysis pass only supports back-edges to Merge nodes."); + } + const Edge* existing_input_edge; + if (edge.dst_input != 1 || dst->num_inputs() != 2 || + !dst->input_edge(0, &existing_input_edge).ok()) { + // TODO(misard) if we see graphs built with a different structure, relax + // this constraint. Leaving it here for now to avoid writing unnecessary + // complex code since we believe graphs generated by front ends all have + // the back edge as the second input to the merge node. + return errors::Internal( + "Internal assumption failed while rewriting an outside_compilation " + "cluster that contains a while loop. Logic assumes back-edge is to " + "port 1 of a 2-input " + "Merge node."); + } + // Connect the existing edge to both inputs of the Merge node so that the + // graph will be well-formed. + (*graph_out) + ->AddEdge(existing_input_edge->src(), + existing_input_edge->src_output(), dst, edge.dst_input); + } + } + return Status::OK(); } @@ -1956,7 +2024,7 @@ Status Encapsulator::MakePrunedGraphCopyAndInline( Status Encapsulator::MakeGraphForOutsideCompilationSends( const Graph& graph, std::unique_ptr* pruned_graph, - ShapeRefiner* shape_refiner, + BackEdgeHelper* back_edge_helper, ShapeRefiner* shape_refiner, std::unordered_map* node_images, FunctionLibraryDefinition* library) { // Find all the send_from_host nodes in all subgraphs, to use as roots for the @@ -1978,10 +2046,15 @@ Status Encapsulator::MakeGraphForOutsideCompilationSends( // nodes, inlining any functions as needed. TF_RETURN_IF_ERROR(MakePrunedGraphCopyAndInline( graph, send_from_host_nodes, pruned_graph, node_images, library)); + FixupSourceAndSinkEdges(pruned_graph->get()); + + // Remove back edges from any cycles in the pruned graph to simplify shape + // inference traversal. They will be fixed up in the per-subgraph shape + // inference graphs stored in the function library. + TF_RETURN_IF_ERROR(back_edge_helper->Remove(pruned_graph->get())); // Perform shape inference on the pruned graph. shape_refiner->set_require_shape_inference_fns(false); - FixupSourceAndSinkEdges(pruned_graph->get()); std::vector post_order; GetReversePostOrder(*(*pruned_graph), &post_order); for (auto node : post_order) { @@ -1999,11 +2072,13 @@ Status Encapsulator::MakeGraphForOutsideCompilationSends( Status Encapsulator::GetShapeInfoForOutsideCompilationSends( Graph* graph_out, FunctionLibraryDefinition* library) { + BackEdgeHelper back_edge_helper; std::unique_ptr pruned_graph; ShapeRefiner shape_refiner(graph_out->versions(), graph_out->op_registry()); std::unordered_map node_images; TF_RETURN_IF_ERROR(MakeGraphForOutsideCompilationSends( - *graph_out, &pruned_graph, &shape_refiner, &node_images, library)); + *graph_out, &pruned_graph, &back_edge_helper, &shape_refiner, + &node_images, library)); if (VLOG_IS_ON(1)) { dump_graph::DumpGraphToFile("pruned_graph_for_shape_inference", @@ -2033,7 +2108,7 @@ Status Encapsulator::GetShapeInfoForOutsideCompilationSends( std::unique_ptr graph; if (send_node != nullptr) { TF_RETURN_IF_ERROR(DoStaticShapeInferenceForOutsideCompilationSend( - *pruned_graph, shape_refiner, recv_at_host_names, + *pruned_graph, back_edge_helper, shape_refiner, recv_at_host_names, node_images[send_node], library, &static_shape, &graph)); if (graph == nullptr) { VLOG(2) << "Send node " << send_node->name() << " shapes"; diff --git a/tensorflow/compiler/jit/shape_inference_helpers.cc b/tensorflow/compiler/jit/shape_inference_helpers.cc new file mode 100644 index 0000000000..d9cfa16526 --- /dev/null +++ b/tensorflow/compiler/jit/shape_inference_helpers.cc @@ -0,0 +1,66 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Contains helpers for use in shape inference. + +#include "tensorflow/compiler/jit/shape_inference_helpers.h" + +#include + +#include "tensorflow/core/graph/graph.h" + +namespace tensorflow { + +Status BackEdgeHelper::Remove(Graph* graph) { + if (graph_ != nullptr) { + return errors::Internal("BackEdgeHelper duplicate call to Remove."); + } + graph_ = graph; + for (Node* n : graph_->nodes()) { + if (n->IsMerge()) { + for (const Edge* e : n->in_edges()) { + if (e->src()->IsNextIteration()) { + back_edges_.push_back( + BackEdge{e, e->src(), e->src_output(), e->dst(), e->dst_input()}); + } + } + } + } + for (const BackEdge& be : back_edges_) { + graph_->RemoveEdge(be.edge); + } + return Status::OK(); +} + +const std::vector& BackEdgeHelper::RemovedEdges() + const { + return back_edges_; +} + +Status BackEdgeHelper::Replace() { + if (graph_ == nullptr) { + return errors::Internal("BackEdgeHelper Replace called before Remove."); + } + if (replaced_) { + return errors::Internal("BackEdgeHelper Replace called more than once."); + } + replaced_ = true; + for (const BackEdge& be : back_edges_) { + graph_->AddEdge(be.src, be.src_output, be.dst, be.dst_input); + } + return Status::OK(); +} + +} // namespace tensorflow diff --git a/tensorflow/compiler/jit/shape_inference_helpers.h b/tensorflow/compiler/jit/shape_inference_helpers.h new file mode 100644 index 0000000000..2f053c9a45 --- /dev/null +++ b/tensorflow/compiler/jit/shape_inference_helpers.h @@ -0,0 +1,65 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_JIT_SHAPE_INFERENCE_HELPERS_H_ +#define TENSORFLOW_COMPILER_JIT_SHAPE_INFERENCE_HELPERS_H_ + +#include + +#include "tensorflow/core/graph/graph.h" + +namespace tensorflow { + +// Helper class to temporarily remove, then replace, the back edges in a +// graph. Simple algorithms for shape inference don't work with cycles, and this +// class can be used to remove cycles before running inference and replace them +// after. Correct usage requires exactly one call to Remove(), followed by any +// number of calls to RemovedEdges() and at most one call to Replace(). The call +// to Replace() is optional if the graph will be discarded without being +// executed, e.g., if it is being used purely for a shape inference pass. +class BackEdgeHelper { + public: + struct BackEdge { + const Edge* edge; + Node* src; + int src_output; + Node* dst; + int dst_input; + }; + + BackEdgeHelper() = default; + // Disallows copy and assign. + BackEdgeHelper(const BackEdgeHelper& other) = delete; + BackEdgeHelper& operator=(const BackEdgeHelper& other) = delete; + + // Temporarily removes all the back edges in graph. + Status Remove(Graph* graph); + + // Gets the list of removed edges. + const std::vector& RemovedEdges() const; + + // Replaces the back edges removed by a prior call to Remove. + Status Replace(); + + private: + Graph* graph_ = nullptr; // not owned + std::vector back_edges_; + // Set once Replace has been called. + bool replaced_ = false; +}; + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_JIT_SHAPE_INFERENCE_HELPERS_H_ -- GitLab From 4a405bc2d7398a0641632439652ec26e310d3359 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 11:23:44 -0700 Subject: [PATCH 245/791] Update ops-related pbtxt files. PiperOrigin-RevId: 192640621 --- .../core/ops/compat/ops_history.v1.pbtxt | 19 +++++++++++++++++++ tensorflow/core/ops/ops.pbtxt | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt index 30d4296326..a45a95ae09 100644 --- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt +++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt @@ -68696,6 +68696,25 @@ op { type: "type" } } +op { + name: "TensorListPushBackBatch" + input_arg { + name: "input_handles" + type: DT_VARIANT + } + input_arg { + name: "tensor" + type_attr: "element_dtype" + } + output_arg { + name: "output_handles" + type: DT_VARIANT + } + attr { + name: "element_dtype" + type: "type" + } +} op { name: "TensorListReserve" input_arg { diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index 0ed039ac2e..afb3dab3fe 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -32047,6 +32047,25 @@ op { type: "type" } } +op { + name: "TensorListPushBackBatch" + input_arg { + name: "input_handles" + type: DT_VARIANT + } + input_arg { + name: "tensor" + type_attr: "element_dtype" + } + output_arg { + name: "output_handles" + type: DT_VARIANT + } + attr { + name: "element_dtype" + type: "type" + } +} op { name: "TensorListReserve" input_arg { -- GitLab From 3ebe39c6152e587137ab580b7e1ec6861f1f22cc Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 11:35:39 -0700 Subject: [PATCH 246/791] Fix lost dependency PiperOrigin-RevId: 192643127 --- .../boosted_trees/estimator_batch/dnn_tree_combined_estimator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator.py b/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator.py index 449c130b2d..9994c84ebd 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator.py +++ b/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator.py @@ -32,6 +32,7 @@ from tensorflow.contrib.boosted_trees.python.training.functions import gbdt_batc from tensorflow.contrib.layers.python.layers import optimizers from tensorflow.contrib.learn.python.learn.estimators import estimator from tensorflow.contrib.learn.python.learn.estimators import head as head_lib +from tensorflow.contrib.learn.python.learn.estimators import model_fn from tensorflow.python.feature_column import feature_column as feature_column_lib from tensorflow.python.framework import ops from tensorflow.python.ops import control_flow_ops -- GitLab From 677156e7e857893fdf4acb8a9931fe2c97ab3246 Mon Sep 17 00:00:00 2001 From: Sami Kama Date: Thu, 12 Apr 2018 11:40:02 -0700 Subject: [PATCH 247/791] Make changes as per reviewer request --- tensorflow/contrib/tensorrt/BUILD | 11 +++++------ .../tensorrt/resources/trt_resource_manager.cc | 8 +++----- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD index 2a55a49097..2ee0c4589c 100644 --- a/tensorflow/contrib/tensorrt/BUILD +++ b/tensorflow/contrib/tensorrt/BUILD @@ -57,7 +57,6 @@ tf_custom_op_library( "ops/trt_engine_op.cc", ], deps = [ - # ":trt_engine_op_kernel", ":trt_shape_function", "//tensorflow/core:lib_proto_parsing", ] + if_tensorrt([ @@ -195,11 +194,11 @@ tf_py_wrap_cc( ) tf_cuda_library( - name = "trt_resource_manager_impl", - srcs = [ - "resources/trt_resource_manager.cc", + name = "trt_resource_manager_impl", + srcs = [ + "resources/trt_resource_manager.cc", ], - hdrs = [ + hdrs = [ "resources/trt_resource_manager.h", ], deps = [ @@ -230,7 +229,7 @@ tf_cuda_library( ] + if_tensorrt([ "@local_config_tensorrt//:nv_infer", ]) + if_static([ - ":trt_resource_manager_impl", + ":trt_resource_manager_impl", ]), ) diff --git a/tensorflow/contrib/tensorrt/resources/trt_resource_manager.cc b/tensorflow/contrib/tensorrt/resources/trt_resource_manager.cc index b9a5a00366..9c3698e5d1 100644 --- a/tensorflow/contrib/tensorrt/resources/trt_resource_manager.cc +++ b/tensorflow/contrib/tensorrt/resources/trt_resource_manager.cc @@ -19,11 +19,9 @@ limitations under the License. namespace tensorflow { namespace tensorrt { -std::shared_ptr -tensorflow::tensorrt::TRTResourceManager::instance() -{ - static std::shared_ptr instance_( - new tensorflow::tensorrt::TRTResourceManager); +std::shared_ptr +tensorflow::tensorrt::TRTResourceManager::instance() { + static std::shared_ptr instance_(new TRTResourceManager); return instance_; } -- GitLab From 024b037e9ad430c4023e3c9d250f3934f38de5cf Mon Sep 17 00:00:00 2001 From: Pete Warden Date: Thu, 12 Apr 2018 11:45:02 -0700 Subject: [PATCH 248/791] Fixed error where no background audio samples were being used when testing no-speech clips PiperOrigin-RevId: 192644704 --- tensorflow/examples/speech_commands/input_data.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tensorflow/examples/speech_commands/input_data.py b/tensorflow/examples/speech_commands/input_data.py index e7db9cddf0..63dd18457f 100644 --- a/tensorflow/examples/speech_commands/input_data.py +++ b/tensorflow/examples/speech_commands/input_data.py @@ -457,7 +457,7 @@ class AudioProcessor(object): self.time_shift_offset_placeholder_: time_shift_offset, } # Choose a section of background noise to mix in. - if use_background: + if use_background or sample['label'] == SILENCE_LABEL: background_index = np.random.randint(len(self.background_data)) background_samples = self.background_data[background_index] background_offset = np.random.randint( @@ -465,7 +465,9 @@ class AudioProcessor(object): background_clipped = background_samples[background_offset:( background_offset + desired_samples)] background_reshaped = background_clipped.reshape([desired_samples, 1]) - if np.random.uniform(0, 1) < background_frequency: + if sample['label'] == SILENCE_LABEL: + background_volume = np.random.uniform(0, 1) + elif np.random.uniform(0, 1) < background_frequency: background_volume = np.random.uniform(0, background_volume_range) else: background_volume = 0 -- GitLab From 10e60219b71fc48e07b0afaa6edeec2d9afac24d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 11:46:26 -0700 Subject: [PATCH 249/791] Go: Update generated wrapper functions for TensorFlow ops. PiperOrigin-RevId: 192644946 --- tensorflow/go/op/wrappers.go | 184 +++++++++++++++++------------------ 1 file changed, 92 insertions(+), 92 deletions(-) diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index 2d3e369328..1d5ebf6687 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -4932,6 +4932,70 @@ func IsNan(scope *Scope, x tf.Output) (y tf.Output) { return op.Output(0) } +// Identity op for gradient debugging. +// +// This op is hidden from public in Python. It is used by TensorFlow Debugger to +// register gradient tensors for gradient debugging. +// This op operates on non-reference-type tensors. +func DebugGradientIdentity(scope *Scope, input tf.Output) (output tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "DebugGradientIdentity", + Input: []tf.Input{ + input, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// ResourceSparseApplyAdadeltaAttr is an optional argument to ResourceSparseApplyAdadelta. +type ResourceSparseApplyAdadeltaAttr func(optionalAttr) + +// ResourceSparseApplyAdadeltaUseLocking sets the optional use_locking attribute to value. +// +// value: If True, updating of the var and accum tensors will be protected by +// a lock; otherwise the behavior is undefined, but may exhibit less contention. +// If not specified, defaults to false +func ResourceSparseApplyAdadeltaUseLocking(value bool) ResourceSparseApplyAdadeltaAttr { + return func(m optionalAttr) { + m["use_locking"] = value + } +} + +// var: Should be from a Variable(). +// +// Arguments: +// +// accum: Should be from a Variable(). +// accum_update: : Should be from a Variable(). +// lr: Learning rate. Must be a scalar. +// rho: Decay factor. Must be a scalar. +// epsilon: Constant factor. Must be a scalar. +// grad: The gradient. +// indices: A vector of indices into the first dimension of var and accum. +// +// Returns the created operation. +func ResourceSparseApplyAdadelta(scope *Scope, var_ tf.Output, accum tf.Output, accum_update tf.Output, lr tf.Output, rho tf.Output, epsilon tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyAdadeltaAttr) (o *tf.Operation) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "ResourceSparseApplyAdadelta", + Input: []tf.Input{ + var_, accum, accum_update, lr, rho, epsilon, grad, indices, + }, + Attrs: attrs, + } + return scope.AddOperation(opspec) +} + // Computes rectified linear gradients for a Relu operation. // // Arguments: @@ -12327,34 +12391,6 @@ func FusedResizeAndPadConv2D(scope *Scope, input tf.Output, size tf.Output, padd return op.Output(0) } -// Inverse 3D fast Fourier transform. -// -// Computes the inverse 3-dimensional discrete Fourier transform over the -// inner-most 3 dimensions of `input`. -// -// Arguments: -// input: A complex64 tensor. -// -// Returns A complex64 tensor of the same shape as `input`. The inner-most 3 -// dimensions of `input` are replaced with their inverse 3D Fourier transform. -// -// @compatibility(numpy) -// Equivalent to np.fft.ifftn with 3 dimensions. -// @end_compatibility -func IFFT3D(scope *Scope, input tf.Output) (output tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "IFFT3D", - Input: []tf.Input{ - input, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // Adds `bias` to `value`. // // This is a deprecated version of BiasAdd and will be soon removed. @@ -19183,6 +19219,34 @@ func Invert(scope *Scope, x tf.Output) (y tf.Output) { return op.Output(0) } +// Inverse 3D fast Fourier transform. +// +// Computes the inverse 3-dimensional discrete Fourier transform over the +// inner-most 3 dimensions of `input`. +// +// Arguments: +// input: A complex64 tensor. +// +// Returns A complex64 tensor of the same shape as `input`. The inner-most 3 +// dimensions of `input` are replaced with their inverse 3D Fourier transform. +// +// @compatibility(numpy) +// Equivalent to np.fft.ifftn with 3 dimensions. +// @end_compatibility +func IFFT3D(scope *Scope, input tf.Output) (output tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "IFFT3D", + Input: []tf.Input{ + input, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Deprecated. Disallowed in GraphDef version >= 2. // // DEPRECATED at GraphDef version 2: Use AdjustContrastv2 instead @@ -22625,70 +22689,6 @@ func TensorArrayGradV2(scope *Scope, handle tf.Output, flow_in tf.Output, source return op.Output(0) } -// ResourceSparseApplyAdadeltaAttr is an optional argument to ResourceSparseApplyAdadelta. -type ResourceSparseApplyAdadeltaAttr func(optionalAttr) - -// ResourceSparseApplyAdadeltaUseLocking sets the optional use_locking attribute to value. -// -// value: If True, updating of the var and accum tensors will be protected by -// a lock; otherwise the behavior is undefined, but may exhibit less contention. -// If not specified, defaults to false -func ResourceSparseApplyAdadeltaUseLocking(value bool) ResourceSparseApplyAdadeltaAttr { - return func(m optionalAttr) { - m["use_locking"] = value - } -} - -// var: Should be from a Variable(). -// -// Arguments: -// -// accum: Should be from a Variable(). -// accum_update: : Should be from a Variable(). -// lr: Learning rate. Must be a scalar. -// rho: Decay factor. Must be a scalar. -// epsilon: Constant factor. Must be a scalar. -// grad: The gradient. -// indices: A vector of indices into the first dimension of var and accum. -// -// Returns the created operation. -func ResourceSparseApplyAdadelta(scope *Scope, var_ tf.Output, accum tf.Output, accum_update tf.Output, lr tf.Output, rho tf.Output, epsilon tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyAdadeltaAttr) (o *tf.Operation) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "ResourceSparseApplyAdadelta", - Input: []tf.Input{ - var_, accum, accum_update, lr, rho, epsilon, grad, indices, - }, - Attrs: attrs, - } - return scope.AddOperation(opspec) -} - -// Identity op for gradient debugging. -// -// This op is hidden from public in Python. It is used by TensorFlow Debugger to -// register gradient tensors for gradient debugging. -// This op operates on non-reference-type tensors. -func DebugGradientIdentity(scope *Scope, input tf.Output) (output tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "DebugGradientIdentity", - Input: []tf.Input{ - input, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // Return substrings from `Tensor` of strings. // // For each string in the input `Tensor`, creates a substring starting at index -- GitLab From 454a22aa29dc2dba355094aabe733cd8419f2788 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 11:51:34 -0700 Subject: [PATCH 250/791] Construct Orthogonal kernels for 2d convolutions. PiperOrigin-RevId: 192645769 --- tensorflow/contrib/framework/__init__.py | 2 + .../python/kernel_tests/init_ops_test.py | 99 +++++++++- tensorflow/python/ops/init_ops.py | 186 +++++++++++++++++- 3 files changed, 282 insertions(+), 5 deletions(-) diff --git a/tensorflow/contrib/framework/__init__.py b/tensorflow/contrib/framework/__init__.py index cbb68bd3eb..a52907f163 100644 --- a/tensorflow/contrib/framework/__init__.py +++ b/tensorflow/contrib/framework/__init__.py @@ -72,6 +72,7 @@ See the @{$python/contrib.framework} guide. @@variable @@VariableDeviceChooser @@convolutional_delta_orthogonal +@@convolutional_orthogonal_2d @@zero_initializer @@load_checkpoint @@ -116,6 +117,7 @@ from tensorflow.python.framework.smart_cond import smart_constant_value from tensorflow.python.framework.tensor_spec import BoundedTensorSpec from tensorflow.python.framework.tensor_spec import TensorSpec from tensorflow.python.ops.init_ops import convolutional_delta_orthogonal +from tensorflow.python.ops.init_ops import convolutional_orthogonal_2d from tensorflow.python.util.all_util import remove_undocumented _allowed_symbols = ['nest'] diff --git a/tensorflow/python/kernel_tests/init_ops_test.py b/tensorflow/python/kernel_tests/init_ops_test.py index 1e5c118cbc..f7a7119b34 100644 --- a/tensorflow/python/kernel_tests/init_ops_test.py +++ b/tensorflow/python/kernel_tests/init_ops_test.py @@ -551,7 +551,6 @@ class OrthogonalInitializerTest(test.TestCase): init2 = init_ops.orthogonal_initializer(gain=3.14, seed=1, dtype=dtype) with self.test_session(graph=ops.Graph(), use_gpu=True): t1 = init1(shape).eval() - with self.test_session(graph=ops.Graph(), use_gpu=True): t2 = init2(shape).eval() return np.allclose(t1, t2 / 3.14, rtol=1e-15, atol=1e-15) @@ -610,7 +609,6 @@ class ConvolutionDeltaOrthogonalInitializerTest(test.TestCase): seed=1, dtype=dtype) with self.test_session(graph=ops.Graph(), use_gpu=True): t1 = init1(shape).eval() - with self.test_session(graph=ops.Graph(), use_gpu=True): t2 = init2(shape).eval() return np.allclose(t1, t2 / 3.14, rtol=1e-15, atol=1e-15) @@ -674,6 +672,103 @@ class ConvolutionDeltaOrthogonalInitializerTest(test.TestCase): self.assertAllClose(abs_value, count, rtol=tol, atol=tol) +class ConvolutionOrthogonal2dInitializerTest(test.TestCase): + + def testInitializerIdentical(self): + for dtype in [dtypes.float32, dtypes.float64]: + init1 = init_ops.convolutional_orthogonal_2d(seed=1, dtype=dtype) + init2 = init_ops.convolutional_orthogonal_2d(seed=1, dtype=dtype) + self.assertTrue(identicaltest(self, init1, init2, (3, 3, 10, 10))) + + def testInitializerDifferent(self): + for dtype in [dtypes.float32, dtypes.float64]: + init1 = init_ops.convolutional_orthogonal_2d(seed=1, dtype=dtype) + init2 = init_ops.convolutional_orthogonal_2d(seed=2, dtype=dtype) + self.assertFalse(identicaltest(self, init1, init2, (3, 3, 10, 10))) + + def testDuplicatedInitializer(self): + init = init_ops.convolutional_orthogonal_2d() + self.assertFalse(duplicated_initializer(self, init, 1, (3, 3, 10, 10))) + + def testInvalidDataType(self): + self.assertRaises( + ValueError, init_ops.convolutional_orthogonal_2d, + dtype=dtypes.string) + + def testInvalidShape(self): + init1 = init_ops.convolutional_orthogonal_2d() + with self.test_session(graph=ops.Graph(), use_gpu=True): + self.assertRaises(ValueError, init1, shape=[3, 3, 6, 5]) + + def testGain(self): + shape = (3, 3, 10, 10) + for dtype in [dtypes.float32, dtypes.float64]: + init1 = init_ops.convolutional_orthogonal_2d(seed=1, dtype=dtype) + init2 = init_ops.convolutional_orthogonal_2d(gain=3.14, + seed=1, dtype=dtype) + with self.test_session(graph=ops.Graph(), use_gpu=True): + t1 = init1(shape).eval() + t2 = init2(shape).eval() + return np.allclose(t1, t2 / 3.14, rtol=1e-15, atol=1e-15) + + def testShapesValues(self): + def circular_pad(input_, width, kernel_size): + """Pad input_ for computing (circular) convolution. + + Args: + input_: the input tensor + width: the width of the tensor. + kernel_size: the kernel size of the filter. + Returns: + a tensor whose width is (width + kernel_size - 1). + """ + beg = kernel_size // 2 + end = kernel_size - 1 - beg + + tmp_up = array_ops.slice(input_, [0, width - beg, 0, 0], + [-1, beg, width, -1]) + tmp_down = array_ops.slice(input_, [0, 0, 0, 0], [-1, end, width, -1]) + tmp = array_ops.concat([tmp_up, input_, tmp_down], 1) + + new_width = width + kernel_size - 1 + tmp_left = array_ops.slice(tmp, [0, 0, width - beg, 0], + [-1, new_width, beg, -1]) + tmp_right = array_ops.slice(tmp, [0, 0, 0, 0], [-1, new_width, end, -1]) + + final = array_ops.concat([tmp_left, tmp, tmp_right], 2) + return final + + cout = 45 + shape = [64, 28, 28, 32] + outputs_shape = shape[0:-1] + [cout] + dtype = dtypes.float32 + tol = 1e-3 + gain = 3.14 + # Check orthogonality/isometry by computing the ratio between + # the 2-norms of the inputs and ouputs. + for kernel_size in [[1, 1], [2, 2], [3, 3], [4, 4], [5, 5]]: + convolution = convolutional.conv2d + inputs = random_ops.random_normal(shape, dtype=dtype) + inputs_2norm = linalg_ops.norm(inputs) + input_with_circular_pad = circular_pad(inputs, shape[1], kernel_size[0]) + outputs = convolution( + input_with_circular_pad, padding="valid", filters=cout, + kernel_size=kernel_size, use_bias=False, + kernel_initializer=init_ops.convolutional_orthogonal_2d(gain=gain)) + outputs_2norm = linalg_ops.norm(outputs) + my_ops = variables.global_variables_initializer() + with self.test_session(use_gpu=True) as sess: + sess.run(my_ops) + # Check the shape of the outputs + t = outputs.eval() + self.assertAllEqual(t.shape, outputs_shape) + # Check isometry of the orthogonal kernel. + self.assertAllClose( + sess.run(inputs_2norm)/np.sqrt(np.prod(shape)), + sess.run(outputs_2norm)/(np.sqrt(np.prod(shape))*np.sqrt(gain)), + rtol=tol, atol=tol) + + class IdentityInitializerTest(test.TestCase): def testInvalidDataType(self): diff --git a/tensorflow/python/ops/init_ops.py b/tensorflow/python/ops/init_ops.py index 9dfe5ffbf4..5ded3f7cc2 100644 --- a/tensorflow/python/ops/init_ops.py +++ b/tensorflow/python/ops/init_ops.py @@ -499,10 +499,10 @@ class Orthogonal(Initializer): Args: gain: multiplicative factor to apply to the orthogonal matrix - dtype: The type of the output. seed: A Python integer. Used to create random seeds. See @{tf.set_random_seed} for behavior. + dtype: The data type. """ def __init__(self, gain=1.0, seed=None, dtype=dtypes.float32): @@ -552,10 +552,10 @@ class ConvolutionDeltaOrthogonal(Initializer): gain: multiplicative factor to apply to the orthogonal matrix. Default is 1. The 2-norm of an input is multiplied by a factor of 'sqrt(gain)' after applying this convolution. - dtype: The type of the output. seed: A Python integer. Used to create random seeds. See @{tf.set_random_seed} for behavior. + dtype: The data type. """ def __init__(self, gain=1.0, seed=None, dtype=dtypes.float32): @@ -581,7 +581,6 @@ class ConvolutionDeltaOrthogonal(Initializer): q, r = linalg_ops.qr(a, full_matrices=False) # Make Q uniform d = array_ops.diag_part(r) - # ph = d / math_ops.abs(d) q *= math_ops.sign(d) q = q[:shape[-2], :] q *= math_ops.sqrt(math_ops.cast(self.gain, dtype=dtype)) @@ -601,6 +600,186 @@ class ConvolutionDeltaOrthogonal(Initializer): return {"gain": self.gain, "seed": self.seed, "dtype": self.dtype.name} +class ConvolutionOrthogonal2D(Initializer): + """Initializer that generates a 2D orthogonal kernel for ConvNets. + + The shape of the tensor must have length 2. The number of input + filters must not exceed the number of output filters. + The orthogonality(==isometry) is exact when the inputs are circular padded. + There are finite-width effects with non-circular padding (e.g. zero padding). + + Args: + gain: multiplicative factor to apply to the orthogonal matrix. Default is 1. + The 2-norm of an input is multiplied by a factor of 'sqrt(gain)' after + applying this convolution. + seed: A Python integer. Used to create random seeds. See + @{tf.set_random_seed} + for behavior. + dtype: The data type. + """ + + def __init__(self, gain=1.0, seed=None, dtype=dtypes.float32): + self.gain = gain + self.dtype = _assert_float_dtype(dtypes.as_dtype(dtype)) + self.seed = seed + + def __call__(self, shape, dtype=None, partition_info=None): + if dtype is None: + dtype = self.dtype + # Check the shape + if len(shape) != 4: + raise ValueError("The tensor to initialize must be four-dimensional") + + if shape[-2] > shape[-1]: + raise ValueError("In_filters cannot be greater than out_filters.") + + if shape[0] != shape[1]: + raise ValueError("Kernel sizes must be equal.") + + kernel = self._orthogonal_kernel(shape[0], shape[2], shape[3]) + kernel *= math_ops.sqrt(math_ops.cast(self.gain, dtype=dtype)) + return kernel + + def get_config(self): + return {"gain": self.gain, "seed": self.seed, "dtype": self.dtype.name} + + # Helper functions. + def _orthogonal_matrix(self, n): + """Construct an n x n orthogonal matrix. + + Args: + n: dimension. + Returns: + a n x n orthogonal matrix. + """ + a = random_ops.random_normal([n, n], dtype=self.dtype, seed=self.seed) + if self.seed: + self.seed += 1 + q, r = linalg_ops.qr(a) + d = array_ops.diag_part(r) + # make q uniform + q *= math_ops.sign(d) + return q + + def _symmetric_projection(self, n): + """Compute a n x n symmetric projection matrix. + + Args: + n: dimension. + Returns: + a n x n symmetric projection matrix, i.e. a matrix P s.t. P=P*P, P=P^T. + """ + q = self._orthogonal_matrix(n) + # randomly zeroing out some columns + mask = math_ops.cast(random_ops.random_normal([n], seed=self.seed) > 0, + self.dtype) + if self.seed: + self.seed += 1 + c = math_ops.multiply(q, mask) + return math_ops.matmul(c, array_ops.matrix_transpose(c)) + + def _dict_to_tensor(self, x, k1, k2): + """Convert a dictionary to a tensor. + + Args: + x: a k1 * k2 dictionary. + k1: first dimension of x. + k2: second dimension of x. + Returns: + a k1 * k2 tensor. + """ + + return array_ops.stack([array_ops.stack([x[i, j] for j in range(k2)]) + for i in range(k1)]) + + def _block_orth(self, p1, p2): + """Construct a 2 x 2 kernel. Used to construct orthgonal kernel. + + Args: + p1: a symmetric projection matrix + p2: a symmetric projection matrix + Returns: + a 2 x 2 kernel [[p1p2, p1(1-p2)], + [(1-p1)p2, (1-p1)(1-p2)]]. + Raises: + ValueError: if the dimensions of p1 and p2 are different. + """ + if p1.shape.as_list() != p2.shape.as_list(): + raise ValueError("The dimension of the matrices must be the same.") + n = p1.shape.as_list()[0] + kernel2x2 = {} + eye = linalg_ops.eye(n, dtype=self.dtype) + kernel2x2[0, 0] = math_ops.matmul(p1, p2) + kernel2x2[0, 1] = math_ops.matmul(p1, (eye - p2)) + kernel2x2[1, 0] = math_ops.matmul((eye - p1), p2) + kernel2x2[1, 1] = math_ops.matmul((eye - p1), (eye - p2)) + + return kernel2x2 + + def _matrix_conv(self, m1, m2): + """Matrix convolution. + + Args: + m1: is a k x k dictionary, each element is a n x n matrix. + m2: is a l x l dictionary, each element is a n x n matrix. + + Returns: + (k + l - 1) * (k + l - 1) dictionary each element is a n x n matrix. + Raises: + ValueError: if the entries of m1 and m2 are of different dimensions. + """ + + n = (m1[0, 0]).shape.as_list()[0] + if n != (m2[0, 0]).shape.as_list()[0]: + raise ValueError("The entries in matrices m1 and m2 " + "must have the same dimensions!") + k = int(np.sqrt(len(m1))) + l = int(np.sqrt(len(m2))) + result = {} + size = k + l - 1 + # Compute matrix convolution between m1 and m2. + for i in range(size): + for j in range(size): + result[i, j] = array_ops.zeros([n, n], self.dtype) + for index1 in range(min(k, i + 1)): + for index2 in range(min(k, j + 1)): + if (i - index1) < l and (j - index2) < l: + result[i, j] += math_ops.matmul(m1[index1, index2], + m2[i - index1, j - index2]) + return result + + def _orthogonal_kernel(self, ksize, cin, cout): + """Construct orthogonal kernel for convolution. + + Args: + ksize: kernel size + cin: number of input channels + cout: number of output channels + Returns: + an [ksize, ksize, cin, cout] orthogonal kernel. + Raises: + ValueError: if cin > cout. + """ + if cin > cout: + raise ValueError("The number of input channels cannot exceed " + "the number of output channels.") + orth = self._orthogonal_matrix(cout)[0:cin, :] + if ksize == 1: + return array_ops.expand_dims(array_ops.expand_dims(orth, 0), 0) + + p = self._block_orth(self._symmetric_projection(cout), + self._symmetric_projection(cout)) + for _ in range(ksize - 2): + temp = self._block_orth(self._symmetric_projection(cout), + self._symmetric_projection(cout)) + p = self._matrix_conv(p, temp) + for i in range(ksize): + for j in range(ksize): + p[i, j] = math_ops.matmul(orth, p[i, j]) + + return self._dict_to_tensor(p, ksize, ksize) + + @tf_export("keras.initializers.Identity", "initializers.identity") class Identity(Initializer): """Initializer that generates the identity matrix. @@ -646,6 +825,7 @@ variance_scaling_initializer = VarianceScaling orthogonal_initializer = Orthogonal identity_initializer = Identity convolutional_delta_orthogonal = ConvolutionDeltaOrthogonal +convolutional_orthogonal_2d = ConvolutionOrthogonal2D # pylint: enable=invalid-name -- GitLab From 583ee0eabfb1bebd0eb533d2ab7a5c17af7e664e Mon Sep 17 00:00:00 2001 From: Igor Ganichev Date: Thu, 12 Apr 2018 11:54:21 -0700 Subject: [PATCH 251/791] Add testCompileTimeConstantsInDefun in xla PiperOrigin-RevId: 192646199 --- tensorflow/compiler/tests/function_test.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tensorflow/compiler/tests/function_test.py b/tensorflow/compiler/tests/function_test.py index 11d8a99ffe..fbc3c994d1 100644 --- a/tensorflow/compiler/tests/function_test.py +++ b/tensorflow/compiler/tests/function_test.py @@ -105,6 +105,28 @@ class FunctionTest(XLATestCase): result = sess.run(call_f) self.assertAllClose(result, expected, rtol=1e-3) + def testCompileTimeConstantsInDefun(self): + """Tests that XLA handles compile-time constants in defuns.""" + with self.test_session() as sess: + + @function.Defun(dtypes.float32, dtypes.int32, dtypes.int32) + def Foo(a, c, d): + # c and d must be known at compile time + x = array_ops.slice(a, c, d) + return x + + a = array_ops.placeholder(dtypes.float32) + c = array_ops.placeholder(dtypes.int32, shape=[4]) + d = array_ops.placeholder(dtypes.int32, shape=[4]) + with self.test_scope(): + call_f = Foo(a, c, d) + result = sess.run(call_f, feed_dict={ + a: np.ones([1, 4, 4, 1]), + c: [0, 0, 0, 0], + d: [1, 2, 2, 1]}) + + self.assertAllEqual(np.ones([1, 2, 2, 1]), result) + # TODO(b/36139787): Re-enable this test when noinline works again. def DISABLED_testFunctionsNoInline(self): -- GitLab From d1ee67c03a29d93fecd427f1a4693cb3fd6e6e38 Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Thu, 12 Apr 2018 11:59:08 -0700 Subject: [PATCH 252/791] Start moving Checkpointable utilities toward core Doesn't add to the public API yet, just shifts code around. Changes: - A tiny bit of renaming (to avoid having _Checkpoint and Checkpoint in the same file) - Removed the garbage collection decorator from a few tests due to the uuid4() garbage issue (apparently core tests get run on Python 2.7.9?) - Renamed "Object" to "CheckpointableObject" in the proto, since core protos have Java bindings and apparently Java had something else in mind for the keyword "Object" :) but otherwise this is a pure move. After this CL I'll propose adding tf.train.Checkpoint to the API (currently tf.contrib.eager.Checkpoint), move the utilities that are still in contrib/eager to their own contrib directory (there will be a few more misc. utilities for inspecting checkpoints and managing dependencies), get tf.train.Saver to read object-based checkpoints for compatibility, and work on Model.save_weights/load_weights. PiperOrigin-RevId: 192646890 --- tensorflow/contrib/cmake/python_modules.txt | 1 - tensorflow/contrib/cmake/python_protos.txt | 1 - .../python/kernel_tests/cudnn_rnn_test.py | 2 +- tensorflow/contrib/eager/proto/BUILD | 13 - tensorflow/contrib/eager/python/BUILD | 13 - .../eager/python/checkpointable_utils.py | 846 ----------- .../eager/python/checkpointable_utils_test.py | 1284 +--------------- .../contrib/eager/python/datasets_test.py | 2 +- .../eager/python/examples/spinn/spinn_test.py | 2 +- .../contrib/eager/python/metrics_test.py | 2 +- tensorflow/contrib/eager/python/tfe.py | 4 +- .../optimizer_v2/checkpointable_utils_test.py | 2 +- tensorflow/core/BUILD | 1 + .../checkpointable_object_graph.proto | 12 +- tensorflow/python/BUILD | 35 + .../python/training/checkpointable_utils.py | 850 ++++++++++- .../training/checkpointable_utils_test.py | 1308 +++++++++++++++++ 17 files changed, 2207 insertions(+), 2171 deletions(-) delete mode 100644 tensorflow/contrib/eager/proto/BUILD rename tensorflow/{contrib/eager/proto => core/protobuf}/checkpointable_object_graph.proto (85%) create mode 100644 tensorflow/python/training/checkpointable_utils_test.py diff --git a/tensorflow/contrib/cmake/python_modules.txt b/tensorflow/contrib/cmake/python_modules.txt index de84af866b..91839194c7 100644 --- a/tensorflow/contrib/cmake/python_modules.txt +++ b/tensorflow/contrib/cmake/python_modules.txt @@ -170,7 +170,6 @@ tensorflow/contrib/distributions/python tensorflow/contrib/distributions/python/ops tensorflow/contrib/distributions/python/ops/bijectors tensorflow/contrib/eager -tensorflow/contrib/eager/proto tensorflow/contrib/eager/python tensorflow/contrib/estimator tensorflow/contrib/estimator/python diff --git a/tensorflow/contrib/cmake/python_protos.txt b/tensorflow/contrib/cmake/python_protos.txt index 0c80d529af..d63c41db84 100644 --- a/tensorflow/contrib/cmake/python_protos.txt +++ b/tensorflow/contrib/cmake/python_protos.txt @@ -5,7 +5,6 @@ tensorflow/python tensorflow/contrib/boosted_trees/proto tensorflow/contrib/cloud/kernels tensorflow/contrib/decision_trees/proto -tensorflow/contrib/eager/proto tensorflow/contrib/gdr tensorflow/contrib/lite/toco tensorflow/contrib/mpi diff --git a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py index 9cc6ca09ad..6fb56b0858 100644 --- a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py +++ b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py @@ -29,7 +29,6 @@ import numpy as np from tensorflow.contrib.cudnn_rnn.python.layers import cudnn_rnn from tensorflow.contrib.cudnn_rnn.python.ops import cudnn_rnn_ops -from tensorflow.contrib.eager.python import checkpointable_utils from tensorflow.contrib.rnn.python.ops import rnn as contrib_rnn_lib from tensorflow.python.eager import backprop from tensorflow.python.eager import context @@ -55,6 +54,7 @@ from tensorflow.python.platform import test from tensorflow.python.platform import tf_logging as logging from tensorflow.python.training import adagrad from tensorflow.python.training import adam +from tensorflow.python.training import checkpointable_utils from tensorflow.python.training import gradient_descent from tensorflow.python.training import momentum from tensorflow.python.training import rmsprop diff --git a/tensorflow/contrib/eager/proto/BUILD b/tensorflow/contrib/eager/proto/BUILD deleted file mode 100644 index b016d2dcb5..0000000000 --- a/tensorflow/contrib/eager/proto/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -licenses(["notice"]) # Apache 2.0 - -exports_files(["LICENSE"]) - -load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library") - -tf_proto_library( - name = "checkpointable_object_graph_proto", - srcs = [ - "checkpointable_object_graph.proto", - ], - visibility = ["//tensorflow/contrib/eager/python:__subpackages__"], -) diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD index d97048405d..04e2d99048 100644 --- a/tensorflow/contrib/eager/python/BUILD +++ b/tensorflow/contrib/eager/python/BUILD @@ -230,21 +230,8 @@ py_library( srcs_version = "PY2AND3", visibility = ["//tensorflow:internal"], deps = [ - "//tensorflow/contrib/eager/proto:checkpointable_object_graph_proto_py", - "//tensorflow/python:constant_op", "//tensorflow/python:control_flow_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:errors", - "//tensorflow/python:framework_ops", - "//tensorflow/python:init_ops", - "//tensorflow/python:pywrap_tensorflow", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:session", - "//tensorflow/python:tensor_shape", "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - "//tensorflow/python/eager:context", ], ) diff --git a/tensorflow/contrib/eager/python/checkpointable_utils.py b/tensorflow/contrib/eager/python/checkpointable_utils.py index 34cb8d0e08..30c4103c5a 100644 --- a/tensorflow/contrib/eager/python/checkpointable_utils.py +++ b/tensorflow/contrib/eager/python/checkpointable_utils.py @@ -17,857 +17,11 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import abc -import collections import functools -import weakref -from tensorflow.contrib.eager.proto import checkpointable_object_graph_pb2 -from tensorflow.python import pywrap_tensorflow -from tensorflow.python.client import session as session_lib -from tensorflow.python.eager import context -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors_impl -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variable_scope from tensorflow.python.training import checkpointable as core_checkpointable -from tensorflow.python.training import checkpointable_utils as core_checkpointable_utils -from tensorflow.python.training import optimizer as optimizer_lib from tensorflow.python.training import saver as saver_lib -from tensorflow.python.util import deprecation - - -_ESCAPE_CHAR = "." # For avoiding conflicts with user-specified names. - -# Keyword for identifying that the next bit of a checkpoint variable name is a -# slot name. Checkpoint names for slot variables look like: -# -# /<_OPTIMIZER_SLOTS_NAME>// -# -# Where is a full path from the checkpoint root to the -# variable being slotted for. -_OPTIMIZER_SLOTS_NAME = _ESCAPE_CHAR + "OPTIMIZER_SLOT" -# Keyword for separating the path to an object from the name of an -# attribute in checkpoint names. Used like: -# /<_OBJECT_ATTRIBUTES_NAME>/ -_OBJECT_ATTRIBUTES_NAME = _ESCAPE_CHAR + "ATTRIBUTES" -# Key where the object graph proto is saved in a TensorBundle -_OBJECT_GRAPH_PROTO_KEY = "_CHECKPOINTABLE_OBJECT_GRAPH" - - -# TODO(allenl): If this ends up in a public API, consider adding LINT.IfChange -# or consolidating the implementation with get_variable. -def _default_getter(name, shape, dtype, initializer=None, - partition_info=None, **kwargs): - """A pared-down version of get_variable which does not reuse variables.""" - dtype = dtypes.as_dtype(dtype) - shape_object = tensor_shape.as_shape(shape) - with ops.init_scope(): - if initializer is None: - initializer, initializing_from_value = ( - variable_scope._get_default_variable_store()._get_default_initializer( # pylint: disable=protected-access - name=name, shape=shape_object, dtype=dtype)) - else: - initializing_from_value = not callable(initializer) - # Same logic as get_variable - variable_dtype = dtype.base_dtype - if initializing_from_value: - if shape is not None: - raise ValueError("If initializer is a constant, do not specify shape.") - initial_value = initializer - else: - # Instantiate initializer if provided initializer is a type object. - if isinstance(initializer, type(init_ops.Initializer)): - initializer = initializer(dtype=dtype) - def initial_value(): - return initializer( - shape_object.as_list(), dtype=dtype, partition_info=partition_info) - return resource_variable_ops.ResourceVariable( - initial_value=initial_value, - name=name, - dtype=variable_dtype, - **kwargs - ) - - -def add_variable(checkpointable, name, shape=None, dtype=dtypes.float32, - initializer=None): - """Add a variable to a Checkpointable with no scope influence.""" - return checkpointable._add_variable_with_custom_getter( # pylint: disable=protected-access - name=name, shape=shape, dtype=dtype, - initializer=initializer, getter=_default_getter) - - -def _breadth_first_checkpointable_traversal(root_checkpointable): - """Find shortest paths to all variables owned by dependencies of root.""" - bfs_sorted = [] - to_visit = collections.deque([root_checkpointable]) - path_to_root = {root_checkpointable: ()} - while to_visit: - current_checkpointable = to_visit.popleft() - current_checkpointable._maybe_initialize_checkpointable() # pylint: disable=protected-access - bfs_sorted.append(current_checkpointable) - for child_checkpointable in ( - current_checkpointable._checkpoint_dependencies): # pylint: disable=protected-access - if child_checkpointable.ref not in path_to_root: - path_to_root[child_checkpointable.ref] = ( - path_to_root[current_checkpointable] + (child_checkpointable,)) - to_visit.append(child_checkpointable.ref) - return bfs_sorted, path_to_root - - -def _escape_local_name(name): - # We need to support slashes in local names for compatibility, since this - # naming scheme is being patched in to things like Layer.add_variable where - # slashes were previously accepted. We also want to use slashes to indicate - # edges traversed to reach the variable, so we escape forward slashes in - # names. - return (name.replace(_ESCAPE_CHAR, _ESCAPE_CHAR + _ESCAPE_CHAR) - .replace(r"/", _ESCAPE_CHAR + "S")) - - -def _object_prefix_from_path(path_to_root): - return "/".join( - (_escape_local_name(checkpointable.name) - for checkpointable in path_to_root)) - - -def _slot_variable_naming_for_optimizer(optimizer_path): - """Make a function for naming slot variables in an optimizer.""" - # Name slot variables: - # - # /<_OPTIMIZER_SLOTS_NAME>// - # - # where is exactly the checkpoint name used for the original - # variable, including the path from the checkpoint root and the local name in - # the object which owns it. Note that we only save slot variables if the - # variable it's slotting for is also being saved. - - optimizer_identifier = "/%s/%s/" % (_OPTIMIZER_SLOTS_NAME, optimizer_path) - - def _name_slot_variable(variable_path, slot_name): - """With an optimizer specified, name a slot variable.""" - return (variable_path - + optimizer_identifier - + _escape_local_name(slot_name)) - - return _name_slot_variable - - -def _serialize_slot_variables(checkpointable_objects, node_ids, object_names): - """Gather and name slot variables.""" - non_slot_objects = list(checkpointable_objects) - slot_variables = {} - for checkpointable in non_slot_objects: - if isinstance(checkpointable, optimizer_lib.Optimizer): - naming_scheme = _slot_variable_naming_for_optimizer( - optimizer_path=object_names[checkpointable]) - slot_names = checkpointable.get_slot_names() - for slot_name in slot_names: - for original_variable_node_id, original_variable in enumerate( - non_slot_objects): - try: - slot_variable = checkpointable.get_slot( - original_variable, slot_name) - except AttributeError: - slot_variable = None - if slot_variable is None: - continue - slot_variable._maybe_initialize_checkpointable() # pylint: disable=protected-access - if slot_variable._checkpoint_dependencies: # pylint: disable=protected-access - # TODO(allenl): Gather dependencies of slot variables. - raise NotImplementedError( - "Currently only variables with no dependencies can be saved as " - "slot variables. File a feature request if this limitation " - "bothers you.") - if slot_variable in node_ids: - raise NotImplementedError( - "A slot variable was re-used as a dependency of a " - "Checkpointable object. This is not currently allowed. File a " - "feature request if this limitation bothers you.") - checkpoint_name = naming_scheme( - variable_path=object_names[original_variable], - slot_name=slot_name) - object_names[slot_variable] = checkpoint_name - slot_variable_node_id = len(checkpointable_objects) - node_ids[slot_variable] = slot_variable_node_id - checkpointable_objects.append(slot_variable) - slot_variable_proto = ( - checkpointable_object_graph_pb2.CheckpointableObjectGraph - .Object.SlotVariableReference( - slot_name=slot_name, - original_variable_node_id=original_variable_node_id, - slot_variable_node_id=slot_variable_node_id)) - slot_variables.setdefault(checkpointable, []).append( - slot_variable_proto) - return slot_variables - - -def _serialize_checkpointables( - checkpointable_objects, node_ids, object_names, slot_variables): - """Name non-slot `Checkpointable`s and add them to `object_graph_proto`.""" - object_graph_proto = ( - checkpointable_object_graph_pb2.CheckpointableObjectGraph()) - named_saveables = {} - - for checkpoint_id, checkpointable in enumerate(checkpointable_objects): - assert node_ids[checkpointable] == checkpoint_id - object_proto = object_graph_proto.nodes.add() - object_proto.slot_variables.extend(slot_variables.get(checkpointable, ())) - object_name = object_names[checkpointable] - for name, saveable_factory in ( - checkpointable._gather_saveables_for_checkpoint().items()): # pylint: disable=protected-access - attribute = object_proto.attributes.add() - attribute.name = name - attribute.checkpoint_key = "%s/%s/%s" % ( - object_name, _OBJECT_ATTRIBUTES_NAME, _escape_local_name(name)) - if callable(saveable_factory): - saveable = saveable_factory(name=attribute.checkpoint_key) - else: - saveable = saveable_factory - # Figure out the name-based Saver's name for this variable. - saver_dict = saver_lib.BaseSaverBuilder.OpListToDict( - [saveable], convert_variable_to_tensor=False) - attribute.full_name, = saver_dict.keys() - named_saveables[attribute.checkpoint_key] = saveable - - for child in checkpointable._checkpoint_dependencies: # pylint: disable=protected-access - child_proto = object_proto.children.add() - child_proto.node_id = node_ids[child.ref] - child_proto.local_name = child.name - - return named_saveables, object_graph_proto - - -def _serialize_object_graph(root_checkpointable): - """Determine checkpoint keys for variables and build a serialized graph. - - Non-slot variables are keyed based on a shortest path from the root saveable - to the object which owns the variable (i.e. the one which called - `Checkpointable._add_variable` to create it). - - Slot variables are keyed based on a shortest path to the variable being - slotted for, a shortest path to their optimizer, and the slot name. - - Args: - root_checkpointable: A `Checkpointable` object whose variables (including - the variables of dependencies, recursively) should be saved. - - Returns: - A tuple of (named_variables, object_graph_proto): - named_variables: A dictionary mapping names to variable objects. - object_graph_proto: A CheckpointableObjectGraph protocol buffer containing - the serialized object graph and variable references. - - Raises: - ValueError: If there are invalid characters in an optimizer's slot names. - """ - checkpointable_objects, path_to_root = ( - _breadth_first_checkpointable_traversal(root_checkpointable)) - object_names = { - obj: _object_prefix_from_path(path) - for obj, path in path_to_root.items()} - node_ids = {node: node_id for node_id, node - in enumerate(checkpointable_objects)} - slot_variables = _serialize_slot_variables( - checkpointable_objects=checkpointable_objects, - node_ids=node_ids, - object_names=object_names) - return _serialize_checkpointables( - checkpointable_objects=checkpointable_objects, - node_ids=node_ids, - object_names=object_names, - slot_variables=slot_variables) - - -def gather_initializers(root_checkpointable): - """Traverse the object graph and find initialization ops. - - Looks for `Checkpointable` objects which are dependencies of - `root_checkpointable` and which have an `initializer` property. Includes - initializers for slot variables only if the variable they are slotting for and - the optimizer are dependencies of `root_checkpointable` (i.e. if they would be - saved with a checkpoint). - - Args: - root_checkpointable: A `Checkpointable` object to gather initializers for. - Returns: - A list of initialization ops. - """ - # TODO(allenl): Extract out gathering logic so the naming logic doesn't have - # to run. - checkpointable_objects, path_to_root = ( - _breadth_first_checkpointable_traversal(root_checkpointable)) - object_names = { - obj: _object_prefix_from_path(path) - for obj, path in path_to_root.items()} - node_ids = {node: node_id for node_id, node - in enumerate(checkpointable_objects)} - _serialize_slot_variables( - checkpointable_objects=checkpointable_objects, - node_ids=node_ids, - object_names=object_names) - return [c.initializer for c in checkpointable_objects - if hasattr(c, "initializer") and c.initializer is not None] - - -class _NoRestoreSaveable(saver_lib.BaseSaverBuilder.SaveableObject): - - def __init__(self, tensor, name): - spec = saver_lib.BaseSaverBuilder.SaveSpec(tensor, "", name) - super(_NoRestoreSaveable, self).__init__(tensor, [spec], name) - - def restore(self, restored_tensors, restored_shapes): - return control_flow_ops.no_op() - - -class _LoadStatus(object): - """Abstract base for load status callbacks.""" - - @abc.abstractmethod - def assert_consumed(self): - """Raises an exception unless a non-trivial restoration has completed.""" - pass - - @abc.abstractmethod - def run_restore_ops(self, session=None): - """Runs restore ops from the checkpoint. Requires a valid checkpoint.""" - pass - - @abc.abstractmethod - def initialize_or_restore(self, session=None): - """Runs restore ops from the checkpoint, or initializes variables.""" - pass - - -class CheckpointLoadStatus(_LoadStatus): - """Checks the status of checkpoint loading and manages restore ops. - - Returned from `Saver.restore`. Since `restore` may defer the loading of values - in the checkpoint which don't yet have corresponding Python objects, - `CheckpointLoadStatus` provides a callback to verify that checkpoint loading - is complete (`assert_consumed`). - - When graph building, `restore` does not run restore ops itself since their - creation may be deferred. The `run_restore_ops` method must be called once all - Python objects with values to restore have been created and added to the - dependency graph (this does not necessarily have to be the whole checkpoint; - calling `run_restore_ops` while `assert_consumed` fails is supported and will - partially restore the checkpoint). - - See `Saver.restore` for usage examples. - """ - - def __init__(self, checkpoint, feed_dict): - self._checkpoint = checkpoint - self._feed_dict = feed_dict - - def assert_consumed(self): - """Asserts that all objects in the checkpoint have been created/matched. - - Returns: - `self` for chaining. - Raises: - AssertionError: If there are any Python objects in the dependency graph - which have not been restored from this checkpoint or a later `restore`, - or if there are any checkpointed values which have not been matched to - Python objects. - """ - for node_id, node in enumerate(self._checkpoint.object_graph_proto.nodes): - checkpointable = self._checkpoint.object_by_proto_id.get(node_id, None) - if checkpointable is None: - raise AssertionError("Unresolved object in checkpoint: %s" % (node,)) - if checkpointable._update_uid < self._checkpoint.restore_uid: # pylint: disable=protected-access - raise AssertionError( - "Object not assigned a value from checkpoint: %s" % (node,)) - if self._checkpoint.slot_restorations: - # Sanity check; this collection should be clear if everything has been - # restored. - raise AssertionError("Unresolved slot restorations: %s" % ( - self._checkpoint.slot_restorations,)) - if self._checkpoint.unused_attributes: - raise AssertionError( - ("Unused attributes in these objects (the attributes exist in the " - "checkpoint but not in the objects): %s") % ( - self._checkpoint.unused_attributes.items(),)) - return self - - def run_restore_ops(self, session=None): - """Run operations to restore objects in the dependency graph.""" - if context.executing_eagerly(): - return # Run eagerly - if session is None: - session = ops.get_default_session() - session.run(self._checkpoint.restore_ops, feed_dict=self._feed_dict) - - def initialize_or_restore(self, session=None): - """Alias for `run_restore_ops`. - - This method has a sibling in `InitializationOnlyStatus` which instead - initializes variables. That type is returned if no checkpoint is specified - in `Saver.restore`. - - Args: - session: The session to run restore ops in. If `None`, uses the default - session. - """ - self.run_restore_ops(session=session) - - -class InitializationOnlyStatus(_LoadStatus): - """Returned from `Saver.restore` when no checkpoint has been specified. - - Objects of this type have the same `assert_consumed` method as - `CheckpointLoadStatus`, but it always fails. However, - `initialize_or_restore` works on objects of both types, and will - initialize variables in `InitializationOnlyStatus` objects or restore them - otherwise. - """ - - def __init__(self, root_checkpointable): - self._root_checkpointable = root_checkpointable - - def assert_consumed(self): - """Assertion for consistency with `CheckpointLoadStatus`. Always fails.""" - raise AssertionError( - "No checkpoint specified (save_path=None); nothing is being restored.") - - def run_restore_ops(self, session=None): - """For consistency with `CheckpointLoadStatus`. - - Use `initialize_or_restore` for initializing if no checkpoint was passed - to `Saver.restore` and restoring otherwise. - - Args: - session: Not used. - """ - raise AssertionError( - "No checkpoint specified, so no restore ops are available " - "(save_path=None to Saver.restore).") - - def initialize_or_restore(self, session=None): - """Runs initialization ops for variables. - - Only objects which would be saved by `Saver.save` will be initialized. See - `gather_initializers` for details. - - This method does nothing when executing eagerly (initializers get run - eagerly). - - Args: - session: The session to run initialization ops in. If `None`, uses the - default session. - """ - if context.executing_eagerly(): - return # run eagerly - if session is None: - session = ops.get_default_session() - session.run(gather_initializers(self._root_checkpointable)) - - -_DEPRECATED_RESTORE_INSTRUCTIONS = ( - "Restoring a name-based tf.train.Saver checkpoint using the object-based " - "restore API. This mode uses global names to match variables, and so is " - "somewhat fragile. It also adds new restore ops to the graph each time it " - "is called. Prefer re-encoding training checkpoints in the object-based " - "format: run save() on the object-based saver (the same one this message " - "is coming from) and use that checkpoint in the future.") - - -class NameBasedSaverStatus(_LoadStatus): - """Status for loading a name-based training checkpoint.""" - - def __init__(self, object_saver, save_path): - self._object_saver = object_saver - self._save_path = save_path - - def assert_consumed(self): - """Assertion for consistency with `CheckpointLoadStatus`. Always fails.""" - raise AssertionError( - "Restoring a name-based checkpoint. No load status is available.") - - @deprecation.deprecated( - date=None, instructions=_DEPRECATED_RESTORE_INSTRUCTIONS) - def run_restore_ops(self, session=None): - """Load the name-based training checkpoint using a new `tf.train.Saver`.""" - if session is None and not context.executing_eagerly(): - session = ops.get_default_session() - with ops.device("/cpu:0"): - saver_lib.Saver(self._object_saver._global_variable_names()).restore( # pylint: disable=protected-access - sess=session, save_path=self._save_path) - - def initialize_or_restore(self, session=None): - """Alias for `run_restore_ops`.""" - self.run_restore_ops(session=session) - - -class _SessionWithFeedDictAdditions(session_lib.SessionInterface): - """Pretends to be a session, inserts extra feeds on run().""" - - def __init__(self, session, feed_additions): - self._wrapped_session = session - self._feed_additions = feed_additions - - def run(self, fetches, feed_dict=None, **kwargs): - if feed_dict is None: - feed_dict = {} - else: - feed_dict = feed_dict.copy() - feed_dict.update(self._feed_additions) - return self._wrapped_session.run( - fetches=fetches, feed_dict=feed_dict, **kwargs) - - -def _copy_saver_with_new_var_list(old_saver, new_var_list): - """Copy a `tf.train.Saver`'s state to a new Saver with different variables.""" - new_saver = saver_lib.Saver(var_list=new_var_list) - # TODO(allenl): Move to copying functionality to Saver? - # pylint: disable=protected-access - new_saver._last_checkpoints = old_saver._last_checkpoints - new_saver._checkpoints_to_be_deleted = old_saver._checkpoints_to_be_deleted - new_saver._next_checkpoint_time = old_saver._next_checkpoint_time - # pylint: enable=protected-access - return new_saver - - -class CheckpointableSaver(object): - """Saves and restores a `Checkpointable` object and its dependencies. - - See `Checkpointable` for details of dependency management. `Saver` wraps - `tf.train.Saver` for saving, including extra information about the graph of - dependencies between Python objects. When restoring, it uses this information - about the save-time dependency graph to more robustly match objects with their - checkpointed values. When executing eagerly, it supports restoring variables - on object creation (see `Saver.restore`). - - Values in a checkpoint are mapped to `Checkpointable` Python objects - (`Variable`s, `Optimizer`s, `Layer`s) based on the names provided when the - checkpoint was written. To avoid breaking existing checkpoints when modifying - a class, dependency names (the names of attributes to which `Checkpointable` - objects are assigned) may not change. These names are local to objects, in - contrast to the `Variable.name`-based save/restore from `tf.train.Saver`, and - so allow additional program transformations. - """ - - def __init__(self, root_checkpointable): - """Configure saving. - - Args: - root_checkpointable: The root of the object graph to save/restore. This - object and all of its dependencies are saved in the checkpoint. When - restoring, objects are matched and restored starting from this root. - """ - # Allow passing in a weak reference to avoid reference cycles when - # `Checkpointable` objects save themselves. - self._root_checkpointable_ref = root_checkpointable - if not context.executing_eagerly(): - with ops.device("/cpu:0"): - self._file_prefix_placeholder = constant_op.constant("model") - else: - self._file_prefix_placeholder = None - - # Op caching for save - self._object_graph_feed_tensor = None - self._last_save_object_graph = None - self._last_save_saver = None - - # Op caching for restore - self._last_restore_object_graph = None - self._last_restore_checkpoint = None - - @property - def _root_checkpointable(self): - if isinstance(self._root_checkpointable_ref, weakref.ref): - derefed = self._root_checkpointable_ref() - assert derefed is not None - return derefed - else: - return self._root_checkpointable_ref - - def save(self, file_prefix, checkpoint_number=None, session=None): - """Save a training checkpoint. - - The saved checkpoint includes variables created by this object and any - Checkpointable objects it depends on at the time `Saver.save()` is called. - - Args: - file_prefix: A prefix to use for the checkpoint filenames - (/path/to/directory/and_a_prefix). Names are generated based on this - prefix and `checkpoint_number`, if provided. - checkpoint_number: An integer variable or Tensor, used to number - checkpoints. Typically this value is saved along with other variables in - training checkpoints, which will happen automatically if it was created - by `root_checkpointable` or one of its dependencies (via - `Checkpointable._add_variable`). - session: The session to evaluate variables in. Ignored when executing - eagerly. If not provided when graph building, the default session is - used. - - Returns: - The full path to the checkpoint. - """ - named_variables, graph_proto = _serialize_object_graph( - self._root_checkpointable) - if not context.executing_eagerly(): - if session is None: - session = ops.get_default_session() - if self._object_graph_feed_tensor is None: - with ops.device("/cpu:0"): - self._object_graph_feed_tensor = constant_op.constant( - "", dtype=dtypes.string) - object_graph_tensor = self._object_graph_feed_tensor - feed_additions = {object_graph_tensor: graph_proto.SerializeToString()} - else: - session = None - with ops.device("/cpu:0"): - object_graph_tensor = constant_op.constant( - graph_proto.SerializeToString(), dtype=dtypes.string) - feed_additions = None - assert _OBJECT_GRAPH_PROTO_KEY not in named_variables - named_variables[_OBJECT_GRAPH_PROTO_KEY] = _NoRestoreSaveable( - tensor=object_graph_tensor, - name=_OBJECT_GRAPH_PROTO_KEY) - if (self._last_save_object_graph != graph_proto - # When executing eagerly, we need to re-create SaveableObjects each time - # save() is called so they pick up new Tensors passed to their - # constructors. That means the Saver needs to be copied with a new - # var_list. - or context.executing_eagerly()): - if self._last_save_object_graph is not None: - self._last_save_saver = _copy_saver_with_new_var_list( - old_saver=self._last_save_saver, new_var_list=named_variables) - else: - self._last_save_saver = saver_lib.Saver(var_list=named_variables) - self._last_save_object_graph = graph_proto - with ops.device("/cpu:0"): - save_path = self._last_save_saver.save( - sess=_SessionWithFeedDictAdditions( - session=session, feed_additions=feed_additions), - save_path=file_prefix, - write_meta_graph=False, - global_step=checkpoint_number) - return save_path - - def _global_variable_names(self): - """Generate a `tf.train.Saver`-style `var_list` using `variable.name`s.""" - named_saveables, graph_proto = _serialize_object_graph( - self._root_checkpointable) - saver_names = {} - for object_proto in graph_proto.nodes: - for attribute_proto in object_proto.attributes: - saver_names[attribute_proto.full_name] = named_saveables[ - attribute_proto.checkpoint_key] - return saver_names - - def restore(self, save_path): - """Restore a training checkpoint. - - Restores `root_checkpointable` and any objects that it tracks - (transitive). Either assigns values immediately if variables to restore have - been created already, or defers restoration until the variables are - created. Dependencies added to the `root_checkpointable` passed to the - constructor after this call will be matched if they have a corresponding - object in the checkpoint. - - When building a graph, restorations are added to the graph but not run. - - To disallow deferred loading, assert immediately that all checkpointed - variables have been matched to variable objects: - - ```python - saver = Saver(root) - saver.restore(path).assert_consumed() - ``` - - An exception will be raised unless every object was matched and its - variables already exist. - - When graph building, `assert_consumed()` indicates that all of the restore - ops which will be created for this checkpoint have been created. They can be - run via the `run_restore_ops()` function of the status object: - - ```python - saver.restore(path).assert_consumed().run_restore_ops() - ``` - - If the checkpoint has not been consumed completely, then the list of restore - ops will grow as more objects are added to the dependency graph. - - Name-based `tf.train.Saver` checkpoints can be loaded using this - method. There is no deferred loading, and names are used to match - variables. No restore ops are created/run until `run_restore_ops()` or - `initialize_or_restore()` are called on the returned status object, even - when executing eagerly. Re-encode name-based checkpoints using this - object-based `Saver.save` as soon as possible. - - Args: - save_path: The path to the checkpoint, as returned by `save` or - `tf.train.latest_checkpoint`. If None (as when there is no latest - checkpoint for `tf.train.latest_checkpoint` to return), returns an - object which may run initializers for objects in the dependency - graph. If the checkpoint was written by the name-based `tf.train.Saver`, - names are used to match variables. - - Returns: - A load status object, which can be used to make assertions about the - status of checkpoint restoration and run initialization/restore ops - (of type `CheckpointLoadStatus`, or `InitializationOnlyStatus` if - `save_path` is `None`). - - If `save_path` points to a name-based checkpoint, a `NameBasedSaverStatus` - object is returned which runs restore ops from a name-based saver. - """ - if save_path is None: - return InitializationOnlyStatus(self._root_checkpointable) - in_graph_mode = not context.executing_eagerly() - if in_graph_mode: - file_prefix_tensor = self._file_prefix_placeholder - file_prefix_feed_dict = {self._file_prefix_placeholder: save_path} - else: - with ops.device("/cpu:0"): - file_prefix_tensor = constant_op.constant(save_path) - file_prefix_feed_dict = None - reader = pywrap_tensorflow.NewCheckpointReader(save_path) - try: - object_graph_string = reader.get_tensor(_OBJECT_GRAPH_PROTO_KEY) - except errors_impl.NotFoundError: - # The object graph proto does not exist in this checkpoint. Try again with - # name-based saving. - return NameBasedSaverStatus(self, save_path) - - object_graph_proto = ( - checkpointable_object_graph_pb2.CheckpointableObjectGraph()) - object_graph_proto.ParseFromString(object_graph_string) - if in_graph_mode and object_graph_proto == self._last_restore_object_graph: - checkpoint = self._last_restore_checkpoint - else: - if in_graph_mode: - dtype_map = None - else: - dtype_map = reader.get_variable_to_dtype_map() - checkpoint = core_checkpointable_utils._Checkpoint( # pylint: disable=protected-access - object_graph_proto=object_graph_proto, - save_path=file_prefix_tensor, - dtype_map=dtype_map) - if in_graph_mode: - if self._last_restore_object_graph is not None: - raise NotImplementedError( - "Using a single Saver to restore different object graphs is not " - "currently supported when graph building. Use a different Saver " - "for each object graph (restore ops will be duplicated), or " - "file a feature request if this limitation bothers you.") - self._last_restore_checkpoint = checkpoint - self._last_restore_object_graph = object_graph_proto - core_checkpointable._CheckpointPosition( # pylint: disable=protected-access - checkpoint=checkpoint, proto_id=0).restore(self._root_checkpointable) - load_status = CheckpointLoadStatus( - checkpoint, feed_dict=file_prefix_feed_dict) - return load_status - - -class Checkpoint(core_checkpointable.Checkpointable): - """A utility class which groups `Checkpointable` objects. - - Accepts arbitrary keyword arguments to its constructor and saves those values - with a checkpoint. Maintains a `save_counter` for numbering checkpoints. - - Example usage: - - ```python - import tensorflow as tf - import tensorflow.contrib.eager as tfe - import os - - checkpoint_directory = "/tmp/training_checkpoints" - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - - root = tfe.Checkpoint(optimizer=optimizer, model=model) - root.restore(tf.train.latest_checkpoint(checkpoint_directory)) - for _ in range(num_training_steps): - optimizer.minimize( ... ) - root.save(file_prefix=checkpoint_prefix) - ``` - - For more manual control over saving, use `tfe.CheckpointableSaver` directly. - - Attributes: - save_counter: Incremented when `save()` is called. Used to number - checkpoints. - """ - - def __init__(self, **kwargs): - """Group objects into a training checkpoint. - - Args: - **kwargs: Keyword arguments are set as attributes of this object, and are - saved with the checkpoint. Attribute values must derive from - `CheckpointableBase`. - Raises: - ValueError: If objects in `kwargs` are not Checkpointable. - """ - super(Checkpoint, self).__init__() - for k, v in sorted(kwargs.items(), key=lambda item: item[0]): - if not isinstance(v, core_checkpointable.CheckpointableBase): - raise ValueError( - ("`Checkpoint` was expecting an object derived from " - "`CheckpointableBase`, got %s.") % (v,)) - setattr(self, k, v) - self._save_counter = None # Created lazily for restore-on-create. - self._saver = CheckpointableSaver(weakref.ref(self)) - - def _maybe_create_save_counter(self): - """Create a save counter if it does not yet exist.""" - if self._save_counter is None: - # Initialized to 0 and incremented before saving. - with ops.device("/cpu:0"): - self._save_counter = add_variable( - self, name="save_counter", initializer=0, dtype=dtypes.int64) - - @property - def save_counter(self): - """An integer variable which starts at zero and is incremented on save. - - Used to number checkpoints. - - Returns: - The save counter variable. - """ - self._maybe_create_save_counter() - return self._save_counter - - def save(self, file_prefix, session=None): - """Save a checkpoint. Wraps `tfe.CheckpointableSaver.save`.""" - in_graph_mode = not context.executing_eagerly() - if in_graph_mode: - if session is None: - session = ops.get_default_session() - if self._save_counter is None: - # When graph building, if this is a new save counter variable then it - # needs to be initialized before assign_add. This is only an issue if - # restore() has not been called first. - session.run(self.save_counter.initializer) - with ops.colocate_with(self.save_counter): - assign_op = self.save_counter.assign_add(1) - if in_graph_mode: - session.run(assign_op) - return self._saver.save( - file_prefix=file_prefix, - checkpoint_number=self.save_counter, - session=session) - - def restore(self, save_path): - """Restore a checkpoint. Wraps `tfe.CheckpointableSaver.restore`.""" - status = self._saver.restore(save_path=save_path) - # Create the save counter now so it gets initialized with other variables - # when graph building. Creating it earlier would lead to double - # initialization when executing eagerly. - self._maybe_create_save_counter() - return status class _CallbackSaveable(saver_lib.BaseSaverBuilder.SaveableObject): diff --git a/tensorflow/contrib/eager/python/checkpointable_utils_test.py b/tensorflow/contrib/eager/python/checkpointable_utils_test.py index b344d50e7f..da04199aaa 100644 --- a/tensorflow/contrib/eager/python/checkpointable_utils_test.py +++ b/tensorflow/contrib/eager/python/checkpointable_utils_test.py @@ -16,59 +16,15 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import functools import os -import six - -from tensorflow.contrib.eager.python import checkpointable_utils -from tensorflow.python.client import session as session_lib -from tensorflow.python.eager import backprop -from tensorflow.python.eager import context -from tensorflow.python.eager import function +from tensorflow.contrib.eager.python import checkpointable_utils as contrib_checkpointable_utils from tensorflow.python.eager import test -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops from tensorflow.python.framework import test_util -from tensorflow.python.keras._impl.keras.engine import sequential -from tensorflow.python.keras._impl.keras.engine import training -from tensorflow.python.keras._impl.keras.layers import core from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import init_ops from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import template -from tensorflow.python.ops import variable_scope -from tensorflow.python.training import adam from tensorflow.python.training import checkpointable -from tensorflow.python.training import saver as core_saver -from tensorflow.python.training import training_util - - -class NonLayerCheckpointable(checkpointable.Checkpointable): - - def __init__(self): - super(NonLayerCheckpointable, self).__init__() - self.a_variable = checkpointable_utils.add_variable( - self, name="a_variable", shape=[]) - - -# pylint: disable=not-callable -class MyModel(training.Model): - """A concrete Model for testing.""" - - def __init__(self): - super(MyModel, self).__init__() - self._named_dense = core.Dense(1, use_bias=True) - self._second = core.Dense(1, use_bias=False) - # We can still track Checkpointables which aren't Layers. - self._non_layer = NonLayerCheckpointable() - - def call(self, values): - ret = self._second(self._named_dense(values)) - return ret +from tensorflow.python.training import checkpointable_utils def _split_variable_closure(variable): @@ -91,7 +47,7 @@ class SaveTensorSlicesAsDeps(checkpointable.CheckpointableBase): def __init__(self): self.combined = resource_variable_ops.ResourceVariable([0., 0., 0., 0.]) - split_dependencies = checkpointable_utils.split_dependency( + split_dependencies = contrib_checkpointable_utils.split_dependency( component_names=("first_half", "second_half"), component_dtypes=(self.combined.dtype,) * 2, fill_save_buffer_fn=_split_variable_closure( @@ -152,1239 +108,5 @@ class SplitTests(test.TestCase): self.evaluate(restore_checkpoint.dep.combined)) -class InterfaceTests(test.TestCase): - - @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) - def testAddVariable(self): - obj = NonLayerCheckpointable() - with self.assertRaisesRegexp(ValueError, "do not specify shape"): - checkpointable_utils.add_variable( - obj, name="shape_specified_twice", shape=[], initializer=1) - constant_initializer = checkpointable_utils.add_variable( - obj, name="constant_initializer", initializer=1) - with variable_scope.variable_scope("some_variable_scope"): - ones_initializer = checkpointable_utils.add_variable( - obj, - name="ones_initializer", - shape=[2], - initializer=init_ops.ones_initializer(dtype=dtypes.float32)) - bare_initializer = checkpointable_utils.add_variable( - obj, - name="bare_initializer", - shape=[2, 2], - dtype=dtypes.float64, - initializer=init_ops.zeros_initializer) - - # Even in graph mode, there are no naming conflicts between objects, only - # naming conflicts within an object. - other_duplicate = resource_variable_ops.ResourceVariable( - name="duplicate", initial_value=1.) - duplicate = checkpointable_utils.add_variable( - obj, name="duplicate", shape=[]) - with self.assertRaisesRegexp(ValueError, "'duplicate' already exists"): - checkpointable_utils.add_variable(obj, name="duplicate", shape=[]) - - self.evaluate(checkpointable_utils.gather_initializers(obj)) - self.assertEqual("constant_initializer:0", constant_initializer.name) - self.assertEqual(1, self.evaluate(constant_initializer)) - self.assertEqual("some_variable_scope/ones_initializer:0", - ones_initializer.name) - self.assertAllEqual([1, 1], self.evaluate(ones_initializer)) - self.assertAllEqual([[0., 0.], - [0., 0.]], self.evaluate(bare_initializer)) - self.assertEqual("a_variable:0", obj.a_variable.name) - self.assertEqual("duplicate:0", other_duplicate.name) - if context.executing_eagerly(): - # When executing eagerly, there's no uniquification of variable names. The - # checkpoint name will be the same. - self.assertEqual("duplicate:0", duplicate.name) - else: - # The .name attribute may be globally influenced, but the checkpoint name - # won't be (tested below). - self.assertEqual("duplicate_1:0", duplicate.name) - named_variables, _ = checkpointable_utils._serialize_object_graph(obj) - expected_checkpoint_names = ( - "a_variable/.ATTRIBUTES/VARIABLE_VALUE", - "bare_initializer/.ATTRIBUTES/VARIABLE_VALUE", - "constant_initializer/.ATTRIBUTES/VARIABLE_VALUE", - "duplicate/.ATTRIBUTES/VARIABLE_VALUE", - "ones_initializer/.ATTRIBUTES/VARIABLE_VALUE", - ) - six.assertCountEqual( - self, expected_checkpoint_names, named_variables.keys()) - - def testInitNotCalled(self): - - class NoInit(checkpointable.Checkpointable): - - def __init__(self): - pass - - # __init__ for Checkpointable will be called implicitly. - checkpointable_utils.add_variable(NoInit(), "var", shape=[]) - - def testShapeDtype(self): - root = checkpointable.Checkpointable() - v1 = checkpointable_utils.add_variable( - root, name="v1", initializer=3., dtype=dtypes.float64) - self.assertEqual(dtypes.float64, v1.dtype) - v2 = checkpointable_utils.add_variable( - root, - name="v2", - shape=[3], - initializer=init_ops.ones_initializer, - dtype=dtypes.float64) - self.assertEqual(dtypes.float64, v2.dtype) - self.assertAllEqual([1., 1., 1.], self.evaluate(v2)) - - -class _MirroringSaveable(core_saver.BaseSaverBuilder.SaveableObject): - - def __init__(self, primary_variable, mirrored_variable, name): - self._primary_variable = primary_variable - self._mirrored_variable = mirrored_variable - tensor = self._primary_variable.read_value() - spec = core_saver.BaseSaverBuilder.SaveSpec( - tensor=tensor, - slice_spec="", - name=name) - super(_MirroringSaveable, self).__init__( - tensor, [spec], name) - - def restore(self, restored_tensors, restored_shapes): - """Restore the same value into both variables.""" - tensor, = restored_tensors - return control_flow_ops.group( - self._primary_variable.assign(tensor), - self._mirrored_variable.assign(tensor)) - - -class _OwnsMirroredVariables(checkpointable.CheckpointableBase): - """A Checkpointable object which returns a more complex SaveableObject.""" - - def __init__(self): - self.non_dep_variable = variable_scope.get_variable( - name="non_dep_variable", initializer=6., use_resource=True) - self.mirrored = variable_scope.get_variable( - name="mirrored", initializer=15., use_resource=True) - - def _gather_saveables_for_checkpoint(self): - def _saveable_factory(name=self.non_dep_variable.name): - return _MirroringSaveable( - primary_variable=self.non_dep_variable, - mirrored_variable=self.mirrored, - name=name) - return {checkpointable.VARIABLE_VALUE_KEY: _saveable_factory} - - # The Saver sorts by name before parsing, so we need a name property. - @property - def name(self): - return self.non_dep_variable.name - - -class CheckpointingTests(test.TestCase): - - @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) - def testNamingWithOptimizer(self): - input_value = constant_op.constant([[3.]]) - model = MyModel() - # A nuisance Model using the same optimizer. Its slot variables should not - # go in the checkpoint, since it is never depended on. - other_model = MyModel() - optimizer = adam.AdamOptimizer(0.001) - optimizer_step = training_util.get_or_create_global_step() - root_checkpointable = checkpointable_utils.Checkpoint( - optimizer=optimizer, model=model, optimizer_step=optimizer_step) - if context.executing_eagerly(): - optimizer.minimize( - lambda: model(input_value), - global_step=optimizer_step) - optimizer.minimize( - lambda: other_model(input_value), - global_step=optimizer_step) - else: - train_op = optimizer.minimize( - model(input_value), global_step=optimizer_step) - optimizer.minimize( - other_model(input_value), - global_step=optimizer_step) - self.evaluate(checkpointable_utils.gather_initializers( - root_checkpointable)) - self.evaluate(train_op) - named_variables, serialized_graph = ( - checkpointable_utils._serialize_object_graph(root_checkpointable)) - expected_checkpoint_names = ( - # Created in the root node, so no prefix. - "optimizer_step", - "model/_second/kernel", - "model/_named_dense/kernel", - "model/_named_dense/bias", - # non-Layer dependency of the model - "model/_non_layer/a_variable", - # The optimizer creates two non-slot variables - "optimizer/beta1_power", - "optimizer/beta2_power", - # Slot variables - "model/_second/kernel/.OPTIMIZER_SLOT/optimizer/m", - "model/_second/kernel/.OPTIMIZER_SLOT/optimizer/v", - "model/_named_dense/kernel/.OPTIMIZER_SLOT/optimizer/m", - "model/_named_dense/kernel/.OPTIMIZER_SLOT/optimizer/v", - "model/_named_dense/bias/.OPTIMIZER_SLOT/optimizer/m", - "model/_named_dense/bias/.OPTIMIZER_SLOT/optimizer/v", - ) - suffix = "/.ATTRIBUTES/VARIABLE_VALUE" - expected_checkpoint_names = [ - name + suffix for name in expected_checkpoint_names] - six.assertCountEqual(self, expected_checkpoint_names, - named_variables.keys()) - # Check that we've mapped to the right variable objects (not exhaustive) - self.assertEqual( - "global_step:0", - named_variables["optimizer_step" + suffix].name) - self.assertEqual( - "my_model/dense_1/kernel:0", - named_variables["model/_second/kernel" + suffix].name) - self.assertEqual( - "my_model/dense/kernel:0", - named_variables["model/_named_dense/kernel" + suffix].name) - self.assertEqual( - "beta1_power:0", - named_variables["optimizer/beta1_power" + suffix].name) - self.assertEqual( - "beta2_power:0", - named_variables["optimizer/beta2_power" + suffix].name) - # Spot check the generated protocol buffers. - self.assertEqual("optimizer", - serialized_graph.nodes[0].children[1].local_name) - optimizer_node = serialized_graph.nodes[serialized_graph.nodes[0].children[ - 1].node_id] - self.assertEqual("beta1_power", - optimizer_node.children[0].local_name) - self.assertEqual("beta1_power", - serialized_graph.nodes[optimizer_node.children[0].node_id] - .attributes[0].full_name) - self.assertEqual( - "my_model/dense/kernel", - serialized_graph.nodes[optimizer_node.slot_variables[0] - .original_variable_node_id] - .attributes[0].full_name) - # We strip off the :0 suffix, as variable.name-based saving does. - self.assertEqual( - "my_model/dense/kernel/Adam", - serialized_graph.nodes[optimizer_node.slot_variables[0] - .slot_variable_node_id] - .attributes[0].full_name) - self.assertEqual( - "my_model/dense/kernel/Adam:0", - optimizer.get_slot( - var=named_variables["model/_named_dense/kernel" + suffix], - name="m").name) - self.assertEqual( - "model/_named_dense/kernel" + suffix, - serialized_graph.nodes[ - optimizer_node.slot_variables[0] - .original_variable_node_id].attributes[0].checkpoint_key) - self.assertEqual("m", optimizer_node.slot_variables[0].slot_name) - self.assertEqual( - "model/_named_dense/kernel/.OPTIMIZER_SLOT/optimizer/m" + suffix, - serialized_graph.nodes[ - optimizer_node.slot_variables[0] - .slot_variable_node_id].attributes[0].checkpoint_key) - - @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) - def testMoreComplexSaveableReturned(self): - v = _OwnsMirroredVariables() - checkpoint = checkpointable_utils.Checkpoint(v=v) - test_dir = self.get_temp_dir() - prefix = os.path.join(test_dir, "ckpt") - self.evaluate(v.non_dep_variable.assign(42.)) - save_path = checkpoint.save(prefix) - self.evaluate(v.non_dep_variable.assign(43.)) - self.evaluate(v.mirrored.assign(44.)) - checkpoint.restore(save_path).assert_consumed().initialize_or_restore() - self.assertEqual(42., self.evaluate(v.non_dep_variable)) - self.assertEqual(42., self.evaluate(v.mirrored)) - self.evaluate(v.non_dep_variable.assign(44.)) - save_path = checkpoint.save(prefix) - self.evaluate(v.non_dep_variable.assign(45.)) - checkpoint.restore(save_path).assert_consumed().initialize_or_restore() - self.assertEqual(44., self.evaluate(v.non_dep_variable)) - self.assertEqual(44., self.evaluate(v.mirrored)) - - @test_util.run_in_graph_and_eager_modes() - def testMoreComplexSaveableReturnedWithGlobalName(self): - # The same object can also be saved using the name-based saver. - v = _OwnsMirroredVariables() - saver = core_saver.Saver(var_list=[v]) - test_dir = self.get_temp_dir() - prefix = os.path.join(test_dir, "ckpt") - self.evaluate(v.non_dep_variable.assign(42.)) - with self.test_session() as sess: - save_path = saver.save(sess, prefix) - self.evaluate(v.non_dep_variable.assign(43.)) - self.evaluate(v.mirrored.assign(44.)) - saver.restore(sess, save_path) - self.assertEqual(42., self.evaluate(v.non_dep_variable)) - self.assertEqual(42., self.evaluate(v.mirrored)) - - @test_util.run_in_graph_and_eager_modes() - def testSaveRestore(self): - model = MyModel() - optimizer = adam.AdamOptimizer(0.001) - root_checkpointable = checkpointable_utils.Checkpoint( - optimizer=optimizer, model=model) - input_value = constant_op.constant([[3.]]) - if context.executing_eagerly(): - optimizer.minimize( - lambda: model(input_value)) - else: - train_op = optimizer.minimize(model(input_value)) - # TODO(allenl): Make initialization more pleasant when graph building. - root_checkpointable.save_counter # pylint: disable=pointless-statement - self.evaluate(checkpointable_utils.gather_initializers( - root_checkpointable)) - self.evaluate(train_op) - prefix = os.path.join(self.get_temp_dir(), "ckpt") - self.evaluate(state_ops.assign(model._named_dense.variables[1], [42.])) - m_bias_slot = optimizer.get_slot(model._named_dense.variables[1], "m") - self.evaluate(state_ops.assign(m_bias_slot, [1.5])) - save_path = root_checkpointable.save(file_prefix=prefix) - self.evaluate(state_ops.assign(model._named_dense.variables[1], [43.])) - self.evaluate(state_ops.assign(root_checkpointable.save_counter, 3)) - optimizer_variables = self.evaluate(optimizer.variables()) - self.evaluate(state_ops.assign(m_bias_slot, [-2.])) - # Immediate restoration - status = root_checkpointable.restore(save_path=save_path).assert_consumed() - status.run_restore_ops() - self.assertAllEqual([42.], self.evaluate(model._named_dense.variables[1])) - self.assertAllEqual(1, self.evaluate(root_checkpointable.save_counter)) - self.assertAllEqual([1.5], self.evaluate(m_bias_slot)) - if not context.executing_eagerly(): - return # Restore-on-create is only supported when executing eagerly - on_create_model = MyModel() - on_create_optimizer = adam.AdamOptimizer( - 0.001, - # Preserve beta1_power and beta2_power when appying gradients so we can - # test that they've been restored correctly. - beta1=1.0, beta2=1.0) - on_create_root = checkpointable_utils.Checkpoint( - optimizer=on_create_optimizer, model=on_create_model) - # Deferred restoration - status = on_create_root.restore(save_path=save_path) - on_create_model(constant_op.constant([[3.]])) # create variables - self.assertAllEqual(1, self.evaluate(on_create_root.save_counter)) - self.assertAllEqual([42.], - self.evaluate( - on_create_model._named_dense.variables[1])) - on_create_m_bias_slot = on_create_optimizer.get_slot( - on_create_model._named_dense.variables[1], "m") - # Optimizer slot variables are created when the original variable is - # restored. - self.assertAllEqual([1.5], self.evaluate(on_create_m_bias_slot)) - self.assertAllEqual(optimizer_variables[2:], - self.evaluate(on_create_optimizer.variables())) - dummy_var = resource_variable_ops.ResourceVariable([1.]) - on_create_optimizer.minimize(loss=dummy_var.read_value) - status.assert_consumed() - beta1_power, beta2_power = on_create_optimizer._get_beta_accumulators() - self.assertAllEqual(optimizer_variables[0], self.evaluate(beta1_power)) - self.assertAllEqual(optimizer_variables[1], self.evaluate(beta2_power)) - - # TODO(allenl): Debug garbage created by this test in python3. - def testDeferredRestorationUsageEager(self): - """An idiomatic eager execution example.""" - num_training_steps = 10 - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - for training_continuation in range(3): - model = MyModel() - optimizer = adam.AdamOptimizer(0.001) - root = checkpointable_utils.Checkpoint( - optimizer=optimizer, model=model, - optimizer_step=training_util.get_or_create_global_step()) - root.restore(core_saver.latest_checkpoint(checkpoint_directory)) - for _ in range(num_training_steps): - # TODO(allenl): Use a Dataset and serialize/checkpoint it. - input_value = constant_op.constant([[3.]]) - optimizer.minimize( - lambda: model(input_value), # pylint: disable=cell-var-from-loop - global_step=root.optimizer_step) - root.save(file_prefix=checkpoint_prefix) - self.assertEqual((training_continuation + 1) * num_training_steps, - root.optimizer_step.numpy()) - - def testUsageGraph(self): - """Expected usage when graph building.""" - with context.graph_mode(): - num_training_steps = 10 - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - for training_continuation in range(3): - with ops.Graph().as_default(): - model = MyModel() - optimizer = adam.AdamOptimizer(0.001) - root = checkpointable_utils.Checkpoint( - optimizer=optimizer, model=model, - global_step=training_util.get_or_create_global_step()) - input_value = constant_op.constant([[3.]]) - train_op = optimizer.minimize( - model(input_value), - global_step=root.global_step) - checkpoint_path = core_saver.latest_checkpoint(checkpoint_directory) - with self.test_session(graph=ops.get_default_graph()) as session: - status = root.restore(save_path=checkpoint_path) - status.initialize_or_restore(session=session) - if checkpoint_path is None: - self.assertEqual(0, training_continuation) - with self.assertRaises(AssertionError): - status.assert_consumed() - else: - status.assert_consumed() - for _ in range(num_training_steps): - session.run(train_op) - root.save(file_prefix=checkpoint_prefix, session=session) - self.assertEqual((training_continuation + 1) * num_training_steps, - session.run(root.global_step)) - self.assertEqual(training_continuation + 1, - session.run(root.save_counter)) - - @test_util.run_in_graph_and_eager_modes() - def testAgnosticUsage(self): - """Graph/eager agnostic usage.""" - # Does create garbage when executing eagerly due to ops.Graph() creation. - num_training_steps = 10 - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - for training_continuation in range(3): - with ops.Graph().as_default(), self.test_session( - graph=ops.get_default_graph()), test_util.device(use_gpu=True): - model = MyModel() - optimizer = adam.AdamOptimizer(0.001) - root = checkpointable_utils.Checkpoint( - optimizer=optimizer, model=model, - global_step=training_util.get_or_create_global_step()) - checkpoint_path = core_saver.latest_checkpoint(checkpoint_directory) - status = root.restore(save_path=checkpoint_path) - input_value = constant_op.constant([[3.]]) - train_fn = functools.partial( - optimizer.minimize, - functools.partial(model, input_value), - global_step=root.global_step) - if not context.executing_eagerly(): - train_fn = functools.partial(self.evaluate, train_fn()) - status.initialize_or_restore() - for _ in range(num_training_steps): - train_fn() - root.save(file_prefix=checkpoint_prefix) - self.assertEqual((training_continuation + 1) * num_training_steps, - self.evaluate(root.global_step)) - self.assertEqual(training_continuation + 1, - self.evaluate(root.save_counter)) - - # pylint: disable=cell-var-from-loop - @test_util.run_in_graph_and_eager_modes() - def testWithDefun(self): - num_training_steps = 2 - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - for training_continuation in range(3): - with ops.Graph().as_default(), self.test_session( - graph=ops.get_default_graph()), test_util.device(use_gpu=True): - model = MyModel() - # Don't actually train so we can test variable values - optimizer = adam.AdamOptimizer(0.) - root = checkpointable_utils.Checkpoint( - optimizer=optimizer, model=model, - global_step=training_util.get_or_create_global_step()) - checkpoint_path = core_saver.latest_checkpoint(checkpoint_directory) - status = root.restore(save_path=checkpoint_path) - def train_fn(): - @function.defun - def _call_model(x): - return model(x) - with backprop.GradientTape() as tape: - loss = _call_model(constant_op.constant([[3.]])) - gradients = tape.gradient(loss, model.variables) - return optimizer.apply_gradients(zip(gradients, model.variables), - global_step=root.global_step) - if not context.executing_eagerly(): - train_fn = functools.partial( - self.evaluate, train_fn()) - status.initialize_or_restore() - for _ in range(num_training_steps): - train_fn() - if training_continuation > 0: - status.assert_consumed() - self.assertAllClose([[42.]], self.evaluate(model.variables[0])) - else: - self.evaluate(model.variables[0].assign([[42.]])) - root.save(file_prefix=checkpoint_prefix) - self.assertEqual((training_continuation + 1) * num_training_steps, - self.evaluate(root.global_step)) - self.assertEqual(training_continuation + 1, - self.evaluate(root.save_counter)) - # pylint: enable=cell-var-from-loop - - def _get_checkpoint_name(self, name): - root = checkpointable.Checkpointable() - checkpointable_utils.add_variable( - root, name=name, shape=[1, 2], dtype=dtypes.float64) - named_variables, _ = checkpointable_utils._serialize_object_graph(root) - checkpoint_name, = named_variables.keys() - with ops.name_scope("root/" + checkpoint_name): - pass # Make sure we can use this as an op name if we prefix it. - return checkpoint_name - - @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) - def testVariableNameEscaping(self): - suffix = "/.ATTRIBUTES/VARIABLE_VALUE" - self.assertEqual(r"a.Sb.Sc" + suffix, self._get_checkpoint_name(r"a/b/c")) - self.assertEqual(r"b" + suffix, self._get_checkpoint_name(r"b")) - self.assertEqual(r"c.S" + suffix, self._get_checkpoint_name(r"c/")) - self.assertEqual(r"d.S..S" + suffix, self._get_checkpoint_name(r"d/.S")) - self.assertEqual(r"d.S..ATTRIBUTES.Sf" + suffix, - self._get_checkpoint_name(r"d/.ATTRIBUTES/f")) - - @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) - def testNumberedPath(self): - root = checkpointable.Checkpointable() - leaf = checkpointable.Checkpointable() - root.leaf = leaf - checkpointable_utils.add_variable(leaf, name="v", shape=[]) - named_variables, _ = checkpointable_utils._serialize_object_graph(root) - variable_name, = named_variables.keys() - self.assertEqual(r"leaf/v/.ATTRIBUTES/VARIABLE_VALUE", variable_name) - - @test_util.run_in_graph_and_eager_modes() - def testLocalNameValidation(self): - root = checkpointable.Checkpointable() - leaf = checkpointable.Checkpointable() - # Dots are escaped, which avoids conflicts with reserved names. - root._track_checkpointable(leaf, name=".ATTRIBUTES") - checkpointable_utils.add_variable(checkpointable=leaf, name="a", shape=[]) - named_variables, _ = checkpointable_utils._serialize_object_graph(root) - name, = named_variables.keys() - self.assertEqual(name, "..ATTRIBUTES/a/.ATTRIBUTES/VARIABLE_VALUE") - - def testAnonymousVarsInInit(self): - - class Model(training.Model): - - def __init__(self): - super(Model, self).__init__() - self.w = resource_variable_ops.ResourceVariable(0.0) - self.b = resource_variable_ops.ResourceVariable(0.0) - self.vars = [self.w, self.b] - - def call(self, x): - return x * self.w + self.b - - with context.eager_mode(): - model = Model() - optimizer = adam.AdamOptimizer(learning_rate=0.05) - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - checkpoint = checkpointable_utils.Checkpoint( - model=model, optimizer=optimizer) - for _ in range(2): - checkpoint.save(checkpoint_prefix) - with backprop.GradientTape() as tape: - loss = (constant_op.constant(1.) - - model(constant_op.constant(1.))) ** 2 - grad = tape.gradient(loss, model.vars) - optimizer.apply_gradients( - [(g, v) for g, v in zip(grad, model.vars)]) - - @test_util.run_in_graph_and_eager_modes() - def testLateDependencyTracking(self): - - class Dependency(checkpointable.Checkpointable): - - def build(self): - self.var = checkpointable_utils.add_variable( - self, "var", initializer=0.) - - class LateDependencies(checkpointable.Checkpointable): - - def add_dep(self): - self.dep = Dependency() - self.dep.build() - - original = LateDependencies() - original.add_dep() - self.evaluate(state_ops.assign(original.dep.var, 123.)) - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - save_path = checkpointable_utils.CheckpointableSaver( - original).save(checkpoint_prefix) - load_into = LateDependencies() - status = checkpointable_utils.CheckpointableSaver( - load_into).restore(save_path) - with self.assertRaises(AssertionError): - status.assert_consumed() - load_into.add_dep() - status.assert_consumed() - status.run_restore_ops() - self.assertEqual(123., self.evaluate(load_into.dep.var)) - - @test_util.run_in_graph_and_eager_modes() - def testDepAfterVar(self): - - class Dependency(checkpointable.Checkpointable): - - def build(self): - self.var = checkpointable_utils.add_variable( - self, "var", initializer=0.) - - class DepAfterVar(checkpointable.Checkpointable): - - def add_dep(self): - dep = Dependency() - dep.build() - self.dep = dep - - dep_after_var = DepAfterVar() - dep_after_var.add_dep() - self.evaluate(state_ops.assign(dep_after_var.dep.var, -14.)) - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - save_path = checkpointable_utils.CheckpointableSaver(dep_after_var).save( - checkpoint_prefix) - - loaded_dep_after_var = DepAfterVar() - status = checkpointable_utils.CheckpointableSaver( - loaded_dep_after_var).restore(save_path) - loaded_dep_after_var.add_dep() - status.assert_consumed() - status.run_restore_ops() - self.assertEqual(-14., self.evaluate(loaded_dep_after_var.dep.var)) - - @test_util.run_in_graph_and_eager_modes() - def testDeferredSlotRestoration(self): - checkpoint_directory = self.get_temp_dir() - - root = checkpointable.Checkpointable() - root.var = checkpointable_utils.add_variable( - root, name="var", initializer=0.) - optimizer = adam.AdamOptimizer(0.1) - if context.executing_eagerly(): - optimizer.minimize(root.var.read_value) - else: - train_op = optimizer.minimize(root.var) - # Note that `optimizer` has not been added as a dependency of - # `root`. Create a one-off grouping so that slot variables for `root.var` - # get initialized too. - self.evaluate(checkpointable_utils.gather_initializers( - checkpointable_utils.Checkpoint(root=root, optimizer=optimizer))) - self.evaluate(train_op) - self.evaluate(state_ops.assign(root.var, 12.)) - no_slots_path = checkpointable_utils.CheckpointableSaver(root).save( - os.path.join(checkpoint_directory, "no_slots")) - root.optimizer = optimizer - self.evaluate(state_ops.assign(root.var, 13.)) - self.evaluate(state_ops.assign(optimizer.get_slot(name="m", var=root.var), - 14.)) - slots_path = checkpointable_utils.CheckpointableSaver(root).save( - os.path.join(checkpoint_directory, "with_slots")) - new_root = checkpointable.Checkpointable() - # Load the slot-containing checkpoint (deferred), then immediately overwrite - # the non-slot variable (also deferred). - slot_status = checkpointable_utils.CheckpointableSaver( - new_root).restore(slots_path) - no_slot_status = checkpointable_utils.CheckpointableSaver( - new_root).restore(no_slots_path) - with self.assertRaises(AssertionError): - no_slot_status.assert_consumed() - new_root.var = checkpointable_utils.add_variable( - new_root, name="var", shape=[]) - no_slot_status.assert_consumed() - no_slot_status.run_restore_ops() - self.assertEqual(12., self.evaluate(new_root.var)) - new_root.optimizer = adam.AdamOptimizer(0.1) - with self.assertRaisesRegexp(AssertionError, "beta1_power"): - slot_status.assert_consumed() - self.assertEqual(12., self.evaluate(new_root.var)) - if context.executing_eagerly(): - # Slot variables are only created with restoring initializers when - # executing eagerly. - self.assertEqual(14., self.evaluate( - new_root.optimizer.get_slot(name="m", var=new_root.var))) - else: - self.assertIs(new_root.optimizer.get_slot(name="m", var=new_root.var), - None) - if context.executing_eagerly(): - new_root.optimizer.minimize(new_root.var.read_value) - else: - train_op = new_root.optimizer.minimize(new_root.var) - # The slot variable now exists; restore() didn't create it, but we should - # now have a restore op for it. - slot_status.run_restore_ops() - self.assertEqual(14., self.evaluate( - new_root.optimizer.get_slot(name="m", var=new_root.var))) - self.evaluate(train_op) - slot_status.assert_consumed() - - @test_util.run_in_graph_and_eager_modes() - def testOverlappingRestores(self): - checkpoint_directory = self.get_temp_dir() - save_root = checkpointable.Checkpointable() - save_root.dep = checkpointable.Checkpointable() - save_root.dep.var = checkpointable_utils.add_variable( - save_root.dep, name="var", initializer=0.) - self.evaluate(state_ops.assign(save_root.dep.var, 12.)) - saver = checkpointable_utils.CheckpointableSaver(save_root) - first_path = saver.save(os.path.join(checkpoint_directory, "first")) - self.evaluate(state_ops.assign(save_root.dep.var, 13.)) - second_path = saver.save(os.path.join(checkpoint_directory, "second")) - - first_root = checkpointable.Checkpointable() - second_root = checkpointable.Checkpointable() - first_status = checkpointable_utils.CheckpointableSaver( - first_root).restore(first_path) - second_status = checkpointable_utils.CheckpointableSaver( - second_root).restore(second_path) - load_dep = checkpointable.Checkpointable() - load_dep.var = checkpointable_utils.add_variable( - load_dep, name="var", shape=[]) - first_root.dep = load_dep - first_status.assert_consumed() - first_status.run_restore_ops() - self.assertEqual(12., self.evaluate(load_dep.var)) - second_root.dep = load_dep - second_status.assert_consumed() - second_status.run_restore_ops() - self.assertEqual(13., self.evaluate(load_dep.var)) - - # Try again with the order of the restore() reversed. The last restore - # determines the final value. - first_root = checkpointable.Checkpointable() - second_root = checkpointable.Checkpointable() - second_status = checkpointable_utils.CheckpointableSaver( - second_root).restore(second_path) - first_status = checkpointable_utils.CheckpointableSaver( - first_root).restore(first_path) - load_dep = checkpointable.Checkpointable() - load_dep.var = checkpointable_utils.add_variable( - load_dep, name="var", shape=[]) - first_root.dep = load_dep - first_status.assert_consumed() - first_status.run_restore_ops() - self.assertEqual(12., self.evaluate(load_dep.var)) - second_root.dep = load_dep - second_status.assert_consumed() - second_status.run_restore_ops() - self.assertEqual(12., self.evaluate(load_dep.var)) - - @test_util.run_in_graph_and_eager_modes() - def testAmbiguousLoad(self): - # Not OK to split one checkpoint object into two - checkpoint_directory = self.get_temp_dir() - save_root = checkpointable.Checkpointable() - save_root.dep_one = checkpointable.Checkpointable() - save_root.dep_two = checkpointable.Checkpointable() - dep_three = checkpointable.Checkpointable() - save_root.dep_one.dep_three = dep_three - save_root.dep_two.dep_three = dep_three - checkpointable_utils.add_variable(dep_three, name="var", initializer=0.) - self.evaluate(checkpointable_utils.gather_initializers(save_root)) - save_path = checkpointable_utils.CheckpointableSaver(save_root).save( - os.path.join(checkpoint_directory, "ckpt")) - load_root = checkpointable.Checkpointable() - checkpointable_utils.CheckpointableSaver(load_root).restore(save_path) - load_root.dep_one = checkpointable.Checkpointable() - load_root.dep_two = checkpointable.Checkpointable() - load_root.dep_one.dep_three = checkpointable.Checkpointable() - with self.assertRaisesRegexp(AssertionError, - "resolved to different objects"): - load_root.dep_two.dep_three = checkpointable.Checkpointable() - - @test_util.run_in_graph_and_eager_modes() - def testObjectsCombined(self): - # Currently fine to load two checkpoint objects into one Python object - checkpoint_directory = self.get_temp_dir() - save_root = checkpointable.Checkpointable() - save_root.dep_one = checkpointable.Checkpointable() - save_root.dep_two = checkpointable.Checkpointable() - checkpointable_utils.add_variable( - save_root.dep_one, name="var1", initializer=32., dtype=dtypes.float64) - checkpointable_utils.add_variable( - save_root.dep_two, name="var2", initializer=64., dtype=dtypes.float64) - self.evaluate(checkpointable_utils.gather_initializers(save_root)) - save_path = checkpointable_utils.CheckpointableSaver(save_root).save( - os.path.join(checkpoint_directory, "ckpt")) - load_root = checkpointable.Checkpointable() - load_root.dep_one = checkpointable.Checkpointable() - load_root.dep_two = load_root.dep_one - v1 = checkpointable_utils.add_variable( - load_root.dep_one, name="var1", shape=[], dtype=dtypes.float64) - v2 = checkpointable_utils.add_variable( - load_root.dep_one, name="var2", shape=[], dtype=dtypes.float64) - status = checkpointable_utils.CheckpointableSaver(load_root).restore( - save_path).assert_consumed() - status.run_restore_ops() - self.assertEqual(32., self.evaluate(v1)) - self.assertEqual(64., self.evaluate(v2)) - - @test_util.run_in_graph_and_eager_modes() - def testDependencyLoop(self): - # Note: this test creates garbage during eager execution because it - # purposefully creates a reference cycle. - first = checkpointable.Checkpointable() - second = checkpointable.Checkpointable() - first.second = second - second.first = first - first.v = checkpointable_utils.add_variable( - first, "v1", initializer=[3., 1., 4.]) - second.v = checkpointable_utils.add_variable( - second, "v2", initializer=[1., 1., 2., 3.]) - self.evaluate(checkpointable_utils.gather_initializers(first)) - checkpoint_directory = self.get_temp_dir() - save_path = checkpointable_utils.CheckpointableSaver(first).save( - os.path.join(checkpoint_directory, "ckpt")) - - # Test deferred loading - first_load = checkpointable.Checkpointable() - status = checkpointable_utils.CheckpointableSaver( - first_load).restore(save_path) - second_load = checkpointable.Checkpointable() - first_load.second = second_load - second_load.first = first_load - with self.assertRaises(AssertionError): - status.assert_consumed() - first_load.v = checkpointable_utils.add_variable( - first_load, "v1", shape=[3]) - second_load.v = checkpointable_utils.add_variable( - second_load, "v2", shape=[4]) - status.assert_consumed() - status.run_restore_ops() - self.assertAllEqual([3., 1., 4.], self.evaluate(first_load.v)) - self.assertAllEqual([1., 1., 2., 3.], self.evaluate(second_load.v)) - - # Test loading when variables have already been created - self.evaluate(first_load.v.assign([2., 7., 1.])) - self.assertAllEqual([2., 7., 1.], self.evaluate(first_load.v)) - self.evaluate(second_load.v.assign([2., 7., 1., 8.])) - self.assertAllEqual([2., 7., 1., 8.], self.evaluate(second_load.v)) - status = checkpointable_utils.CheckpointableSaver(first_load).restore( - save_path).assert_consumed() - status.run_restore_ops() - self.assertAllEqual([3., 1., 4.], self.evaluate(first_load.v)) - self.assertAllEqual([1., 1., 2., 3.], self.evaluate(second_load.v)) - - @test_util.run_in_graph_and_eager_modes() - def testRestoreOnAssign(self): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - save_graph = ops.Graph() - with save_graph.as_default(), self.test_session(save_graph): - first = checkpointable.Checkpointable() - first.var1 = variable_scope.get_variable( - name="outside_var", initializer=0.) - first.var2 = variable_scope.get_variable( - name="blah", initializer=0.) - self.evaluate(first.var1.assign(4.)) - self.evaluate(first.var2.assign(8.)) - save_path = checkpointable_utils.CheckpointableSaver(first).save( - checkpoint_prefix) - restore_graph = ops.Graph() - with restore_graph.as_default(), self.test_session(restore_graph): - second = checkpointable.Checkpointable() - second.var2 = variable_scope.get_variable( - name="blah", initializer=0.) - status = checkpointable_utils.CheckpointableSaver( - second).restore(save_path) - recreated_var1 = variable_scope.get_variable( - name="outside_var", initializer=0.) - status.run_restore_ops() - self.assertEqual(8., self.evaluate(second.var2)) - self.evaluate(recreated_var1.assign(-2.)) - self.assertEqual(-2., self.evaluate(recreated_var1)) - second.var1 = recreated_var1 - status.run_restore_ops() - self.assertEqual(4., self.evaluate(recreated_var1)) - - def testManySavesGraph(self): - """Saves after the first should not modify the graph.""" - with context.graph_mode(): - graph = ops.Graph() - with graph.as_default(), self.test_session(graph): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - obj = checkpointable.Checkpointable() - obj.var = variable_scope.get_variable(name="v", initializer=0.) - obj.opt = adam.AdamOptimizer(0.1) - obj.opt.minimize(obj.var.read_value()) - self.evaluate(checkpointable_utils.gather_initializers(obj)) - saver = checkpointable_utils.CheckpointableSaver(obj) - saver.save(checkpoint_prefix) - before_ops = graph.get_operations() - saver.save(checkpoint_prefix) - self.assertEqual(before_ops, graph.get_operations()) - - @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) - def testCheckpointCleanup(self): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - obj = checkpointable.Checkpointable() - obj.var = variable_scope.get_variable(name="v", initializer=0.) - self.evaluate(checkpointable_utils.gather_initializers(obj)) - saver = checkpointable_utils.Checkpoint(obj=obj) - for _ in range(10): - saver.save(checkpoint_prefix) - expected_filenames = ["checkpoint"] - for checkpoint_number in range(6, 11): - expected_filenames.append("ckpt-%d.index" % (checkpoint_number,)) - expected_filenames.append( - "ckpt-%d.data-00000-of-00001" % (checkpoint_number,)) - six.assertCountEqual( - self, - expected_filenames, - os.listdir(checkpoint_directory)) - - @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) - def testCheckpointCleanupChangingVarList(self): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - obj = checkpointable.Checkpointable() - obj.var = variable_scope.get_variable(name="v", initializer=0.) - self.evaluate(checkpointable_utils.gather_initializers(obj)) - checkpoint = checkpointable_utils.Checkpoint(obj=obj) - looped_variables = [] - for iteration in range(10): - new_variable = resource_variable_ops.ResourceVariable(iteration) - self.evaluate(new_variable.initializer) - setattr(checkpoint, "var_%d" % iteration, new_variable) - checkpoint.save(checkpoint_prefix) - looped_variables.append(new_variable) - expected_filenames = ["checkpoint"] - # We've copied the saver each time, but checkpoint management should still - # be consistent. - for checkpoint_number in range(6, 11): - expected_filenames.append("ckpt-%d.index" % (checkpoint_number,)) - expected_filenames.append( - "ckpt-%d.data-00000-of-00001" % (checkpoint_number,)) - six.assertCountEqual( - self, - expected_filenames, - os.listdir(checkpoint_directory)) - for v in looped_variables: - self.evaluate(v.assign(314)) - checkpoint.restore(checkpoint_prefix + "-6").run_restore_ops() - self.assertEqual(314, self.evaluate(checkpoint.var_9)) - self.assertEqual(314, self.evaluate(checkpoint.var_8)) - self.assertEqual(314, self.evaluate(checkpoint.var_6)) - self.assertEqual(5, self.evaluate(checkpoint.var_5)) - self.assertEqual(1, self.evaluate(checkpoint.var_1)) - self.assertEqual(0, self.evaluate(checkpoint.var_0)) - if context.executing_eagerly(): - checkpoint.restore(checkpoint_prefix + "-10").run_restore_ops() - self.assertEqual(9, self.evaluate(checkpoint.var_9)) - self.assertEqual(8, self.evaluate(checkpoint.var_8)) - self.assertEqual(1, self.evaluate(checkpoint.var_1)) - self.assertEqual(0, self.evaluate(checkpoint.var_0)) - else: - # Restoring into modified graphs is an error while graph building. - with self.assertRaises(NotImplementedError): - checkpoint.restore(checkpoint_prefix + "-10").run_restore_ops() - - def testManyRestoresGraph(self): - """Restores after the first should not modify the graph.""" - with context.graph_mode(): - graph = ops.Graph() - with graph.as_default(), self.test_session(graph): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - obj = checkpointable.Checkpointable() - obj.var = variable_scope.get_variable(name="v", initializer=0.) - obj.opt = adam.AdamOptimizer(0.1) - obj.opt.minimize(obj.var.read_value()) - self.evaluate(checkpointable_utils.gather_initializers(obj)) - saver = checkpointable_utils.CheckpointableSaver(obj) - save_path = saver.save(checkpoint_prefix) - saver.restore(save_path) - before_ops = graph.get_operations() - saver.restore(save_path) - self.assertEqual(before_ops, graph.get_operations()) - - def testMultipleGraphsNonSlotVariables(self): - with context.graph_mode(): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - optimizer = adam.AdamOptimizer(0.001) - # Construct a model in one graph - first_graph = ops.Graph() - first_session = session_lib.Session(graph=first_graph) - with first_graph.as_default(), first_session.as_default(): - first_variable = resource_variable_ops.ResourceVariable([1.]) - first_root_checkpointable = checkpointable_utils.Checkpoint( - optimizer=optimizer, variable=first_variable) - train_op = optimizer.minimize(first_variable.read_value) - self.evaluate(checkpointable_utils.gather_initializers( - first_root_checkpointable)) - self.evaluate(train_op) - self.evaluate(first_variable.assign([1.])) - self.evaluate(optimizer.get_slot( - var=first_variable, name="m").assign([2.])) - beta1_power, _ = optimizer._get_beta_accumulators() - self.evaluate(beta1_power.assign(3.)) - - # Save and load in a second graph - second_graph = ops.Graph() - with second_graph.as_default(), session_lib.Session(graph=second_graph): - second_variable = resource_variable_ops.ResourceVariable([1.]) - second_root_checkpointable = checkpointable_utils.Checkpoint( - optimizer=optimizer, variable=second_variable) - train_op = optimizer.minimize(second_variable.read_value) - second_root_checkpointable.restore(None).initialize_or_restore() - self.evaluate(train_op) - self.evaluate(second_variable.assign([4.])) - self.evaluate(optimizer.get_slot( - var=second_variable, name="m").assign([5.])) - beta1_power, _ = optimizer._get_beta_accumulators() - self.evaluate(beta1_power.assign(6.)) - save_path = second_root_checkpointable.save(checkpoint_prefix) - self.evaluate(second_variable.assign([7.])) - self.evaluate(optimizer.get_slot( - var=second_variable, name="m").assign([8.])) - beta1_power, _ = optimizer._get_beta_accumulators() - self.assertAllEqual(6., self.evaluate(beta1_power)) - status = second_root_checkpointable.restore(save_path) - status.assert_consumed().run_restore_ops() - self.assertAllEqual([4.], self.evaluate(second_variable)) - self.assertAllEqual([5.], self.evaluate(optimizer.get_slot( - var=second_variable, name="m"))) - beta1_power, _ = optimizer._get_beta_accumulators() - self.assertAllEqual(6., self.evaluate(beta1_power)) - - # Check that the first graph is unmolested - with first_graph.as_default(), first_session.as_default(): - self.assertAllEqual([1.], self.evaluate(first_variable)) - self.assertAllEqual([2.], self.evaluate(optimizer.get_slot( - var=first_variable, name="m"))) - beta1_power, _ = optimizer._get_beta_accumulators() - self.assertAllEqual(3., self.evaluate(beta1_power)) - - @test_util.run_in_graph_and_eager_modes() - def test_sequential(self): - model = sequential.Sequential() - checkpoint = checkpointable_utils.Checkpoint(model=model) - model.add(core.Dense(4)) - second_dense = core.Dense(5) - model.add(second_dense) - model(constant_op.constant([[1.]])) - checkpoint.restore(None).initialize_or_restore() - self.evaluate(second_dense.bias.assign( - constant_op.constant([1., 2., 3., 4., 5.]))) - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - save_path = checkpoint.save(checkpoint_prefix) - self.evaluate(second_dense.bias.assign( - constant_op.constant([5., 6., 7., 8., 9.]))) - checkpoint.restore(save_path).assert_consumed().run_restore_ops() - self.assertAllEqual([1., 2., 3., 4., 5.], self.evaluate(second_dense.bias)) - - deferred_sequential = sequential.Sequential() - deferred_sequential_checkpoint = checkpointable_utils.Checkpoint( - model=deferred_sequential) - status = deferred_sequential_checkpoint.restore(save_path) - deferred_sequential.add(core.Dense(4)) - deferred_sequential(constant_op.constant([[1.]])) - deferred_second_dense = core.Dense(5) - deferred_sequential.add(deferred_second_dense) - deferred_sequential(constant_op.constant([[1.]])) - status.run_restore_ops() - self.assertAllEqual([1., 2., 3., 4., 5.], - self.evaluate(deferred_second_dense.bias)) - - -class TemplateTests(test.TestCase): - - @test_util.run_in_graph_and_eager_modes() - def test_checkpointable_save_restore(self): - - def _templated(): - v = variable_scope.get_variable( - "v", shape=[1], initializer=init_ops.zeros_initializer()) - v2 = variable_scope.get_variable( - "v2", shape=[1], initializer=init_ops.zeros_initializer()) - return v, v + 1., v2 - - save_template = template.make_template("s1", _templated) - save_root = checkpointable_utils.Checkpoint(my_template=save_template) - v1_save, _, v2_save = save_template() - self.evaluate(v1_save.assign([12.])) - self.evaluate(v2_save.assign([14.])) - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - save_path = save_root.save(checkpoint_prefix) - - load_template = template.make_template("s2", _templated) - load_root = checkpointable_utils.Checkpoint(my_template=load_template) - status = load_root.restore(save_path) - var, var_plus_one, var2 = load_template() - self.assertEqual(2, len(load_template._checkpoint_dependencies)) - self.assertEqual("v", load_template._checkpoint_dependencies[0].name) - self.assertEqual("v2", load_template._checkpoint_dependencies[1].name) - status.assert_consumed().run_restore_ops() - self.assertAllEqual([12.], self.evaluate(var)) - self.assertAllEqual([13.], self.evaluate(var_plus_one)) - self.assertAllEqual([14.], self.evaluate(var2)) - - @test_util.run_in_graph_and_eager_modes() - def test_checkpointable_save_restore_nested(self): - - def _inner_template(): - v = variable_scope.get_variable( - "v", shape=[1], initializer=init_ops.zeros_initializer()) - return v - - def _outer_template(): - first_inner = template.make_template("i1", _inner_template) - second_inner = template.make_template("i2", _inner_template) - v1 = first_inner() - v2 = second_inner() - v3 = second_inner() - return (first_inner, second_inner), (v1, v2, v3) - - with variable_scope.variable_scope("ignored"): - save_template = template.make_template("s1", _outer_template) - save_root = checkpointable_utils.Checkpoint(my_template=save_template) - (inner_template_one, inner_template_two), _ = save_template() - self.evaluate(inner_template_one.variables[0].assign([20.])) - self.evaluate(inner_template_two.variables[0].assign([25.])) - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - save_path = save_root.save(checkpoint_prefix) - - load_template = template.make_template("s2", _outer_template) - load_root = checkpointable_utils.Checkpoint(my_template=load_template) - status = load_root.restore(save_path) - (inner_template_one, inner_template_two), (v1, v2, v3) = load_template() - outer_template_dependencies = load_root.my_template._checkpoint_dependencies - self.assertEqual(2, len(outer_template_dependencies)) - self.assertEqual("i1", outer_template_dependencies[0].name) - self.assertIs(inner_template_one, outer_template_dependencies[0].ref) - self.assertEqual("i2", outer_template_dependencies[1].name) - self.assertIs(inner_template_two, outer_template_dependencies[1].ref) - self.assertEqual(1, len(inner_template_one._checkpoint_dependencies)) - self.assertEqual("v", inner_template_one._checkpoint_dependencies[0].name) - self.assertEqual(1, len(inner_template_two._checkpoint_dependencies)) - self.assertEqual("v", inner_template_two._checkpoint_dependencies[0].name) - status.assert_consumed().run_restore_ops() - self.assertAllEqual([20.], self.evaluate(v1)) - self.assertAllEqual([25.], self.evaluate(v2)) - self.assertAllEqual([25.], self.evaluate(v3)) - - -class CheckpointCompatibilityTests(test.TestCase): - - def _initialized_model(self): - input_value = constant_op.constant([[3.]]) - model = MyModel() - optimizer = adam.AdamOptimizer(0.001) - optimizer_step = training_util.get_or_create_global_step() - root_checkpointable = checkpointable_utils.Checkpoint( - optimizer=optimizer, model=model, optimizer_step=optimizer_step) - train_op = optimizer.minimize( - functools.partial(model, input_value), - global_step=optimizer_step) - self.evaluate(checkpointable_utils.gather_initializers( - root_checkpointable)) - self.evaluate(train_op) - # A regular variable, a slot variable, and a non-slot Optimizer variable - # with known values to check when loading. - self.evaluate(model._named_dense.bias.assign([1.])) - self.evaluate(optimizer.get_slot( - var=model._named_dense.bias, name="m").assign([2.])) - beta1_power, _ = optimizer._get_beta_accumulators() - self.evaluate(beta1_power.assign(3.)) - return root_checkpointable - - def _set_sentinels(self, root_checkpointable): - self.evaluate(root_checkpointable.model._named_dense.bias.assign([101.])) - self.evaluate( - root_checkpointable.optimizer.get_slot( - var=root_checkpointable.model._named_dense.bias, name="m") - .assign([102.])) - beta1_power, _ = root_checkpointable.optimizer._get_beta_accumulators() - self.evaluate(beta1_power.assign(103.)) - - def _check_sentinels(self, root_checkpointable): - self.assertAllEqual( - [1.], self.evaluate(root_checkpointable.model._named_dense.bias)) - self.assertAllEqual([2.], self.evaluate( - root_checkpointable.optimizer.get_slot( - var=root_checkpointable.model._named_dense.bias, name="m"))) - beta1_power, _ = root_checkpointable.optimizer._get_beta_accumulators() - self.assertAllEqual(3., self.evaluate(beta1_power)) - - def _write_name_based_checkpoint(self): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - with context.graph_mode(): - save_graph = ops.Graph() - with save_graph.as_default(), self.test_session( - graph=save_graph) as session: - root = self._initialized_model() - name_saver = core_saver.Saver() - return name_saver.save( - sess=session, save_path=checkpoint_prefix, - global_step=root.optimizer_step) - - @test_util.run_in_graph_and_eager_modes() - def testLoadFromNameBasedSaver(self): - """Save a name-based checkpoint, load it using the object-based API.""" - with test_util.device(use_gpu=True): - save_path = self._write_name_based_checkpoint() - root = self._initialized_model() - self._set_sentinels(root) - with self.assertRaises(AssertionError): - self._check_sentinels(root) - object_saver = checkpointable_utils.CheckpointableSaver(root) - status = object_saver.restore(save_path) - with self.assertRaises(AssertionError): - status.assert_consumed() - status.run_restore_ops() - self._check_sentinels(root) - self._set_sentinels(root) - status.initialize_or_restore() - self._check_sentinels(root) - - # TODO(allenl): Test for the core name-based saver loading object-based - # checkpoints once object-based checkpointing is in core. - - def testSaveGraphLoadEager(self): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - with context.graph_mode(): - save_graph = ops.Graph() - with save_graph.as_default(), self.test_session( - graph=save_graph) as session: - root = self._initialized_model() - object_saver = checkpointable_utils.CheckpointableSaver(root) - save_path = object_saver.save( - session=session, file_prefix=checkpoint_prefix) - with context.eager_mode(): - root = self._initialized_model() - self._set_sentinels(root) - root.restore(save_path).assert_consumed() - self._check_sentinels(root) - - def testSaveEagerLoadGraph(self): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - with context.eager_mode(): - root = self._initialized_model() - object_saver = checkpointable_utils.CheckpointableSaver(root) - save_path = object_saver.save(file_prefix=checkpoint_prefix) - with context.graph_mode(): - save_graph = ops.Graph() - with save_graph.as_default(), self.test_session( - graph=save_graph): - root = self._initialized_model() - self._set_sentinels(root) - root.restore(save_path).assert_consumed().run_restore_ops() - self._check_sentinels(root) - if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/eager/python/datasets_test.py b/tensorflow/contrib/eager/python/datasets_test.py index f76a896d3d..7b123707cc 100644 --- a/tensorflow/contrib/eager/python/datasets_test.py +++ b/tensorflow/contrib/eager/python/datasets_test.py @@ -27,7 +27,6 @@ from tensorflow.contrib import lookup from tensorflow.contrib.data.python.ops import prefetching_ops from tensorflow.contrib.data.python.ops import threadpool from tensorflow.contrib.data.python.ops import unique -from tensorflow.contrib.eager.python import checkpointable_utils from tensorflow.contrib.eager.python import datasets from tensorflow.python.data import Dataset from tensorflow.python.eager import test @@ -38,6 +37,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor from tensorflow.python.ops import math_ops from tensorflow.python.ops import script_ops +from tensorflow.python.training import checkpointable_utils class IteratorTest(test.TestCase): diff --git a/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py b/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py index 9adf47d505..f825a2a736 100644 --- a/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py +++ b/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py @@ -33,8 +33,8 @@ import tensorflow as tf import tensorflow.contrib.eager as tfe from tensorflow.contrib.eager.python.examples.spinn import data from third_party.examples.eager.spinn import spinn -from tensorflow.contrib.eager.proto import checkpointable_object_graph_pb2 from tensorflow.contrib.summary import summary_test_util +from tensorflow.core.protobuf import checkpointable_object_graph_pb2 from tensorflow.python.eager import test from tensorflow.python.framework import test_util from tensorflow.python.training import checkpoint_utils diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py index 28f5f286eb..f0fe4ce8c5 100644 --- a/tensorflow/contrib/eager/python/metrics_test.py +++ b/tensorflow/contrib/eager/python/metrics_test.py @@ -21,7 +21,6 @@ from __future__ import print_function import os import tempfile -from tensorflow.contrib.eager.python import checkpointable_utils from tensorflow.contrib.eager.python import metrics from tensorflow.contrib.summary import summary_test_util from tensorflow.python.eager import context @@ -31,6 +30,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import summary_ops_v2 as summary_ops +from tensorflow.python.training import checkpointable_utils from tensorflow.python.training import training_util diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py index c6f3f20e78..79dd117854 100644 --- a/tensorflow/contrib/eager/python/tfe.py +++ b/tensorflow/contrib/eager/python/tfe.py @@ -84,8 +84,6 @@ from __future__ import print_function # pylint:disable=g-bad-import-order,g-import-not-at-top,unused-import # from tensorflow.contrib.eager.python import metrics -from tensorflow.contrib.eager.python.checkpointable_utils import CheckpointableSaver -from tensorflow.contrib.eager.python.checkpointable_utils import Checkpoint from tensorflow.contrib.eager.python.datasets import Iterator from tensorflow.contrib.eager.python.network import Network from tensorflow.contrib.eager.python.network import Sequential @@ -123,6 +121,8 @@ from tensorflow.python.ops.variable_scope import EagerVariableStore from tensorflow.python.ops import script_ops from tensorflow.python.ops import template from tensorflow.python.training.checkpointable import Checkpointable +from tensorflow.python.training.checkpointable_utils import CheckpointableSaver +from tensorflow.python.training.checkpointable_utils import Checkpoint from tensorflow.python.util.all_util import remove_undocumented py_func = script_ops.eager_py_func diff --git a/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py b/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py index 54bc23cdef..6ade4ccd52 100644 --- a/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py +++ b/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py @@ -24,7 +24,6 @@ import os import six -from tensorflow.contrib.eager.python import checkpointable_utils from tensorflow.contrib.optimizer_v2 import adam from tensorflow.python.client import session as session_lib from tensorflow.python.eager import backprop @@ -42,6 +41,7 @@ from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import state_ops from tensorflow.python.ops import variable_scope from tensorflow.python.training import checkpointable +from tensorflow.python.training import checkpointable_utils from tensorflow.python.training import saver as core_saver from tensorflow.python.training import training_util diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 118955219b..97e0095e05 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -212,6 +212,7 @@ CORE_PROTO_SRCS = [ # ones with individual proto_library targets. ADDITIONAL_CORE_PROTO_SRCS = [ "example/example_parser_configuration.proto", + "protobuf/checkpointable_object_graph.proto", "protobuf/control_flow.proto", # TODO(ebrevdo): Re-enable once CriticalSection is in core. # "protobuf/critical_section.proto", diff --git a/tensorflow/contrib/eager/proto/checkpointable_object_graph.proto b/tensorflow/core/protobuf/checkpointable_object_graph.proto similarity index 85% rename from tensorflow/contrib/eager/proto/checkpointable_object_graph.proto rename to tensorflow/core/protobuf/checkpointable_object_graph.proto index 024765acb2..651f692f6d 100644 --- a/tensorflow/contrib/eager/proto/checkpointable_object_graph.proto +++ b/tensorflow/core/protobuf/checkpointable_object_graph.proto @@ -2,14 +2,14 @@ syntax = "proto3"; option cc_enable_arenas = true; -package tensorflow.contrib.eager; +package tensorflow; -// Prototype format which saves extra information about the objects which own -// variables, allowing for more robust checkpoint loading into modified -// programs. Currently stored in its own entry in a TensorBundle. +// A TensorBundle addition which saves extra information about the objects which +// own variables, allowing for more robust checkpoint loading into modified +// programs. message CheckpointableObjectGraph { - message Object { + message CheckpointableObject { message ObjectReference { // An index into `CheckpointableObjectGraph.nodes`, indicating the object // being referenced. @@ -51,5 +51,5 @@ message CheckpointableObjectGraph { repeated SlotVariableReference slot_variables = 3; } - repeated Object nodes = 1; + repeated CheckpointableObject nodes = 1; } diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 9707b370c0..559926d415 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -2943,6 +2943,7 @@ py_library( srcs_version = "PY2AND3", deps = [ ":array_ops", + ":array_ops_gen", ":checkpoint_ops_gen", ":client", ":control_flow_ops", @@ -2978,6 +2979,7 @@ py_library( ":variables", "//third_party/py/numpy", "@six_archive//:six", + "//tensorflow/core:protos_all_py", "//tensorflow/python/eager:backprop", "//tensorflow/python/eager:context", "//tensorflow/python/ops/losses", @@ -3010,6 +3012,39 @@ py_test( ], ) +py_test( + name = "checkpointable_utils_test", + srcs = ["training/checkpointable_utils_test.py"], + srcs_version = "PY2AND3", + tags = [ + "no_windows", # TODO: needs investigation on Windows + "notsan", # b/74395663 + ], + deps = [ + ":checkpointable", + ":constant_op", + ":control_flow_ops", + ":dtypes", + ":framework_ops", + ":framework_test_lib", + ":init_ops", + ":resource_variable_ops", + ":session", + ":state_ops", + ":template", + ":training", + ":training_util", + ":variable_scope", + "//tensorflow/python/eager:backprop", + "//tensorflow/python/eager:context", + "//tensorflow/python/eager:function", + "//tensorflow/python/eager:test", + "//tensorflow/python/keras:engine", + "//tensorflow/python/keras:layers", + "@six_archive//:six", + ], +) + py_test( name = "distribute_test", size = "small", diff --git a/tensorflow/python/training/checkpointable_utils.py b/tensorflow/python/training/checkpointable_utils.py index 32123f87ef..da99d2ec31 100644 --- a/tensorflow/python/training/checkpointable_utils.py +++ b/tensorflow/python/training/checkpointable_utils.py @@ -17,14 +17,48 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import abc +import collections import weakref +from tensorflow.core.protobuf import checkpointable_object_graph_pb2 +from tensorflow.python import pywrap_tensorflow +from tensorflow.python.client import session as session_lib +from tensorflow.python.eager import context +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops -from tensorflow.python.training import checkpointable +from tensorflow.python.framework import tensor_shape +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import variable_scope +from tensorflow.python.training import checkpointable as checkpointable_lib +from tensorflow.python.training import optimizer as optimizer_lib from tensorflow.python.training import saver as saver_lib +from tensorflow.python.util import deprecation -class _Checkpoint(object): +_ESCAPE_CHAR = "." # For avoiding conflicts with user-specified names. + +# Keyword for identifying that the next bit of a checkpoint variable name is a +# slot name. Checkpoint names for slot variables look like: +# +# /<_OPTIMIZER_SLOTS_NAME>// +# +# Where is a full path from the checkpoint root to the +# variable being slotted for. +_OPTIMIZER_SLOTS_NAME = _ESCAPE_CHAR + "OPTIMIZER_SLOT" +# Keyword for separating the path to an object from the name of an +# attribute in checkpoint names. Used like: +# /<_OBJECT_ATTRIBUTES_NAME>/ +_OBJECT_ATTRIBUTES_NAME = _ESCAPE_CHAR + "ATTRIBUTES" +# Key where the object graph proto is saved in a TensorBundle +_OBJECT_GRAPH_PROTO_KEY = "_CHECKPOINTABLE_OBJECT_GRAPH" + + +class _CheckpointRestoreCoordinator(object): """Holds the status of an object-based checkpoint load.""" def __init__(self, object_graph_proto, save_path, dtype_map=None): @@ -72,7 +106,817 @@ class _Checkpoint(object): # `node` refers to an `Optimizer`, since only these have slot variables. self.slot_restorations.setdefault( slot_reference.original_variable_node_id, []).append( - checkpointable._SlotVariableRestoration( # pylint: disable=protected-access + checkpointable_lib._SlotVariableRestoration( # pylint: disable=protected-access optimizer_id=node_index, slot_variable_id=slot_reference.slot_variable_node_id, slot_name=slot_reference.slot_name)) + + +# TODO(allenl): If this ends up in a public API, consider adding LINT.IfChange +# or consolidating the implementation with get_variable. +def _default_getter(name, shape, dtype, initializer=None, + partition_info=None, **kwargs): + """A pared-down version of get_variable which does not reuse variables.""" + dtype = dtypes.as_dtype(dtype) + shape_object = tensor_shape.as_shape(shape) + with ops.init_scope(): + if initializer is None: + initializer, initializing_from_value = ( + variable_scope._get_default_variable_store()._get_default_initializer( # pylint: disable=protected-access + name=name, shape=shape_object, dtype=dtype)) + else: + initializing_from_value = not callable(initializer) + # Same logic as get_variable + variable_dtype = dtype.base_dtype + if initializing_from_value: + if shape is not None: + raise ValueError("If initializer is a constant, do not specify shape.") + initial_value = initializer + else: + # Instantiate initializer if provided initializer is a type object. + if isinstance(initializer, type(init_ops.Initializer)): + initializer = initializer(dtype=dtype) + def initial_value(): + return initializer( + shape_object.as_list(), dtype=dtype, partition_info=partition_info) + return resource_variable_ops.ResourceVariable( + initial_value=initial_value, + name=name, + dtype=variable_dtype, + **kwargs + ) + + +def add_variable(checkpointable, name, shape=None, dtype=dtypes.float32, + initializer=None): + """Add a variable to a Checkpointable with no scope influence.""" + return checkpointable._add_variable_with_custom_getter( # pylint: disable=protected-access + name=name, shape=shape, dtype=dtype, + initializer=initializer, getter=_default_getter) + + +def _breadth_first_checkpointable_traversal(root_checkpointable): + """Find shortest paths to all variables owned by dependencies of root.""" + bfs_sorted = [] + to_visit = collections.deque([root_checkpointable]) + path_to_root = {root_checkpointable: ()} + while to_visit: + current_checkpointable = to_visit.popleft() + current_checkpointable._maybe_initialize_checkpointable() # pylint: disable=protected-access + bfs_sorted.append(current_checkpointable) + for child_checkpointable in ( + current_checkpointable._checkpoint_dependencies): # pylint: disable=protected-access + if child_checkpointable.ref not in path_to_root: + path_to_root[child_checkpointable.ref] = ( + path_to_root[current_checkpointable] + (child_checkpointable,)) + to_visit.append(child_checkpointable.ref) + return bfs_sorted, path_to_root + + +def _escape_local_name(name): + # We need to support slashes in local names for compatibility, since this + # naming scheme is being patched in to things like Layer.add_variable where + # slashes were previously accepted. We also want to use slashes to indicate + # edges traversed to reach the variable, so we escape forward slashes in + # names. + return (name.replace(_ESCAPE_CHAR, _ESCAPE_CHAR + _ESCAPE_CHAR) + .replace(r"/", _ESCAPE_CHAR + "S")) + + +def _object_prefix_from_path(path_to_root): + return "/".join( + (_escape_local_name(checkpointable.name) + for checkpointable in path_to_root)) + + +def _slot_variable_naming_for_optimizer(optimizer_path): + """Make a function for naming slot variables in an optimizer.""" + # Name slot variables: + # + # /<_OPTIMIZER_SLOTS_NAME>// + # + # where is exactly the checkpoint name used for the original + # variable, including the path from the checkpoint root and the local name in + # the object which owns it. Note that we only save slot variables if the + # variable it's slotting for is also being saved. + + optimizer_identifier = "/%s/%s/" % (_OPTIMIZER_SLOTS_NAME, optimizer_path) + + def _name_slot_variable(variable_path, slot_name): + """With an optimizer specified, name a slot variable.""" + return (variable_path + + optimizer_identifier + + _escape_local_name(slot_name)) + + return _name_slot_variable + + +def _serialize_slot_variables(checkpointable_objects, node_ids, object_names): + """Gather and name slot variables.""" + non_slot_objects = list(checkpointable_objects) + slot_variables = {} + for checkpointable in non_slot_objects: + if isinstance(checkpointable, optimizer_lib.Optimizer): + naming_scheme = _slot_variable_naming_for_optimizer( + optimizer_path=object_names[checkpointable]) + slot_names = checkpointable.get_slot_names() + for slot_name in slot_names: + for original_variable_node_id, original_variable in enumerate( + non_slot_objects): + try: + slot_variable = checkpointable.get_slot( + original_variable, slot_name) + except AttributeError: + slot_variable = None + if slot_variable is None: + continue + slot_variable._maybe_initialize_checkpointable() # pylint: disable=protected-access + if slot_variable._checkpoint_dependencies: # pylint: disable=protected-access + # TODO(allenl): Gather dependencies of slot variables. + raise NotImplementedError( + "Currently only variables with no dependencies can be saved as " + "slot variables. File a feature request if this limitation " + "bothers you.") + if slot_variable in node_ids: + raise NotImplementedError( + "A slot variable was re-used as a dependency of a " + "Checkpointable object. This is not currently allowed. File a " + "feature request if this limitation bothers you.") + checkpoint_name = naming_scheme( + variable_path=object_names[original_variable], + slot_name=slot_name) + object_names[slot_variable] = checkpoint_name + slot_variable_node_id = len(checkpointable_objects) + node_ids[slot_variable] = slot_variable_node_id + checkpointable_objects.append(slot_variable) + slot_variable_proto = ( + checkpointable_object_graph_pb2.CheckpointableObjectGraph + .CheckpointableObject.SlotVariableReference( + slot_name=slot_name, + original_variable_node_id=original_variable_node_id, + slot_variable_node_id=slot_variable_node_id)) + slot_variables.setdefault(checkpointable, []).append( + slot_variable_proto) + return slot_variables + + +def _serialize_checkpointables( + checkpointable_objects, node_ids, object_names, slot_variables): + """Name non-slot `Checkpointable`s and add them to `object_graph_proto`.""" + object_graph_proto = ( + checkpointable_object_graph_pb2.CheckpointableObjectGraph()) + named_saveables = {} + + for checkpoint_id, checkpointable in enumerate(checkpointable_objects): + assert node_ids[checkpointable] == checkpoint_id + object_proto = object_graph_proto.nodes.add() + object_proto.slot_variables.extend(slot_variables.get(checkpointable, ())) + object_name = object_names[checkpointable] + for name, saveable_factory in ( + checkpointable._gather_saveables_for_checkpoint().items()): # pylint: disable=protected-access + attribute = object_proto.attributes.add() + attribute.name = name + attribute.checkpoint_key = "%s/%s/%s" % ( + object_name, _OBJECT_ATTRIBUTES_NAME, _escape_local_name(name)) + if callable(saveable_factory): + saveable = saveable_factory(name=attribute.checkpoint_key) + else: + saveable = saveable_factory + # Figure out the name-based Saver's name for this variable. + saver_dict = saver_lib.BaseSaverBuilder.OpListToDict( + [saveable], convert_variable_to_tensor=False) + attribute.full_name, = saver_dict.keys() + named_saveables[attribute.checkpoint_key] = saveable + + for child in checkpointable._checkpoint_dependencies: # pylint: disable=protected-access + child_proto = object_proto.children.add() + child_proto.node_id = node_ids[child.ref] + child_proto.local_name = child.name + + return named_saveables, object_graph_proto + + +def _serialize_object_graph(root_checkpointable): + """Determine checkpoint keys for variables and build a serialized graph. + + Non-slot variables are keyed based on a shortest path from the root saveable + to the object which owns the variable (i.e. the one which called + `Checkpointable._add_variable` to create it). + + Slot variables are keyed based on a shortest path to the variable being + slotted for, a shortest path to their optimizer, and the slot name. + + Args: + root_checkpointable: A `Checkpointable` object whose variables (including + the variables of dependencies, recursively) should be saved. + + Returns: + A tuple of (named_variables, object_graph_proto): + named_variables: A dictionary mapping names to variable objects. + object_graph_proto: A CheckpointableObjectGraph protocol buffer containing + the serialized object graph and variable references. + + Raises: + ValueError: If there are invalid characters in an optimizer's slot names. + """ + checkpointable_objects, path_to_root = ( + _breadth_first_checkpointable_traversal(root_checkpointable)) + object_names = { + obj: _object_prefix_from_path(path) + for obj, path in path_to_root.items()} + node_ids = {node: node_id for node_id, node + in enumerate(checkpointable_objects)} + slot_variables = _serialize_slot_variables( + checkpointable_objects=checkpointable_objects, + node_ids=node_ids, + object_names=object_names) + return _serialize_checkpointables( + checkpointable_objects=checkpointable_objects, + node_ids=node_ids, + object_names=object_names, + slot_variables=slot_variables) + + +def gather_initializers(root_checkpointable): + """Traverse the object graph and find initialization ops. + + Looks for `Checkpointable` objects which are dependencies of + `root_checkpointable` and which have an `initializer` property. Includes + initializers for slot variables only if the variable they are slotting for and + the optimizer are dependencies of `root_checkpointable` (i.e. if they would be + saved with a checkpoint). + + Args: + root_checkpointable: A `Checkpointable` object to gather initializers for. + Returns: + A list of initialization ops. + """ + # TODO(allenl): Extract out gathering logic so the naming logic doesn't have + # to run. + checkpointable_objects, path_to_root = ( + _breadth_first_checkpointable_traversal(root_checkpointable)) + object_names = { + obj: _object_prefix_from_path(path) + for obj, path in path_to_root.items()} + node_ids = {node: node_id for node_id, node + in enumerate(checkpointable_objects)} + _serialize_slot_variables( + checkpointable_objects=checkpointable_objects, + node_ids=node_ids, + object_names=object_names) + return [c.initializer for c in checkpointable_objects + if hasattr(c, "initializer") and c.initializer is not None] + + +class _NoRestoreSaveable(saver_lib.BaseSaverBuilder.SaveableObject): + + def __init__(self, tensor, name): + spec = saver_lib.BaseSaverBuilder.SaveSpec(tensor, "", name) + super(_NoRestoreSaveable, self).__init__(tensor, [spec], name) + + def restore(self, restored_tensors, restored_shapes): + return control_flow_ops.no_op() + + +class _LoadStatus(object): + """Abstract base for load status callbacks.""" + + @abc.abstractmethod + def assert_consumed(self): + """Raises an exception unless a non-trivial restoration has completed.""" + pass + + @abc.abstractmethod + def run_restore_ops(self, session=None): + """Runs restore ops from the checkpoint. Requires a valid checkpoint.""" + pass + + @abc.abstractmethod + def initialize_or_restore(self, session=None): + """Runs restore ops from the checkpoint, or initializes variables.""" + pass + + +class CheckpointLoadStatus(_LoadStatus): + """Checks the status of checkpoint loading and manages restore ops. + + Returned from `Saver.restore`. Since `restore` may defer the loading of values + in the checkpoint which don't yet have corresponding Python objects, + `CheckpointLoadStatus` provides a callback to verify that checkpoint loading + is complete (`assert_consumed`). + + When graph building, `restore` does not run restore ops itself since their + creation may be deferred. The `run_restore_ops` method must be called once all + Python objects with values to restore have been created and added to the + dependency graph (this does not necessarily have to be the whole checkpoint; + calling `run_restore_ops` while `assert_consumed` fails is supported and will + partially restore the checkpoint). + + See `Saver.restore` for usage examples. + """ + + def __init__(self, checkpoint, feed_dict): + self._checkpoint = checkpoint + self._feed_dict = feed_dict + + def assert_consumed(self): + """Asserts that all objects in the checkpoint have been created/matched. + + Returns: + `self` for chaining. + Raises: + AssertionError: If there are any Python objects in the dependency graph + which have not been restored from this checkpoint or a later `restore`, + or if there are any checkpointed values which have not been matched to + Python objects. + """ + for node_id, node in enumerate(self._checkpoint.object_graph_proto.nodes): + checkpointable = self._checkpoint.object_by_proto_id.get(node_id, None) + if checkpointable is None: + raise AssertionError("Unresolved object in checkpoint: %s" % (node,)) + if checkpointable._update_uid < self._checkpoint.restore_uid: # pylint: disable=protected-access + raise AssertionError( + "Object not assigned a value from checkpoint: %s" % (node,)) + if self._checkpoint.slot_restorations: + # Sanity check; this collection should be clear if everything has been + # restored. + raise AssertionError("Unresolved slot restorations: %s" % ( + self._checkpoint.slot_restorations,)) + if self._checkpoint.unused_attributes: + raise AssertionError( + ("Unused attributes in these objects (the attributes exist in the " + "checkpoint but not in the objects): %s") % ( + self._checkpoint.unused_attributes.items(),)) + return self + + def run_restore_ops(self, session=None): + """Run operations to restore objects in the dependency graph.""" + if context.executing_eagerly(): + return # Run eagerly + if session is None: + session = ops.get_default_session() + session.run(self._checkpoint.restore_ops, feed_dict=self._feed_dict) + + def initialize_or_restore(self, session=None): + """Alias for `run_restore_ops`. + + This method has a sibling in `InitializationOnlyStatus` which instead + initializes variables. That type is returned if no checkpoint is specified + in `Saver.restore`. + + Args: + session: The session to run restore ops in. If `None`, uses the default + session. + """ + self.run_restore_ops(session=session) + + +class InitializationOnlyStatus(_LoadStatus): + """Returned from `Saver.restore` when no checkpoint has been specified. + + Objects of this type have the same `assert_consumed` method as + `CheckpointLoadStatus`, but it always fails. However, + `initialize_or_restore` works on objects of both types, and will + initialize variables in `InitializationOnlyStatus` objects or restore them + otherwise. + """ + + def __init__(self, root_checkpointable): + self._root_checkpointable = root_checkpointable + + def assert_consumed(self): + """Assertion for consistency with `CheckpointLoadStatus`. Always fails.""" + raise AssertionError( + "No checkpoint specified (save_path=None); nothing is being restored.") + + def run_restore_ops(self, session=None): + """For consistency with `CheckpointLoadStatus`. + + Use `initialize_or_restore` for initializing if no checkpoint was passed + to `Saver.restore` and restoring otherwise. + + Args: + session: Not used. + """ + raise AssertionError( + "No checkpoint specified, so no restore ops are available " + "(save_path=None to Saver.restore).") + + def initialize_or_restore(self, session=None): + """Runs initialization ops for variables. + + Only objects which would be saved by `Saver.save` will be initialized. See + `gather_initializers` for details. + + This method does nothing when executing eagerly (initializers get run + eagerly). + + Args: + session: The session to run initialization ops in. If `None`, uses the + default session. + """ + if context.executing_eagerly(): + return # run eagerly + if session is None: + session = ops.get_default_session() + session.run(gather_initializers(self._root_checkpointable)) + + +_DEPRECATED_RESTORE_INSTRUCTIONS = ( + "Restoring a name-based tf.train.Saver checkpoint using the object-based " + "restore API. This mode uses global names to match variables, and so is " + "somewhat fragile. It also adds new restore ops to the graph each time it " + "is called. Prefer re-encoding training checkpoints in the object-based " + "format: run save() on the object-based saver (the same one this message " + "is coming from) and use that checkpoint in the future.") + + +class NameBasedSaverStatus(_LoadStatus): + """Status for loading a name-based training checkpoint.""" + + def __init__(self, object_saver, save_path): + self._object_saver = object_saver + self._save_path = save_path + + def assert_consumed(self): + """Assertion for consistency with `CheckpointLoadStatus`. Always fails.""" + raise AssertionError( + "Restoring a name-based checkpoint. No load status is available.") + + @deprecation.deprecated( + date=None, instructions=_DEPRECATED_RESTORE_INSTRUCTIONS) + def run_restore_ops(self, session=None): + """Load the name-based training checkpoint using a new `tf.train.Saver`.""" + if session is None and not context.executing_eagerly(): + session = ops.get_default_session() + with ops.device("/cpu:0"): + saver_lib.Saver(self._object_saver._global_variable_names()).restore( # pylint: disable=protected-access + sess=session, save_path=self._save_path) + + def initialize_or_restore(self, session=None): + """Alias for `run_restore_ops`.""" + self.run_restore_ops(session=session) + + +class _SessionWithFeedDictAdditions(session_lib.SessionInterface): + """Pretends to be a session, inserts extra feeds on run().""" + + def __init__(self, session, feed_additions): + self._wrapped_session = session + self._feed_additions = feed_additions + + def run(self, fetches, feed_dict=None, **kwargs): + if feed_dict is None: + feed_dict = {} + else: + feed_dict = feed_dict.copy() + feed_dict.update(self._feed_additions) + return self._wrapped_session.run( + fetches=fetches, feed_dict=feed_dict, **kwargs) + + +def _copy_saver_with_new_var_list(old_saver, new_var_list): + """Copy a `tf.train.Saver`'s state to a new Saver with different variables.""" + new_saver = saver_lib.Saver(var_list=new_var_list) + # TODO(allenl): Move to copying functionality to Saver? + # pylint: disable=protected-access + new_saver._last_checkpoints = old_saver._last_checkpoints + new_saver._checkpoints_to_be_deleted = old_saver._checkpoints_to_be_deleted + new_saver._next_checkpoint_time = old_saver._next_checkpoint_time + # pylint: enable=protected-access + return new_saver + + +class CheckpointableSaver(object): + """Saves and restores a `Checkpointable` object and its dependencies. + + See `Checkpointable` for details of dependency management. `Saver` wraps + `tf.train.Saver` for saving, including extra information about the graph of + dependencies between Python objects. When restoring, it uses this information + about the save-time dependency graph to more robustly match objects with their + checkpointed values. When executing eagerly, it supports restoring variables + on object creation (see `Saver.restore`). + + Values in a checkpoint are mapped to `Checkpointable` Python objects + (`Variable`s, `Optimizer`s, `Layer`s) based on the names provided when the + checkpoint was written. To avoid breaking existing checkpoints when modifying + a class, dependency names (the names of attributes to which `Checkpointable` + objects are assigned) may not change. These names are local to objects, in + contrast to the `Variable.name`-based save/restore from `tf.train.Saver`, and + so allow additional program transformations. + """ + + def __init__(self, root_checkpointable): + """Configure saving. + + Args: + root_checkpointable: The root of the object graph to save/restore. This + object and all of its dependencies are saved in the checkpoint. When + restoring, objects are matched and restored starting from this root. + """ + # Allow passing in a weak reference to avoid reference cycles when + # `Checkpointable` objects save themselves. + self._root_checkpointable_ref = root_checkpointable + if not context.executing_eagerly(): + with ops.device("/cpu:0"): + self._file_prefix_placeholder = constant_op.constant("model") + else: + self._file_prefix_placeholder = None + + # Op caching for save + self._object_graph_feed_tensor = None + self._last_save_object_graph = None + self._last_save_saver = None + + # Op caching for restore + self._last_restore_object_graph = None + self._last_restore_checkpoint = None + + @property + def _root_checkpointable(self): + if isinstance(self._root_checkpointable_ref, weakref.ref): + derefed = self._root_checkpointable_ref() + assert derefed is not None + return derefed + else: + return self._root_checkpointable_ref + + def save(self, file_prefix, checkpoint_number=None, session=None): + """Save a training checkpoint. + + The saved checkpoint includes variables created by this object and any + Checkpointable objects it depends on at the time `Saver.save()` is called. + + Args: + file_prefix: A prefix to use for the checkpoint filenames + (/path/to/directory/and_a_prefix). Names are generated based on this + prefix and `checkpoint_number`, if provided. + checkpoint_number: An integer variable or Tensor, used to number + checkpoints. Typically this value is saved along with other variables in + training checkpoints, which will happen automatically if it was created + by `root_checkpointable` or one of its dependencies (via + `Checkpointable._add_variable`). + session: The session to evaluate variables in. Ignored when executing + eagerly. If not provided when graph building, the default session is + used. + + Returns: + The full path to the checkpoint. + """ + named_variables, graph_proto = _serialize_object_graph( + self._root_checkpointable) + if not context.executing_eagerly(): + if session is None: + session = ops.get_default_session() + if self._object_graph_feed_tensor is None: + with ops.device("/cpu:0"): + self._object_graph_feed_tensor = constant_op.constant( + "", dtype=dtypes.string) + object_graph_tensor = self._object_graph_feed_tensor + feed_additions = {object_graph_tensor: graph_proto.SerializeToString()} + else: + session = None + with ops.device("/cpu:0"): + object_graph_tensor = constant_op.constant( + graph_proto.SerializeToString(), dtype=dtypes.string) + feed_additions = None + assert _OBJECT_GRAPH_PROTO_KEY not in named_variables + named_variables[_OBJECT_GRAPH_PROTO_KEY] = _NoRestoreSaveable( + tensor=object_graph_tensor, + name=_OBJECT_GRAPH_PROTO_KEY) + if (self._last_save_object_graph != graph_proto + # When executing eagerly, we need to re-create SaveableObjects each time + # save() is called so they pick up new Tensors passed to their + # constructors. That means the Saver needs to be copied with a new + # var_list. + or context.executing_eagerly()): + if self._last_save_object_graph is not None: + self._last_save_saver = _copy_saver_with_new_var_list( + old_saver=self._last_save_saver, new_var_list=named_variables) + else: + self._last_save_saver = saver_lib.Saver(var_list=named_variables) + self._last_save_object_graph = graph_proto + with ops.device("/cpu:0"): + save_path = self._last_save_saver.save( + sess=_SessionWithFeedDictAdditions( + session=session, feed_additions=feed_additions), + save_path=file_prefix, + write_meta_graph=False, + global_step=checkpoint_number) + return save_path + + def _global_variable_names(self): + """Generate a `tf.train.Saver`-style `var_list` using `variable.name`s.""" + named_saveables, graph_proto = _serialize_object_graph( + self._root_checkpointable) + saver_names = {} + for object_proto in graph_proto.nodes: + for attribute_proto in object_proto.attributes: + saver_names[attribute_proto.full_name] = named_saveables[ + attribute_proto.checkpoint_key] + return saver_names + + def restore(self, save_path): + """Restore a training checkpoint. + + Restores `root_checkpointable` and any objects that it tracks + (transitive). Either assigns values immediately if variables to restore have + been created already, or defers restoration until the variables are + created. Dependencies added to the `root_checkpointable` passed to the + constructor after this call will be matched if they have a corresponding + object in the checkpoint. + + When building a graph, restorations are added to the graph but not run. + + To disallow deferred loading, assert immediately that all checkpointed + variables have been matched to variable objects: + + ```python + saver = Saver(root) + saver.restore(path).assert_consumed() + ``` + + An exception will be raised unless every object was matched and its + variables already exist. + + When graph building, `assert_consumed()` indicates that all of the restore + ops which will be created for this checkpoint have been created. They can be + run via the `run_restore_ops()` function of the status object: + + ```python + saver.restore(path).assert_consumed().run_restore_ops() + ``` + + If the checkpoint has not been consumed completely, then the list of restore + ops will grow as more objects are added to the dependency graph. + + Name-based `tf.train.Saver` checkpoints can be loaded using this + method. There is no deferred loading, and names are used to match + variables. No restore ops are created/run until `run_restore_ops()` or + `initialize_or_restore()` are called on the returned status object, even + when executing eagerly. Re-encode name-based checkpoints using this + object-based `Saver.save` as soon as possible. + + Args: + save_path: The path to the checkpoint, as returned by `save` or + `tf.train.latest_checkpoint`. If None (as when there is no latest + checkpoint for `tf.train.latest_checkpoint` to return), returns an + object which may run initializers for objects in the dependency + graph. If the checkpoint was written by the name-based `tf.train.Saver`, + names are used to match variables. + + Returns: + A load status object, which can be used to make assertions about the + status of checkpoint restoration and run initialization/restore ops + (of type `CheckpointLoadStatus`, or `InitializationOnlyStatus` if + `save_path` is `None`). + + If `save_path` points to a name-based checkpoint, a `NameBasedSaverStatus` + object is returned which runs restore ops from a name-based saver. + """ + if save_path is None: + return InitializationOnlyStatus(self._root_checkpointable) + in_graph_mode = not context.executing_eagerly() + if in_graph_mode: + file_prefix_tensor = self._file_prefix_placeholder + file_prefix_feed_dict = {self._file_prefix_placeholder: save_path} + else: + with ops.device("/cpu:0"): + file_prefix_tensor = constant_op.constant(save_path) + file_prefix_feed_dict = None + reader = pywrap_tensorflow.NewCheckpointReader(save_path) + try: + object_graph_string = reader.get_tensor(_OBJECT_GRAPH_PROTO_KEY) + except errors_impl.NotFoundError: + # The object graph proto does not exist in this checkpoint. Try again with + # name-based saving. + return NameBasedSaverStatus(self, save_path) + + object_graph_proto = ( + checkpointable_object_graph_pb2.CheckpointableObjectGraph()) + object_graph_proto.ParseFromString(object_graph_string) + if in_graph_mode and object_graph_proto == self._last_restore_object_graph: + checkpoint = self._last_restore_checkpoint + else: + if in_graph_mode: + dtype_map = None + else: + dtype_map = reader.get_variable_to_dtype_map() + checkpoint = _CheckpointRestoreCoordinator( + object_graph_proto=object_graph_proto, + save_path=file_prefix_tensor, + dtype_map=dtype_map) + if in_graph_mode: + if self._last_restore_object_graph is not None: + raise NotImplementedError( + "Using a single Saver to restore different object graphs is not " + "currently supported when graph building. Use a different Saver " + "for each object graph (restore ops will be duplicated), or " + "file a feature request if this limitation bothers you.") + self._last_restore_checkpoint = checkpoint + self._last_restore_object_graph = object_graph_proto + checkpointable_lib._CheckpointPosition( # pylint: disable=protected-access + checkpoint=checkpoint, proto_id=0).restore(self._root_checkpointable) + load_status = CheckpointLoadStatus( + checkpoint, feed_dict=file_prefix_feed_dict) + return load_status + + +class Checkpoint(checkpointable_lib.Checkpointable): + """A utility class which groups `Checkpointable` objects. + + Accepts arbitrary keyword arguments to its constructor and saves those values + with a checkpoint. Maintains a `save_counter` for numbering checkpoints. + + Example usage: + + ```python + import tensorflow as tf + import tensorflow.contrib.eager as tfe + import os + + checkpoint_directory = "/tmp/training_checkpoints" + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + + root = tfe.Checkpoint(optimizer=optimizer, model=model) + root.restore(tf.train.latest_checkpoint(checkpoint_directory)) + for _ in range(num_training_steps): + optimizer.minimize( ... ) + root.save(file_prefix=checkpoint_prefix) + ``` + + For more manual control over saving, use `tfe.CheckpointableSaver` directly. + + Attributes: + save_counter: Incremented when `save()` is called. Used to number + checkpoints. + """ + + def __init__(self, **kwargs): + """Group objects into a training checkpoint. + + Args: + **kwargs: Keyword arguments are set as attributes of this object, and are + saved with the checkpoint. Attribute values must derive from + `CheckpointableBase`. + Raises: + ValueError: If objects in `kwargs` are not Checkpointable. + """ + super(Checkpoint, self).__init__() + for k, v in sorted(kwargs.items(), key=lambda item: item[0]): + if not isinstance(v, checkpointable_lib.CheckpointableBase): + raise ValueError( + ("`Checkpoint` was expecting an object derived from " + "`CheckpointableBase`, got %s.") % (v,)) + setattr(self, k, v) + self._save_counter = None # Created lazily for restore-on-create. + self._saver = CheckpointableSaver(weakref.ref(self)) + + def _maybe_create_save_counter(self): + """Create a save counter if it does not yet exist.""" + if self._save_counter is None: + # Initialized to 0 and incremented before saving. + with ops.device("/cpu:0"): + self._save_counter = add_variable( + self, name="save_counter", initializer=0, dtype=dtypes.int64) + + @property + def save_counter(self): + """An integer variable which starts at zero and is incremented on save. + + Used to number checkpoints. + + Returns: + The save counter variable. + """ + self._maybe_create_save_counter() + return self._save_counter + + def save(self, file_prefix, session=None): + """Save a checkpoint. Wraps `tfe.CheckpointableSaver.save`.""" + in_graph_mode = not context.executing_eagerly() + if in_graph_mode: + if session is None: + session = ops.get_default_session() + if self._save_counter is None: + # When graph building, if this is a new save counter variable then it + # needs to be initialized before assign_add. This is only an issue if + # restore() has not been called first. + session.run(self.save_counter.initializer) + with ops.colocate_with(self.save_counter): + assign_op = self.save_counter.assign_add(1) + if in_graph_mode: + session.run(assign_op) + return self._saver.save( + file_prefix=file_prefix, + checkpoint_number=self.save_counter, + session=session) + + def restore(self, save_path): + """Restore a checkpoint. Wraps `tfe.CheckpointableSaver.restore`.""" + status = self._saver.restore(save_path=save_path) + # Create the save counter now so it gets initialized with other variables + # when graph building. Creating it earlier would lead to double + # initialization when executing eagerly. + self._maybe_create_save_counter() + return status diff --git a/tensorflow/python/training/checkpointable_utils_test.py b/tensorflow/python/training/checkpointable_utils_test.py new file mode 100644 index 0000000000..ddf9820616 --- /dev/null +++ b/tensorflow/python/training/checkpointable_utils_test.py @@ -0,0 +1,1308 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools +import os + +import six + +from tensorflow.python.client import session as session_lib +from tensorflow.python.eager import backprop +from tensorflow.python.eager import context +from tensorflow.python.eager import function +from tensorflow.python.eager import test +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util +from tensorflow.python.keras._impl.keras.engine import sequential +from tensorflow.python.keras._impl.keras.engine import training +from tensorflow.python.keras._impl.keras.layers import core +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import state_ops +from tensorflow.python.ops import template +from tensorflow.python.ops import variable_scope +from tensorflow.python.training import adam +from tensorflow.python.training import checkpointable +from tensorflow.python.training import checkpointable_utils +from tensorflow.python.training import saver as saver_lib +from tensorflow.python.training import training_util + + +class NonLayerCheckpointable(checkpointable.Checkpointable): + + def __init__(self): + super(NonLayerCheckpointable, self).__init__() + self.a_variable = checkpointable_utils.add_variable( + self, name="a_variable", shape=[]) + + +# pylint: disable=not-callable +class MyModel(training.Model): + """A concrete Model for testing.""" + + def __init__(self): + super(MyModel, self).__init__() + self._named_dense = core.Dense(1, use_bias=True) + self._second = core.Dense(1, use_bias=False) + # We can still track Checkpointables which aren't Layers. + self._non_layer = NonLayerCheckpointable() + + def call(self, values): + ret = self._second(self._named_dense(values)) + return ret + + +class InterfaceTests(test.TestCase): + + @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) + def testAddVariable(self): + obj = NonLayerCheckpointable() + with self.assertRaisesRegexp(ValueError, "do not specify shape"): + checkpointable_utils.add_variable( + obj, name="shape_specified_twice", shape=[], initializer=1) + constant_initializer = checkpointable_utils.add_variable( + obj, name="constant_initializer", initializer=1) + with variable_scope.variable_scope("some_variable_scope"): + ones_initializer = checkpointable_utils.add_variable( + obj, + name="ones_initializer", + shape=[2], + initializer=init_ops.ones_initializer(dtype=dtypes.float32)) + bare_initializer = checkpointable_utils.add_variable( + obj, + name="bare_initializer", + shape=[2, 2], + dtype=dtypes.float64, + initializer=init_ops.zeros_initializer) + + # Even in graph mode, there are no naming conflicts between objects, only + # naming conflicts within an object. + other_duplicate = resource_variable_ops.ResourceVariable( + name="duplicate", initial_value=1.) + duplicate = checkpointable_utils.add_variable( + obj, name="duplicate", shape=[]) + with self.assertRaisesRegexp(ValueError, "'duplicate' already exists"): + checkpointable_utils.add_variable(obj, name="duplicate", shape=[]) + + self.evaluate(checkpointable_utils.gather_initializers(obj)) + self.assertEqual("constant_initializer:0", constant_initializer.name) + self.assertEqual(1, self.evaluate(constant_initializer)) + self.assertEqual("some_variable_scope/ones_initializer:0", + ones_initializer.name) + self.assertAllEqual([1, 1], self.evaluate(ones_initializer)) + self.assertAllEqual([[0., 0.], + [0., 0.]], self.evaluate(bare_initializer)) + self.assertEqual("a_variable:0", obj.a_variable.name) + self.assertEqual("duplicate:0", other_duplicate.name) + if context.executing_eagerly(): + # When executing eagerly, there's no uniquification of variable names. The + # checkpoint name will be the same. + self.assertEqual("duplicate:0", duplicate.name) + else: + # The .name attribute may be globally influenced, but the checkpoint name + # won't be (tested below). + self.assertEqual("duplicate_1:0", duplicate.name) + named_variables, _ = checkpointable_utils._serialize_object_graph(obj) + expected_checkpoint_names = ( + "a_variable/.ATTRIBUTES/VARIABLE_VALUE", + "bare_initializer/.ATTRIBUTES/VARIABLE_VALUE", + "constant_initializer/.ATTRIBUTES/VARIABLE_VALUE", + "duplicate/.ATTRIBUTES/VARIABLE_VALUE", + "ones_initializer/.ATTRIBUTES/VARIABLE_VALUE", + ) + six.assertCountEqual( + self, expected_checkpoint_names, named_variables.keys()) + + def testInitNotCalled(self): + + class NoInit(checkpointable.Checkpointable): + + def __init__(self): + pass + + # __init__ for Checkpointable will be called implicitly. + checkpointable_utils.add_variable(NoInit(), "var", shape=[]) + + def testShapeDtype(self): + root = checkpointable.Checkpointable() + v1 = checkpointable_utils.add_variable( + root, name="v1", initializer=3., dtype=dtypes.float64) + self.assertEqual(dtypes.float64, v1.dtype) + v2 = checkpointable_utils.add_variable( + root, + name="v2", + shape=[3], + initializer=init_ops.ones_initializer, + dtype=dtypes.float64) + self.assertEqual(dtypes.float64, v2.dtype) + self.assertAllEqual([1., 1., 1.], self.evaluate(v2)) + + +class _MirroringSaveable(saver_lib.BaseSaverBuilder.SaveableObject): + + def __init__(self, primary_variable, mirrored_variable, name): + self._primary_variable = primary_variable + self._mirrored_variable = mirrored_variable + tensor = self._primary_variable.read_value() + spec = saver_lib.BaseSaverBuilder.SaveSpec( + tensor=tensor, + slice_spec="", + name=name) + super(_MirroringSaveable, self).__init__( + tensor, [spec], name) + + def restore(self, restored_tensors, restored_shapes): + """Restore the same value into both variables.""" + tensor, = restored_tensors + return control_flow_ops.group( + self._primary_variable.assign(tensor), + self._mirrored_variable.assign(tensor)) + + +class _OwnsMirroredVariables(checkpointable.CheckpointableBase): + """A Checkpointable object which returns a more complex SaveableObject.""" + + def __init__(self): + self.non_dep_variable = variable_scope.get_variable( + name="non_dep_variable", initializer=6., use_resource=True) + self.mirrored = variable_scope.get_variable( + name="mirrored", initializer=15., use_resource=True) + + def _gather_saveables_for_checkpoint(self): + def _saveable_factory(name=self.non_dep_variable.name): + return _MirroringSaveable( + primary_variable=self.non_dep_variable, + mirrored_variable=self.mirrored, + name=name) + return {checkpointable.VARIABLE_VALUE_KEY: _saveable_factory} + + # The Saver sorts by name before parsing, so we need a name property. + @property + def name(self): + return self.non_dep_variable.name + + +class CheckpointingTests(test.TestCase): + + @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) + def testNamingWithOptimizer(self): + input_value = constant_op.constant([[3.]]) + model = MyModel() + # A nuisance Model using the same optimizer. Its slot variables should not + # go in the checkpoint, since it is never depended on. + other_model = MyModel() + optimizer = adam.AdamOptimizer(0.001) + optimizer_step = training_util.get_or_create_global_step() + root_checkpointable = checkpointable_utils.Checkpoint( + optimizer=optimizer, model=model, optimizer_step=optimizer_step) + if context.executing_eagerly(): + optimizer.minimize( + lambda: model(input_value), + global_step=optimizer_step) + optimizer.minimize( + lambda: other_model(input_value), + global_step=optimizer_step) + else: + train_op = optimizer.minimize( + model(input_value), global_step=optimizer_step) + optimizer.minimize( + other_model(input_value), + global_step=optimizer_step) + self.evaluate(checkpointable_utils.gather_initializers( + root_checkpointable)) + self.evaluate(train_op) + named_variables, serialized_graph = ( + checkpointable_utils._serialize_object_graph(root_checkpointable)) + expected_checkpoint_names = ( + # Created in the root node, so no prefix. + "optimizer_step", + "model/_second/kernel", + "model/_named_dense/kernel", + "model/_named_dense/bias", + # non-Layer dependency of the model + "model/_non_layer/a_variable", + # The optimizer creates two non-slot variables + "optimizer/beta1_power", + "optimizer/beta2_power", + # Slot variables + "model/_second/kernel/.OPTIMIZER_SLOT/optimizer/m", + "model/_second/kernel/.OPTIMIZER_SLOT/optimizer/v", + "model/_named_dense/kernel/.OPTIMIZER_SLOT/optimizer/m", + "model/_named_dense/kernel/.OPTIMIZER_SLOT/optimizer/v", + "model/_named_dense/bias/.OPTIMIZER_SLOT/optimizer/m", + "model/_named_dense/bias/.OPTIMIZER_SLOT/optimizer/v", + ) + suffix = "/.ATTRIBUTES/VARIABLE_VALUE" + expected_checkpoint_names = [ + name + suffix for name in expected_checkpoint_names] + six.assertCountEqual(self, expected_checkpoint_names, + named_variables.keys()) + # Check that we've mapped to the right variable objects (not exhaustive) + self.assertEqual( + "global_step:0", + named_variables["optimizer_step" + suffix].name) + self.assertEqual( + "my_model/dense_1/kernel:0", + named_variables["model/_second/kernel" + suffix].name) + self.assertEqual( + "my_model/dense/kernel:0", + named_variables["model/_named_dense/kernel" + suffix].name) + self.assertEqual( + "beta1_power:0", + named_variables["optimizer/beta1_power" + suffix].name) + self.assertEqual( + "beta2_power:0", + named_variables["optimizer/beta2_power" + suffix].name) + # Spot check the generated protocol buffers. + self.assertEqual("optimizer", + serialized_graph.nodes[0].children[1].local_name) + optimizer_node = serialized_graph.nodes[serialized_graph.nodes[0].children[ + 1].node_id] + self.assertEqual("beta1_power", + optimizer_node.children[0].local_name) + self.assertEqual("beta1_power", + serialized_graph.nodes[optimizer_node.children[0].node_id] + .attributes[0].full_name) + self.assertEqual( + "my_model/dense/kernel", + serialized_graph.nodes[optimizer_node.slot_variables[0] + .original_variable_node_id] + .attributes[0].full_name) + # We strip off the :0 suffix, as variable.name-based saving does. + self.assertEqual( + "my_model/dense/kernel/Adam", + serialized_graph.nodes[optimizer_node.slot_variables[0] + .slot_variable_node_id] + .attributes[0].full_name) + self.assertEqual( + "my_model/dense/kernel/Adam:0", + optimizer.get_slot( + var=named_variables["model/_named_dense/kernel" + suffix], + name="m").name) + self.assertEqual( + "model/_named_dense/kernel" + suffix, + serialized_graph.nodes[ + optimizer_node.slot_variables[0] + .original_variable_node_id].attributes[0].checkpoint_key) + self.assertEqual("m", optimizer_node.slot_variables[0].slot_name) + self.assertEqual( + "model/_named_dense/kernel/.OPTIMIZER_SLOT/optimizer/m" + suffix, + serialized_graph.nodes[ + optimizer_node.slot_variables[0] + .slot_variable_node_id].attributes[0].checkpoint_key) + + @test_util.run_in_graph_and_eager_modes() + def testMoreComplexSaveableReturned(self): + v = _OwnsMirroredVariables() + checkpoint = checkpointable_utils.Checkpoint(v=v) + test_dir = self.get_temp_dir() + prefix = os.path.join(test_dir, "ckpt") + self.evaluate(v.non_dep_variable.assign(42.)) + save_path = checkpoint.save(prefix) + self.evaluate(v.non_dep_variable.assign(43.)) + self.evaluate(v.mirrored.assign(44.)) + checkpoint.restore(save_path).assert_consumed().initialize_or_restore() + self.assertEqual(42., self.evaluate(v.non_dep_variable)) + self.assertEqual(42., self.evaluate(v.mirrored)) + self.evaluate(v.non_dep_variable.assign(44.)) + save_path = checkpoint.save(prefix) + self.evaluate(v.non_dep_variable.assign(45.)) + checkpoint.restore(save_path).assert_consumed().initialize_or_restore() + self.assertEqual(44., self.evaluate(v.non_dep_variable)) + self.assertEqual(44., self.evaluate(v.mirrored)) + + @test_util.run_in_graph_and_eager_modes() + def testMoreComplexSaveableReturnedWithGlobalName(self): + # The same object can also be saved using the name-based saver. + v = _OwnsMirroredVariables() + saver = saver_lib.Saver(var_list=[v]) + test_dir = self.get_temp_dir() + prefix = os.path.join(test_dir, "ckpt") + self.evaluate(v.non_dep_variable.assign(42.)) + with self.test_session() as sess: + save_path = saver.save(sess, prefix) + self.evaluate(v.non_dep_variable.assign(43.)) + self.evaluate(v.mirrored.assign(44.)) + saver.restore(sess, save_path) + self.assertEqual(42., self.evaluate(v.non_dep_variable)) + self.assertEqual(42., self.evaluate(v.mirrored)) + + @test_util.run_in_graph_and_eager_modes() + def testSaveRestore(self): + model = MyModel() + optimizer = adam.AdamOptimizer(0.001) + root_checkpointable = checkpointable_utils.Checkpoint( + optimizer=optimizer, model=model) + input_value = constant_op.constant([[3.]]) + if context.executing_eagerly(): + optimizer.minimize( + lambda: model(input_value)) + else: + train_op = optimizer.minimize(model(input_value)) + # TODO(allenl): Make initialization more pleasant when graph building. + root_checkpointable.save_counter # pylint: disable=pointless-statement + self.evaluate(checkpointable_utils.gather_initializers( + root_checkpointable)) + self.evaluate(train_op) + prefix = os.path.join(self.get_temp_dir(), "ckpt") + self.evaluate(state_ops.assign(model._named_dense.variables[1], [42.])) + m_bias_slot = optimizer.get_slot(model._named_dense.variables[1], "m") + self.evaluate(state_ops.assign(m_bias_slot, [1.5])) + save_path = root_checkpointable.save(file_prefix=prefix) + self.evaluate(state_ops.assign(model._named_dense.variables[1], [43.])) + self.evaluate(state_ops.assign(root_checkpointable.save_counter, 3)) + optimizer_variables = self.evaluate(optimizer.variables()) + self.evaluate(state_ops.assign(m_bias_slot, [-2.])) + # Immediate restoration + status = root_checkpointable.restore(save_path=save_path).assert_consumed() + status.run_restore_ops() + self.assertAllEqual([42.], self.evaluate(model._named_dense.variables[1])) + self.assertAllEqual(1, self.evaluate(root_checkpointable.save_counter)) + self.assertAllEqual([1.5], self.evaluate(m_bias_slot)) + if not context.executing_eagerly(): + return # Restore-on-create is only supported when executing eagerly + on_create_model = MyModel() + on_create_optimizer = adam.AdamOptimizer( + 0.001, + # Preserve beta1_power and beta2_power when appying gradients so we can + # test that they've been restored correctly. + beta1=1.0, beta2=1.0) + on_create_root = checkpointable_utils.Checkpoint( + optimizer=on_create_optimizer, model=on_create_model) + # Deferred restoration + status = on_create_root.restore(save_path=save_path) + on_create_model(constant_op.constant([[3.]])) # create variables + self.assertAllEqual(1, self.evaluate(on_create_root.save_counter)) + self.assertAllEqual([42.], + self.evaluate( + on_create_model._named_dense.variables[1])) + on_create_m_bias_slot = on_create_optimizer.get_slot( + on_create_model._named_dense.variables[1], "m") + # Optimizer slot variables are created when the original variable is + # restored. + self.assertAllEqual([1.5], self.evaluate(on_create_m_bias_slot)) + self.assertAllEqual(optimizer_variables[2:], + self.evaluate(on_create_optimizer.variables())) + dummy_var = resource_variable_ops.ResourceVariable([1.]) + on_create_optimizer.minimize(loss=dummy_var.read_value) + status.assert_consumed() + beta1_power, beta2_power = on_create_optimizer._get_beta_accumulators() + self.assertAllEqual(optimizer_variables[0], self.evaluate(beta1_power)) + self.assertAllEqual(optimizer_variables[1], self.evaluate(beta2_power)) + + # TODO(allenl): Debug garbage created by this test in python3. + def testDeferredRestorationUsageEager(self): + """An idiomatic eager execution example.""" + num_training_steps = 10 + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + for training_continuation in range(3): + model = MyModel() + optimizer = adam.AdamOptimizer(0.001) + root = checkpointable_utils.Checkpoint( + optimizer=optimizer, model=model, + optimizer_step=training_util.get_or_create_global_step()) + root.restore(saver_lib.latest_checkpoint(checkpoint_directory)) + for _ in range(num_training_steps): + # TODO(allenl): Use a Dataset and serialize/checkpoint it. + input_value = constant_op.constant([[3.]]) + optimizer.minimize( + lambda: model(input_value), # pylint: disable=cell-var-from-loop + global_step=root.optimizer_step) + root.save(file_prefix=checkpoint_prefix) + self.assertEqual((training_continuation + 1) * num_training_steps, + root.optimizer_step.numpy()) + + def testUsageGraph(self): + """Expected usage when graph building.""" + with context.graph_mode(): + num_training_steps = 10 + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + for training_continuation in range(3): + with ops.Graph().as_default(): + model = MyModel() + optimizer = adam.AdamOptimizer(0.001) + root = checkpointable_utils.Checkpoint( + optimizer=optimizer, model=model, + global_step=training_util.get_or_create_global_step()) + input_value = constant_op.constant([[3.]]) + train_op = optimizer.minimize( + model(input_value), + global_step=root.global_step) + checkpoint_path = saver_lib.latest_checkpoint(checkpoint_directory) + with self.test_session(graph=ops.get_default_graph()) as session: + status = root.restore(save_path=checkpoint_path) + status.initialize_or_restore(session=session) + if checkpoint_path is None: + self.assertEqual(0, training_continuation) + with self.assertRaises(AssertionError): + status.assert_consumed() + else: + status.assert_consumed() + for _ in range(num_training_steps): + session.run(train_op) + root.save(file_prefix=checkpoint_prefix, session=session) + self.assertEqual((training_continuation + 1) * num_training_steps, + session.run(root.global_step)) + self.assertEqual(training_continuation + 1, + session.run(root.save_counter)) + + @test_util.run_in_graph_and_eager_modes() + def testAgnosticUsage(self): + """Graph/eager agnostic usage.""" + # Does create garbage when executing eagerly due to ops.Graph() creation. + num_training_steps = 10 + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + for training_continuation in range(3): + with ops.Graph().as_default(), self.test_session( + graph=ops.get_default_graph()), test_util.device(use_gpu=True): + model = MyModel() + optimizer = adam.AdamOptimizer(0.001) + root = checkpointable_utils.Checkpoint( + optimizer=optimizer, model=model, + global_step=training_util.get_or_create_global_step()) + checkpoint_path = saver_lib.latest_checkpoint(checkpoint_directory) + status = root.restore(save_path=checkpoint_path) + input_value = constant_op.constant([[3.]]) + train_fn = functools.partial( + optimizer.minimize, + functools.partial(model, input_value), + global_step=root.global_step) + if not context.executing_eagerly(): + train_fn = functools.partial(self.evaluate, train_fn()) + status.initialize_or_restore() + for _ in range(num_training_steps): + train_fn() + root.save(file_prefix=checkpoint_prefix) + self.assertEqual((training_continuation + 1) * num_training_steps, + self.evaluate(root.global_step)) + self.assertEqual(training_continuation + 1, + self.evaluate(root.save_counter)) + + # pylint: disable=cell-var-from-loop + @test_util.run_in_graph_and_eager_modes() + def testWithDefun(self): + num_training_steps = 2 + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + for training_continuation in range(3): + with ops.Graph().as_default(), self.test_session( + graph=ops.get_default_graph()), test_util.device(use_gpu=True): + model = MyModel() + # Don't actually train so we can test variable values + optimizer = adam.AdamOptimizer(0.) + root = checkpointable_utils.Checkpoint( + optimizer=optimizer, model=model, + global_step=training_util.get_or_create_global_step()) + checkpoint_path = saver_lib.latest_checkpoint(checkpoint_directory) + status = root.restore(save_path=checkpoint_path) + def train_fn(): + @function.defun + def _call_model(x): + return model(x) + with backprop.GradientTape() as tape: + loss = _call_model(constant_op.constant([[3.]])) + gradients = tape.gradient(loss, model.variables) + return optimizer.apply_gradients(zip(gradients, model.variables), + global_step=root.global_step) + if not context.executing_eagerly(): + train_fn = functools.partial( + self.evaluate, train_fn()) + status.initialize_or_restore() + for _ in range(num_training_steps): + train_fn() + if training_continuation > 0: + status.assert_consumed() + self.assertAllClose([[42.]], self.evaluate(model.variables[0])) + else: + self.evaluate(model.variables[0].assign([[42.]])) + root.save(file_prefix=checkpoint_prefix) + self.assertEqual((training_continuation + 1) * num_training_steps, + self.evaluate(root.global_step)) + self.assertEqual(training_continuation + 1, + self.evaluate(root.save_counter)) + # pylint: enable=cell-var-from-loop + + def _get_checkpoint_name(self, name): + root = checkpointable.Checkpointable() + checkpointable_utils.add_variable( + root, name=name, shape=[1, 2], dtype=dtypes.float64) + named_variables, _ = checkpointable_utils._serialize_object_graph(root) + checkpoint_name, = named_variables.keys() + with ops.name_scope("root/" + checkpoint_name): + pass # Make sure we can use this as an op name if we prefix it. + return checkpoint_name + + @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) + def testVariableNameEscaping(self): + suffix = "/.ATTRIBUTES/VARIABLE_VALUE" + self.assertEqual(r"a.Sb.Sc" + suffix, self._get_checkpoint_name(r"a/b/c")) + self.assertEqual(r"b" + suffix, self._get_checkpoint_name(r"b")) + self.assertEqual(r"c.S" + suffix, self._get_checkpoint_name(r"c/")) + self.assertEqual(r"d.S..S" + suffix, self._get_checkpoint_name(r"d/.S")) + self.assertEqual(r"d.S..ATTRIBUTES.Sf" + suffix, + self._get_checkpoint_name(r"d/.ATTRIBUTES/f")) + + @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) + def testNumberedPath(self): + root = checkpointable.Checkpointable() + leaf = checkpointable.Checkpointable() + root.leaf = leaf + checkpointable_utils.add_variable(leaf, name="v", shape=[]) + named_variables, _ = checkpointable_utils._serialize_object_graph(root) + variable_name, = named_variables.keys() + self.assertEqual(r"leaf/v/.ATTRIBUTES/VARIABLE_VALUE", variable_name) + + @test_util.run_in_graph_and_eager_modes() + def testLocalNameValidation(self): + root = checkpointable.Checkpointable() + leaf = checkpointable.Checkpointable() + # Dots are escaped, which avoids conflicts with reserved names. + root._track_checkpointable(leaf, name=".ATTRIBUTES") + checkpointable_utils.add_variable(checkpointable=leaf, name="a", shape=[]) + named_variables, _ = checkpointable_utils._serialize_object_graph(root) + name, = named_variables.keys() + self.assertEqual(name, "..ATTRIBUTES/a/.ATTRIBUTES/VARIABLE_VALUE") + + def testAnonymousVarsInInit(self): + + class Model(training.Model): + + def __init__(self): + super(Model, self).__init__() + self.w = resource_variable_ops.ResourceVariable(0.0) + self.b = resource_variable_ops.ResourceVariable(0.0) + self.vars = [self.w, self.b] + + def call(self, x): + return x * self.w + self.b + + with context.eager_mode(): + model = Model() + optimizer = adam.AdamOptimizer(learning_rate=0.05) + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + checkpoint = checkpointable_utils.Checkpoint( + model=model, optimizer=optimizer) + for _ in range(2): + checkpoint.save(checkpoint_prefix) + with backprop.GradientTape() as tape: + loss = (constant_op.constant(1.) + - model(constant_op.constant(1.))) ** 2 + grad = tape.gradient(loss, model.vars) + optimizer.apply_gradients( + [(g, v) for g, v in zip(grad, model.vars)]) + + @test_util.run_in_graph_and_eager_modes() + def testLateDependencyTracking(self): + + class Dependency(checkpointable.Checkpointable): + + def build(self): + self.var = checkpointable_utils.add_variable( + self, "var", initializer=0.) + + class LateDependencies(checkpointable.Checkpointable): + + def add_dep(self): + self.dep = Dependency() + self.dep.build() + + original = LateDependencies() + original.add_dep() + self.evaluate(state_ops.assign(original.dep.var, 123.)) + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + save_path = checkpointable_utils.CheckpointableSaver( + original).save(checkpoint_prefix) + load_into = LateDependencies() + status = checkpointable_utils.CheckpointableSaver( + load_into).restore(save_path) + with self.assertRaises(AssertionError): + status.assert_consumed() + load_into.add_dep() + status.assert_consumed() + status.run_restore_ops() + self.assertEqual(123., self.evaluate(load_into.dep.var)) + + @test_util.run_in_graph_and_eager_modes() + def testDepAfterVar(self): + + class Dependency(checkpointable.Checkpointable): + + def build(self): + self.var = checkpointable_utils.add_variable( + self, "var", initializer=0.) + + class DepAfterVar(checkpointable.Checkpointable): + + def add_dep(self): + dep = Dependency() + dep.build() + self.dep = dep + + dep_after_var = DepAfterVar() + dep_after_var.add_dep() + self.evaluate(state_ops.assign(dep_after_var.dep.var, -14.)) + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + save_path = checkpointable_utils.CheckpointableSaver(dep_after_var).save( + checkpoint_prefix) + + loaded_dep_after_var = DepAfterVar() + status = checkpointable_utils.CheckpointableSaver( + loaded_dep_after_var).restore(save_path) + loaded_dep_after_var.add_dep() + status.assert_consumed() + status.run_restore_ops() + self.assertEqual(-14., self.evaluate(loaded_dep_after_var.dep.var)) + + @test_util.run_in_graph_and_eager_modes() + def testDeferredSlotRestoration(self): + checkpoint_directory = self.get_temp_dir() + + root = checkpointable.Checkpointable() + root.var = checkpointable_utils.add_variable( + root, name="var", initializer=0.) + optimizer = adam.AdamOptimizer(0.1) + if context.executing_eagerly(): + optimizer.minimize(root.var.read_value) + else: + train_op = optimizer.minimize(root.var) + # Note that `optimizer` has not been added as a dependency of + # `root`. Create a one-off grouping so that slot variables for `root.var` + # get initialized too. + self.evaluate(checkpointable_utils.gather_initializers( + checkpointable_utils.Checkpoint(root=root, optimizer=optimizer))) + self.evaluate(train_op) + self.evaluate(state_ops.assign(root.var, 12.)) + no_slots_path = checkpointable_utils.CheckpointableSaver(root).save( + os.path.join(checkpoint_directory, "no_slots")) + root.optimizer = optimizer + self.evaluate(state_ops.assign(root.var, 13.)) + self.evaluate(state_ops.assign(optimizer.get_slot(name="m", var=root.var), + 14.)) + slots_path = checkpointable_utils.CheckpointableSaver(root).save( + os.path.join(checkpoint_directory, "with_slots")) + new_root = checkpointable.Checkpointable() + # Load the slot-containing checkpoint (deferred), then immediately overwrite + # the non-slot variable (also deferred). + slot_status = checkpointable_utils.CheckpointableSaver( + new_root).restore(slots_path) + no_slot_status = checkpointable_utils.CheckpointableSaver( + new_root).restore(no_slots_path) + with self.assertRaises(AssertionError): + no_slot_status.assert_consumed() + new_root.var = checkpointable_utils.add_variable( + new_root, name="var", shape=[]) + no_slot_status.assert_consumed() + no_slot_status.run_restore_ops() + self.assertEqual(12., self.evaluate(new_root.var)) + new_root.optimizer = adam.AdamOptimizer(0.1) + with self.assertRaisesRegexp(AssertionError, "beta1_power"): + slot_status.assert_consumed() + self.assertEqual(12., self.evaluate(new_root.var)) + if context.executing_eagerly(): + # Slot variables are only created with restoring initializers when + # executing eagerly. + self.assertEqual(14., self.evaluate( + new_root.optimizer.get_slot(name="m", var=new_root.var))) + else: + self.assertIs(new_root.optimizer.get_slot(name="m", var=new_root.var), + None) + if context.executing_eagerly(): + new_root.optimizer.minimize(new_root.var.read_value) + else: + train_op = new_root.optimizer.minimize(new_root.var) + # The slot variable now exists; restore() didn't create it, but we should + # now have a restore op for it. + slot_status.run_restore_ops() + self.assertEqual(14., self.evaluate( + new_root.optimizer.get_slot(name="m", var=new_root.var))) + self.evaluate(train_op) + slot_status.assert_consumed() + + @test_util.run_in_graph_and_eager_modes() + def testOverlappingRestores(self): + checkpoint_directory = self.get_temp_dir() + save_root = checkpointable.Checkpointable() + save_root.dep = checkpointable.Checkpointable() + save_root.dep.var = checkpointable_utils.add_variable( + save_root.dep, name="var", initializer=0.) + self.evaluate(state_ops.assign(save_root.dep.var, 12.)) + saver = checkpointable_utils.CheckpointableSaver(save_root) + first_path = saver.save(os.path.join(checkpoint_directory, "first")) + self.evaluate(state_ops.assign(save_root.dep.var, 13.)) + second_path = saver.save(os.path.join(checkpoint_directory, "second")) + + first_root = checkpointable.Checkpointable() + second_root = checkpointable.Checkpointable() + first_status = checkpointable_utils.CheckpointableSaver( + first_root).restore(first_path) + second_status = checkpointable_utils.CheckpointableSaver( + second_root).restore(second_path) + load_dep = checkpointable.Checkpointable() + load_dep.var = checkpointable_utils.add_variable( + load_dep, name="var", shape=[]) + first_root.dep = load_dep + first_status.assert_consumed() + first_status.run_restore_ops() + self.assertEqual(12., self.evaluate(load_dep.var)) + second_root.dep = load_dep + second_status.assert_consumed() + second_status.run_restore_ops() + self.assertEqual(13., self.evaluate(load_dep.var)) + + # Try again with the order of the restore() reversed. The last restore + # determines the final value. + first_root = checkpointable.Checkpointable() + second_root = checkpointable.Checkpointable() + second_status = checkpointable_utils.CheckpointableSaver( + second_root).restore(second_path) + first_status = checkpointable_utils.CheckpointableSaver( + first_root).restore(first_path) + load_dep = checkpointable.Checkpointable() + load_dep.var = checkpointable_utils.add_variable( + load_dep, name="var", shape=[]) + first_root.dep = load_dep + first_status.assert_consumed() + first_status.run_restore_ops() + self.assertEqual(12., self.evaluate(load_dep.var)) + second_root.dep = load_dep + second_status.assert_consumed() + second_status.run_restore_ops() + self.assertEqual(12., self.evaluate(load_dep.var)) + + @test_util.run_in_graph_and_eager_modes() + def testAmbiguousLoad(self): + # Not OK to split one checkpoint object into two + checkpoint_directory = self.get_temp_dir() + save_root = checkpointable.Checkpointable() + save_root.dep_one = checkpointable.Checkpointable() + save_root.dep_two = checkpointable.Checkpointable() + dep_three = checkpointable.Checkpointable() + save_root.dep_one.dep_three = dep_three + save_root.dep_two.dep_three = dep_three + checkpointable_utils.add_variable(dep_three, name="var", initializer=0.) + self.evaluate(checkpointable_utils.gather_initializers(save_root)) + save_path = checkpointable_utils.CheckpointableSaver(save_root).save( + os.path.join(checkpoint_directory, "ckpt")) + load_root = checkpointable.Checkpointable() + checkpointable_utils.CheckpointableSaver(load_root).restore(save_path) + load_root.dep_one = checkpointable.Checkpointable() + load_root.dep_two = checkpointable.Checkpointable() + load_root.dep_one.dep_three = checkpointable.Checkpointable() + with self.assertRaisesRegexp(AssertionError, + "resolved to different objects"): + load_root.dep_two.dep_three = checkpointable.Checkpointable() + + @test_util.run_in_graph_and_eager_modes() + def testObjectsCombined(self): + # Currently fine to load two checkpoint objects into one Python object + checkpoint_directory = self.get_temp_dir() + save_root = checkpointable.Checkpointable() + save_root.dep_one = checkpointable.Checkpointable() + save_root.dep_two = checkpointable.Checkpointable() + checkpointable_utils.add_variable( + save_root.dep_one, name="var1", initializer=32., dtype=dtypes.float64) + checkpointable_utils.add_variable( + save_root.dep_two, name="var2", initializer=64., dtype=dtypes.float64) + self.evaluate(checkpointable_utils.gather_initializers(save_root)) + save_path = checkpointable_utils.CheckpointableSaver(save_root).save( + os.path.join(checkpoint_directory, "ckpt")) + load_root = checkpointable.Checkpointable() + load_root.dep_one = checkpointable.Checkpointable() + load_root.dep_two = load_root.dep_one + v1 = checkpointable_utils.add_variable( + load_root.dep_one, name="var1", shape=[], dtype=dtypes.float64) + v2 = checkpointable_utils.add_variable( + load_root.dep_one, name="var2", shape=[], dtype=dtypes.float64) + status = checkpointable_utils.CheckpointableSaver(load_root).restore( + save_path).assert_consumed() + status.run_restore_ops() + self.assertEqual(32., self.evaluate(v1)) + self.assertEqual(64., self.evaluate(v2)) + + @test_util.run_in_graph_and_eager_modes() + def testDependencyLoop(self): + # Note: this test creates garbage during eager execution because it + # purposefully creates a reference cycle. + first = checkpointable.Checkpointable() + second = checkpointable.Checkpointable() + first.second = second + second.first = first + first.v = checkpointable_utils.add_variable( + first, "v1", initializer=[3., 1., 4.]) + second.v = checkpointable_utils.add_variable( + second, "v2", initializer=[1., 1., 2., 3.]) + self.evaluate(checkpointable_utils.gather_initializers(first)) + checkpoint_directory = self.get_temp_dir() + save_path = checkpointable_utils.CheckpointableSaver(first).save( + os.path.join(checkpoint_directory, "ckpt")) + + # Test deferred loading + first_load = checkpointable.Checkpointable() + status = checkpointable_utils.CheckpointableSaver( + first_load).restore(save_path) + second_load = checkpointable.Checkpointable() + first_load.second = second_load + second_load.first = first_load + with self.assertRaises(AssertionError): + status.assert_consumed() + first_load.v = checkpointable_utils.add_variable( + first_load, "v1", shape=[3]) + second_load.v = checkpointable_utils.add_variable( + second_load, "v2", shape=[4]) + status.assert_consumed() + status.run_restore_ops() + self.assertAllEqual([3., 1., 4.], self.evaluate(first_load.v)) + self.assertAllEqual([1., 1., 2., 3.], self.evaluate(second_load.v)) + + # Test loading when variables have already been created + self.evaluate(first_load.v.assign([2., 7., 1.])) + self.assertAllEqual([2., 7., 1.], self.evaluate(first_load.v)) + self.evaluate(second_load.v.assign([2., 7., 1., 8.])) + self.assertAllEqual([2., 7., 1., 8.], self.evaluate(second_load.v)) + status = checkpointable_utils.CheckpointableSaver(first_load).restore( + save_path).assert_consumed() + status.run_restore_ops() + self.assertAllEqual([3., 1., 4.], self.evaluate(first_load.v)) + self.assertAllEqual([1., 1., 2., 3.], self.evaluate(second_load.v)) + + @test_util.run_in_graph_and_eager_modes() + def testRestoreOnAssign(self): + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + save_graph = ops.Graph() + with save_graph.as_default(), self.test_session(save_graph): + first = checkpointable.Checkpointable() + first.var1 = variable_scope.get_variable( + name="outside_var", initializer=0.) + first.var2 = variable_scope.get_variable( + name="blah", initializer=0.) + self.evaluate(first.var1.assign(4.)) + self.evaluate(first.var2.assign(8.)) + save_path = checkpointable_utils.CheckpointableSaver(first).save( + checkpoint_prefix) + restore_graph = ops.Graph() + with restore_graph.as_default(), self.test_session(restore_graph): + second = checkpointable.Checkpointable() + second.var2 = variable_scope.get_variable( + name="blah", initializer=0.) + status = checkpointable_utils.CheckpointableSaver( + second).restore(save_path) + recreated_var1 = variable_scope.get_variable( + name="outside_var", initializer=0.) + status.run_restore_ops() + self.assertEqual(8., self.evaluate(second.var2)) + self.evaluate(recreated_var1.assign(-2.)) + self.assertEqual(-2., self.evaluate(recreated_var1)) + second.var1 = recreated_var1 + status.run_restore_ops() + self.assertEqual(4., self.evaluate(recreated_var1)) + + def testManySavesGraph(self): + """Saves after the first should not modify the graph.""" + with context.graph_mode(): + graph = ops.Graph() + with graph.as_default(), self.test_session(graph): + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + obj = checkpointable.Checkpointable() + obj.var = variable_scope.get_variable(name="v", initializer=0.) + obj.opt = adam.AdamOptimizer(0.1) + obj.opt.minimize(obj.var.read_value()) + self.evaluate(checkpointable_utils.gather_initializers(obj)) + saver = checkpointable_utils.CheckpointableSaver(obj) + saver.save(checkpoint_prefix) + before_ops = graph.get_operations() + saver.save(checkpoint_prefix) + self.assertEqual(before_ops, graph.get_operations()) + + @test_util.run_in_graph_and_eager_modes() + def testCheckpointCleanup(self): + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + obj = checkpointable.Checkpointable() + obj.var = variable_scope.get_variable(name="v", initializer=0.) + self.evaluate(checkpointable_utils.gather_initializers(obj)) + saver = checkpointable_utils.Checkpoint(obj=obj) + for _ in range(10): + saver.save(checkpoint_prefix) + expected_filenames = ["checkpoint"] + for checkpoint_number in range(6, 11): + expected_filenames.append("ckpt-%d.index" % (checkpoint_number,)) + expected_filenames.append( + "ckpt-%d.data-00000-of-00001" % (checkpoint_number,)) + six.assertCountEqual( + self, + expected_filenames, + os.listdir(checkpoint_directory)) + + @test_util.run_in_graph_and_eager_modes() + def testCheckpointCleanupChangingVarList(self): + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + obj = checkpointable.Checkpointable() + obj.var = variable_scope.get_variable(name="v", initializer=0.) + self.evaluate(checkpointable_utils.gather_initializers(obj)) + checkpoint = checkpointable_utils.Checkpoint(obj=obj) + looped_variables = [] + for iteration in range(10): + new_variable = resource_variable_ops.ResourceVariable(iteration) + self.evaluate(new_variable.initializer) + setattr(checkpoint, "var_%d" % iteration, new_variable) + checkpoint.save(checkpoint_prefix) + looped_variables.append(new_variable) + expected_filenames = ["checkpoint"] + # We've copied the saver each time, but checkpoint management should still + # be consistent. + for checkpoint_number in range(6, 11): + expected_filenames.append("ckpt-%d.index" % (checkpoint_number,)) + expected_filenames.append( + "ckpt-%d.data-00000-of-00001" % (checkpoint_number,)) + six.assertCountEqual( + self, + expected_filenames, + os.listdir(checkpoint_directory)) + for v in looped_variables: + self.evaluate(v.assign(314)) + checkpoint.restore(checkpoint_prefix + "-6").run_restore_ops() + self.assertEqual(314, self.evaluate(checkpoint.var_9)) + self.assertEqual(314, self.evaluate(checkpoint.var_8)) + self.assertEqual(314, self.evaluate(checkpoint.var_6)) + self.assertEqual(5, self.evaluate(checkpoint.var_5)) + self.assertEqual(1, self.evaluate(checkpoint.var_1)) + self.assertEqual(0, self.evaluate(checkpoint.var_0)) + if context.executing_eagerly(): + checkpoint.restore(checkpoint_prefix + "-10").run_restore_ops() + self.assertEqual(9, self.evaluate(checkpoint.var_9)) + self.assertEqual(8, self.evaluate(checkpoint.var_8)) + self.assertEqual(1, self.evaluate(checkpoint.var_1)) + self.assertEqual(0, self.evaluate(checkpoint.var_0)) + else: + # Restoring into modified graphs is an error while graph building. + with self.assertRaises(NotImplementedError): + checkpoint.restore(checkpoint_prefix + "-10").run_restore_ops() + + def testManyRestoresGraph(self): + """Restores after the first should not modify the graph.""" + with context.graph_mode(): + graph = ops.Graph() + with graph.as_default(), self.test_session(graph): + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + obj = checkpointable.Checkpointable() + obj.var = variable_scope.get_variable(name="v", initializer=0.) + obj.opt = adam.AdamOptimizer(0.1) + obj.opt.minimize(obj.var.read_value()) + self.evaluate(checkpointable_utils.gather_initializers(obj)) + saver = checkpointable_utils.CheckpointableSaver(obj) + save_path = saver.save(checkpoint_prefix) + saver.restore(save_path) + before_ops = graph.get_operations() + saver.restore(save_path) + self.assertEqual(before_ops, graph.get_operations()) + + def testMultipleGraphsNonSlotVariables(self): + with context.graph_mode(): + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + optimizer = adam.AdamOptimizer(0.001) + # Construct a model in one graph + first_graph = ops.Graph() + first_session = session_lib.Session(graph=first_graph) + with first_graph.as_default(), first_session.as_default(): + first_variable = resource_variable_ops.ResourceVariable([1.]) + first_root_checkpointable = checkpointable_utils.Checkpoint( + optimizer=optimizer, variable=first_variable) + train_op = optimizer.minimize(first_variable.read_value) + self.evaluate(checkpointable_utils.gather_initializers( + first_root_checkpointable)) + self.evaluate(train_op) + self.evaluate(first_variable.assign([1.])) + self.evaluate(optimizer.get_slot( + var=first_variable, name="m").assign([2.])) + beta1_power, _ = optimizer._get_beta_accumulators() + self.evaluate(beta1_power.assign(3.)) + + # Save and load in a second graph + second_graph = ops.Graph() + with second_graph.as_default(), session_lib.Session(graph=second_graph): + second_variable = resource_variable_ops.ResourceVariable([1.]) + second_root_checkpointable = checkpointable_utils.Checkpoint( + optimizer=optimizer, variable=second_variable) + train_op = optimizer.minimize(second_variable.read_value) + second_root_checkpointable.restore(None).initialize_or_restore() + self.evaluate(train_op) + self.evaluate(second_variable.assign([4.])) + self.evaluate(optimizer.get_slot( + var=second_variable, name="m").assign([5.])) + beta1_power, _ = optimizer._get_beta_accumulators() + self.evaluate(beta1_power.assign(6.)) + save_path = second_root_checkpointable.save(checkpoint_prefix) + self.evaluate(second_variable.assign([7.])) + self.evaluate(optimizer.get_slot( + var=second_variable, name="m").assign([8.])) + beta1_power, _ = optimizer._get_beta_accumulators() + self.assertAllEqual(6., self.evaluate(beta1_power)) + status = second_root_checkpointable.restore(save_path) + status.assert_consumed().run_restore_ops() + self.assertAllEqual([4.], self.evaluate(second_variable)) + self.assertAllEqual([5.], self.evaluate(optimizer.get_slot( + var=second_variable, name="m"))) + beta1_power, _ = optimizer._get_beta_accumulators() + self.assertAllEqual(6., self.evaluate(beta1_power)) + + # Check that the first graph is unmolested + with first_graph.as_default(), first_session.as_default(): + self.assertAllEqual([1.], self.evaluate(first_variable)) + self.assertAllEqual([2.], self.evaluate(optimizer.get_slot( + var=first_variable, name="m"))) + beta1_power, _ = optimizer._get_beta_accumulators() + self.assertAllEqual(3., self.evaluate(beta1_power)) + + @test_util.run_in_graph_and_eager_modes() + def test_sequential(self): + model = sequential.Sequential() + checkpoint = checkpointable_utils.Checkpoint(model=model) + model.add(core.Dense(4)) + second_dense = core.Dense(5) + model.add(second_dense) + model(constant_op.constant([[1.]])) + checkpoint.restore(None).initialize_or_restore() + self.evaluate(second_dense.bias.assign( + constant_op.constant([1., 2., 3., 4., 5.]))) + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + save_path = checkpoint.save(checkpoint_prefix) + self.evaluate(second_dense.bias.assign( + constant_op.constant([5., 6., 7., 8., 9.]))) + checkpoint.restore(save_path).assert_consumed().run_restore_ops() + self.assertAllEqual([1., 2., 3., 4., 5.], self.evaluate(second_dense.bias)) + + deferred_sequential = sequential.Sequential() + deferred_sequential_checkpoint = checkpointable_utils.Checkpoint( + model=deferred_sequential) + status = deferred_sequential_checkpoint.restore(save_path) + deferred_sequential.add(core.Dense(4)) + deferred_sequential(constant_op.constant([[1.]])) + deferred_second_dense = core.Dense(5) + deferred_sequential.add(deferred_second_dense) + deferred_sequential(constant_op.constant([[1.]])) + status.run_restore_ops() + self.assertAllEqual([1., 2., 3., 4., 5.], + self.evaluate(deferred_second_dense.bias)) + + +class TemplateTests(test.TestCase): + + @test_util.run_in_graph_and_eager_modes() + def test_checkpointable_save_restore(self): + + def _templated(): + v = variable_scope.get_variable( + "v", shape=[1], initializer=init_ops.zeros_initializer()) + v2 = variable_scope.get_variable( + "v2", shape=[1], initializer=init_ops.zeros_initializer()) + return v, v + 1., v2 + + save_template = template.make_template("s1", _templated) + save_root = checkpointable_utils.Checkpoint(my_template=save_template) + v1_save, _, v2_save = save_template() + self.evaluate(v1_save.assign([12.])) + self.evaluate(v2_save.assign([14.])) + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + save_path = save_root.save(checkpoint_prefix) + + load_template = template.make_template("s2", _templated) + load_root = checkpointable_utils.Checkpoint(my_template=load_template) + status = load_root.restore(save_path) + var, var_plus_one, var2 = load_template() + self.assertEqual(2, len(load_template._checkpoint_dependencies)) + self.assertEqual("v", load_template._checkpoint_dependencies[0].name) + self.assertEqual("v2", load_template._checkpoint_dependencies[1].name) + status.assert_consumed().run_restore_ops() + self.assertAllEqual([12.], self.evaluate(var)) + self.assertAllEqual([13.], self.evaluate(var_plus_one)) + self.assertAllEqual([14.], self.evaluate(var2)) + + @test_util.run_in_graph_and_eager_modes() + def test_checkpointable_save_restore_nested(self): + + def _inner_template(): + v = variable_scope.get_variable( + "v", shape=[1], initializer=init_ops.zeros_initializer()) + return v + + def _outer_template(): + first_inner = template.make_template("i1", _inner_template) + second_inner = template.make_template("i2", _inner_template) + v1 = first_inner() + v2 = second_inner() + v3 = second_inner() + return (first_inner, second_inner), (v1, v2, v3) + + with variable_scope.variable_scope("ignored"): + save_template = template.make_template("s1", _outer_template) + save_root = checkpointable_utils.Checkpoint(my_template=save_template) + (inner_template_one, inner_template_two), _ = save_template() + self.evaluate(inner_template_one.variables[0].assign([20.])) + self.evaluate(inner_template_two.variables[0].assign([25.])) + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + save_path = save_root.save(checkpoint_prefix) + + load_template = template.make_template("s2", _outer_template) + load_root = checkpointable_utils.Checkpoint(my_template=load_template) + status = load_root.restore(save_path) + (inner_template_one, inner_template_two), (v1, v2, v3) = load_template() + outer_template_dependencies = load_root.my_template._checkpoint_dependencies + self.assertEqual(2, len(outer_template_dependencies)) + self.assertEqual("i1", outer_template_dependencies[0].name) + self.assertIs(inner_template_one, outer_template_dependencies[0].ref) + self.assertEqual("i2", outer_template_dependencies[1].name) + self.assertIs(inner_template_two, outer_template_dependencies[1].ref) + self.assertEqual(1, len(inner_template_one._checkpoint_dependencies)) + self.assertEqual("v", inner_template_one._checkpoint_dependencies[0].name) + self.assertEqual(1, len(inner_template_two._checkpoint_dependencies)) + self.assertEqual("v", inner_template_two._checkpoint_dependencies[0].name) + status.assert_consumed().run_restore_ops() + self.assertAllEqual([20.], self.evaluate(v1)) + self.assertAllEqual([25.], self.evaluate(v2)) + self.assertAllEqual([25.], self.evaluate(v3)) + + +class CheckpointCompatibilityTests(test.TestCase): + + def _initialized_model(self): + input_value = constant_op.constant([[3.]]) + model = MyModel() + optimizer = adam.AdamOptimizer(0.001) + optimizer_step = training_util.get_or_create_global_step() + root_checkpointable = checkpointable_utils.Checkpoint( + optimizer=optimizer, model=model, optimizer_step=optimizer_step) + train_op = optimizer.minimize( + functools.partial(model, input_value), + global_step=optimizer_step) + self.evaluate(checkpointable_utils.gather_initializers( + root_checkpointable)) + self.evaluate(train_op) + # A regular variable, a slot variable, and a non-slot Optimizer variable + # with known values to check when loading. + self.evaluate(model._named_dense.bias.assign([1.])) + self.evaluate(optimizer.get_slot( + var=model._named_dense.bias, name="m").assign([2.])) + beta1_power, _ = optimizer._get_beta_accumulators() + self.evaluate(beta1_power.assign(3.)) + return root_checkpointable + + def _set_sentinels(self, root_checkpointable): + self.evaluate(root_checkpointable.model._named_dense.bias.assign([101.])) + self.evaluate( + root_checkpointable.optimizer.get_slot( + var=root_checkpointable.model._named_dense.bias, name="m") + .assign([102.])) + beta1_power, _ = root_checkpointable.optimizer._get_beta_accumulators() + self.evaluate(beta1_power.assign(103.)) + + def _check_sentinels(self, root_checkpointable): + self.assertAllEqual( + [1.], self.evaluate(root_checkpointable.model._named_dense.bias)) + self.assertAllEqual([2.], self.evaluate( + root_checkpointable.optimizer.get_slot( + var=root_checkpointable.model._named_dense.bias, name="m"))) + beta1_power, _ = root_checkpointable.optimizer._get_beta_accumulators() + self.assertAllEqual(3., self.evaluate(beta1_power)) + + def _write_name_based_checkpoint(self): + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + with context.graph_mode(): + save_graph = ops.Graph() + with save_graph.as_default(), self.test_session( + graph=save_graph) as session: + root = self._initialized_model() + name_saver = saver_lib.Saver() + return name_saver.save( + sess=session, save_path=checkpoint_prefix, + global_step=root.optimizer_step) + + @test_util.run_in_graph_and_eager_modes() + def testLoadFromNameBasedSaver(self): + """Save a name-based checkpoint, load it using the object-based API.""" + with test_util.device(use_gpu=True): + save_path = self._write_name_based_checkpoint() + root = self._initialized_model() + self._set_sentinels(root) + with self.assertRaises(AssertionError): + self._check_sentinels(root) + object_saver = checkpointable_utils.CheckpointableSaver(root) + status = object_saver.restore(save_path) + with self.assertRaises(AssertionError): + status.assert_consumed() + status.run_restore_ops() + self._check_sentinels(root) + self._set_sentinels(root) + status.initialize_or_restore() + self._check_sentinels(root) + + # TODO(allenl): Test for the core name-based saver loading object-based + # checkpoints once object-based checkpointing is in core. + + def testSaveGraphLoadEager(self): + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + with context.graph_mode(): + save_graph = ops.Graph() + with save_graph.as_default(), self.test_session( + graph=save_graph) as session: + root = self._initialized_model() + object_saver = checkpointable_utils.CheckpointableSaver(root) + save_path = object_saver.save( + session=session, file_prefix=checkpoint_prefix) + with context.eager_mode(): + root = self._initialized_model() + self._set_sentinels(root) + root.restore(save_path).assert_consumed() + self._check_sentinels(root) + + def testSaveEagerLoadGraph(self): + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + with context.eager_mode(): + root = self._initialized_model() + object_saver = checkpointable_utils.CheckpointableSaver(root) + save_path = object_saver.save(file_prefix=checkpoint_prefix) + with context.graph_mode(): + save_graph = ops.Graph() + with save_graph.as_default(), self.test_session( + graph=save_graph): + root = self._initialized_model() + self._set_sentinels(root) + root.restore(save_path).assert_consumed().run_restore_ops() + self._check_sentinels(root) + +if __name__ == "__main__": + test.main() -- GitLab From 6d2316d4a75be1c603e4edd08a33e1098a28b070 Mon Sep 17 00:00:00 2001 From: Igor Ganichev Date: Thu, 12 Apr 2018 12:04:48 -0700 Subject: [PATCH 253/791] Add FunctionTest.testLayerInDefun PiperOrigin-RevId: 192647818 --- tensorflow/python/eager/BUILD | 2 ++ tensorflow/python/eager/function_test.py | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD index 8c0d3feece..b3268c9047 100644 --- a/tensorflow/python/eager/BUILD +++ b/tensorflow/python/eager/BUILD @@ -142,6 +142,8 @@ cuda_py_test( ":tape", ":test", "//tensorflow/python:clip_ops", + "//tensorflow/python:init_ops", + "//tensorflow/python:layers", "//tensorflow/python:math_ops", "//tensorflow/python:resource_variable_ops", ], diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py index 9af197981b..65dde75e60 100644 --- a/tensorflow/python/eager/function_test.py +++ b/tensorflow/python/eager/function_test.py @@ -29,9 +29,11 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import function as tf_function from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.layers import convolutional from tensorflow.python.ops import array_ops from tensorflow.python.ops import clip_ops from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import init_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import variable_scope @@ -104,6 +106,7 @@ class FunctionTest(test.TestCase): matmul = function.defun(math_ops.matmul) pair = collections.namedtuple('pair', ['a', 'b']) + def a_times_b(inputs): return matmul(inputs.a['a'], inputs.b['b']) @@ -312,6 +315,7 @@ class FunctionTest(test.TestCase): x = variable_scope.get_variable( 'v', initializer=constant_op.constant(1.0)) return x * constant_op.constant(2.0) + with self.assertRaisesRegexp(ValueError, 'No trainable variables were accessed'): backprop.implicit_val_and_grad(f)() @@ -581,6 +585,7 @@ class FunctionTest(test.TestCase): with ops.name_scope('foo'): v = resource_variable_ops.ResourceVariable(0.0, name='bar') self.assertEqual(v.name, 'foo/bar:0') + create_variable() def testVariableNamesRespectNameScopesWithDefunInGraph(self): @@ -590,9 +595,25 @@ class FunctionTest(test.TestCase): with ops.name_scope('foo'): v = resource_variable_ops.ResourceVariable([1.0, 2.0], name='bar') self.assertEqual(v.name, 'foo/bar:0') + with ops.get_default_graph().as_default(): create_variable() + def testLayerInDefun(self): + conv = convolutional.Conv2D( + filters=1, + kernel_size=2, + kernel_initializer=init_ops.ones_initializer(), + bias_initializer=init_ops.zeros_initializer()) + + @function.defun + def model(x): + return conv(x) + + x = array_ops.ones([1, 2, 2, 1]) + y = model(x) + self.assertAllEqual([[[[4.0]]]], y.numpy()) + class AutomaticControlDependenciesTest(test.TestCase): -- GitLab From 6308e58e32e0d238e7df35b4c8a5935c3327d79a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 12:09:43 -0700 Subject: [PATCH 254/791] Add softsign bijector. PiperOrigin-RevId: 192648596 --- tensorflow/contrib/distributions/BUILD | 19 +++ .../kernel_tests/bijectors/softsign_test.py | 111 ++++++++++++++++++ .../python/ops/bijectors/__init__.py | 2 + .../python/ops/bijectors/softsign.py | 86 ++++++++++++++ 4 files changed, 218 insertions(+) create mode 100644 tensorflow/contrib/distributions/python/kernel_tests/bijectors/softsign_test.py create mode 100644 tensorflow/contrib/distributions/python/ops/bijectors/softsign.py diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD index fec6eafd4a..20e432b88d 100644 --- a/tensorflow/contrib/distributions/BUILD +++ b/tensorflow/contrib/distributions/BUILD @@ -1174,6 +1174,25 @@ cuda_py_test( ], ) +cuda_py_test( + name = "softsign_test", + size = "small", + srcs = ["python/kernel_tests/bijectors/softsign_test.py"], + additional_deps = [ + ":bijectors_py", + ":distributions_py", + "//third_party/py/numpy", + "@six_archive//:six", + "//tensorflow/contrib/linalg:linalg_py", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:math_ops", + "//tensorflow/python:platform_test", + ], +) + cuda_py_test( name = "square_test", size = "small", diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/softsign_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/softsign_test.py new file mode 100644 index 0000000000..2ac06fce55 --- /dev/null +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/softsign_test.py @@ -0,0 +1,111 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for Bijector.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.contrib.distributions.python.ops.bijectors.softsign import Softsign +from tensorflow.python.framework import test_util +from tensorflow.python.ops.distributions.bijector_test_util import assert_bijective_and_finite +from tensorflow.python.ops.distributions.bijector_test_util import assert_scalar_congruency +from tensorflow.python.platform import test + + +class SoftsignBijectorTest(test.TestCase): + """Tests the correctness of the Y = g(X) = X / (1 + |X|) transformation.""" + + def _softsign(self, x): + return x / (1. + np.abs(x)) + + def _softsign_ildj_before_reduction(self, y): + """Inverse log det jacobian, before being reduced.""" + return -2. * np.log1p(-np.abs(y)) + + def setUp(self): + self._rng = np.random.RandomState(42) + + @test_util.run_in_graph_and_eager_modes() + def testBijectorBounds(self): + bijector = Softsign(validate_args=True) + with self.test_session(): + with self.assertRaisesOpError("greater than -1"): + bijector.inverse(-3.).eval() + with self.assertRaisesOpError("greater than -1"): + bijector.inverse_log_det_jacobian(-3., event_ndims=0).eval() + + with self.assertRaisesOpError("less than 1"): + bijector.inverse(3.).eval() + with self.assertRaisesOpError("less than 1"): + bijector.inverse_log_det_jacobian(3., event_ndims=0).eval() + + @test_util.run_in_graph_and_eager_modes() + def testBijectorForwardInverse(self): + bijector = Softsign(validate_args=True) + self.assertEqual("softsign", bijector.name) + x = 2. * self._rng.randn(2, 10) + y = self._softsign(x) + + self.assertAllClose(y, self.evaluate(bijector.forward(x))) + self.assertAllClose(x, self.evaluate(bijector.inverse(y))) + + @test_util.run_in_graph_and_eager_modes() + def testBijectorLogDetJacobianEventDimsZero(self): + bijector = Softsign(validate_args=True) + y = self._rng.rand(2, 10) + # No reduction needed if event_dims = 0. + ildj = self._softsign_ildj_before_reduction(y) + + self.assertAllClose(ildj, self.evaluate( + bijector.inverse_log_det_jacobian(y, event_ndims=0))) + + @test_util.run_in_graph_and_eager_modes() + def testBijectorForwardInverseEventDimsOne(self): + bijector = Softsign(validate_args=True) + self.assertEqual("softsign", bijector.name) + x = 2. * self._rng.randn(2, 10) + y = self._softsign(x) + self.assertAllClose(y, self.evaluate(bijector.forward(x))) + self.assertAllClose(x, self.evaluate(bijector.inverse(y))) + + @test_util.run_in_graph_and_eager_modes() + def testBijectorLogDetJacobianEventDimsOne(self): + bijector = Softsign(validate_args=True) + y = self._rng.rand(2, 10) + ildj_before = self._softsign_ildj_before_reduction(y) + ildj = np.sum(ildj_before, axis=1) + self.assertAllClose( + ildj, self.evaluate( + bijector.inverse_log_det_jacobian(y, event_ndims=1))) + + def testScalarCongruency(self): + with self.test_session(): + bijector = Softsign(validate_args=True) + assert_scalar_congruency(bijector, lower_x=-20., upper_x=20.) + + def testBijectiveAndFinite(self): + with self.test_session(): + bijector = Softsign(validate_args=True) + x = np.linspace(-20., 20., 100).astype(np.float32) + y = np.linspace(-0.99, 0.99, 100).astype(np.float32) + assert_bijective_and_finite( + bijector, x, y, event_ndims=0, rtol=1e-3, atol=1e-3) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py b/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py index bc6b02542e..babce80396 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py @@ -38,6 +38,7 @@ @@SinhArcsinh @@SoftmaxCentered @@Softplus +@@Softsign @@Square @@Weibull @@ -74,6 +75,7 @@ from tensorflow.contrib.distributions.python.ops.bijectors.sigmoid import * from tensorflow.contrib.distributions.python.ops.bijectors.sinh_arcsinh import * from tensorflow.contrib.distributions.python.ops.bijectors.softmax_centered import * from tensorflow.contrib.distributions.python.ops.bijectors.softplus import * +from tensorflow.contrib.distributions.python.ops.bijectors.softsign import * from tensorflow.contrib.distributions.python.ops.bijectors.square import * from tensorflow.python.ops.distributions.bijector import * from tensorflow.python.ops.distributions.identity_bijector import Identity diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/softsign.py b/tensorflow/contrib/distributions/python/ops/bijectors/softsign.py new file mode 100644 index 0000000000..b4a658c171 --- /dev/null +++ b/tensorflow/contrib/distributions/python/ops/bijectors/softsign.py @@ -0,0 +1,86 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Softsign bijector.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.ops import check_ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops.distributions import bijector + + +__all__ = [ + "Softsign", +] + + +class Softsign(bijector.Bijector): + """Bijector which computes `Y = g(X) = X / (1 + |X|)`. + + The softsign `Bijector` has the following two useful properties: + + * The domain is all real numbers + * `softsign(x) approx sgn(x)`, for large `|x|`. + + #### Examples + + ```python + # Create the Y = softsign(X) transform. + softsign = Softsign() + x = [[[1., 2], + [3, 4]], + [[5, 6], + [7, 8]]] + x / (1 + abs(x)) == softsign.forward(x) + x / (1 - abs(x)) == softsign.inverse(x) + ``` + """ + + def __init__(self, validate_args=False, name="softsign"): + super(Softsign, self).__init__( + forward_min_event_ndims=0, + validate_args=validate_args, + name=name) + + def _forward(self, x): + return x / (1. + math_ops.abs(x)) + + def _inverse(self, y): + y = self._maybe_assert_valid_y(y) + return y / (1. - math_ops.abs(y)) + + def _forward_log_det_jacobian(self, x): + return -2. * math_ops.log1p(math_ops.abs(x)) + + def _inverse_log_det_jacobian(self, y): + y = self._maybe_assert_valid_y(y) + return -2. * math_ops.log1p(-math_ops.abs(y)) + + def _maybe_assert_valid_y(self, y): + if not self.validate_args: + return y + is_valid = [ + check_ops.assert_greater( + y, math_ops.cast(-1., dtype=y.dtype.base_dtype), + message="Inverse transformation input must be greater than -1."), + check_ops.assert_less( + y, math_ops.cast(1., dtype=y.dtype.base_dtype), + message="Inverse transformation input must be less than 1.") + ] + + return control_flow_ops.with_dependencies(is_valid, y) -- GitLab From ecacd206c44811baa75bef07b2ce99cd1021163c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 12:12:16 -0700 Subject: [PATCH 255/791] [XLA] Redesign: add XlaComputation::IsNull. PiperOrigin-RevId: 192649052 --- tensorflow/compiler/xla/client/xla_client/xla_computation.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow/compiler/xla/client/xla_client/xla_computation.h b/tensorflow/compiler/xla/client/xla_client/xla_computation.h index 2a3c695266..7182908666 100644 --- a/tensorflow/compiler/xla/client/xla_client/xla_computation.h +++ b/tensorflow/compiler/xla/client/xla_client/xla_computation.h @@ -44,6 +44,9 @@ class XlaComputation { const HloModuleProto& proto() const { return proto_; } + // Returns true if this object is a null Computation. + bool IsNull() const { return unique_id_ == -1; } + private: XlaComputation(const int64 unique_id) : unique_id_(unique_id) {} HloModuleProto* mutable_proto() { return &proto_; } -- GitLab From 1a014c6d62bad0e58e3c8a1e31beb396daa19c13 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 12:29:48 -0700 Subject: [PATCH 256/791] Restore dependency on estimator utils from model. PiperOrigin-RevId: 192651583 --- tensorflow/contrib/boosted_trees/estimator_batch/BUILD | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD index 0f65881aee..8cff1a3bb1 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD +++ b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD @@ -28,12 +28,13 @@ py_library( srcs = ["model.py"], srcs_version = "PY2AND3", deps = [ + ":estimator_utils", ":trainer_hooks", "//tensorflow/contrib/boosted_trees:gbdt_batch", "//tensorflow/contrib/boosted_trees:model_ops_py", "//tensorflow/python:framework_ops", "//tensorflow/python:state_ops", - "//tensorflow/python:training", + "//tensorflow/python:training_util", ], ) -- GitLab From 7bf6efa2d8e1172df47c1c4a8a09a007a1a09e8f Mon Sep 17 00:00:00 2001 From: Loo Rong Jie Date: Fri, 13 Apr 2018 03:36:52 +0800 Subject: [PATCH 257/791] Replace all COMPILER_MSVC to _MSC_VER and _WIN32 accordingly (#18448) * Replace all COMPILER_MSVC to _MSC_VER and _WIN32 accordingly * One more ARRAYSIZE to TF_ARRAYSIZE * Delete non-existing include --- tensorflow/c/c_api.h | 4 ++-- tensorflow/c/c_api_experimental.h | 4 ++-- tensorflow/c/eager/c_api.h | 4 ++-- tensorflow/compiler/aot/runtime.cc | 4 ++-- tensorflow/contrib/cmake/CMakeLists.txt | 2 +- tensorflow/core/framework/numeric_types.h | 4 ++-- tensorflow/core/lib/gtl/manual_constructor.h | 2 +- tensorflow/core/lib/strings/stringprintf.cc | 10 ++-------- tensorflow/core/lib/strings/stringprintf_test.cc | 4 ++-- tensorflow/core/util/memmapped_file_system.cc | 2 +- tensorflow/core/util/memmapped_file_system.h | 4 ++-- tensorflow/stream_executor/cuda/cuda_driver.cc | 14 +++----------- .../stream_executor/cuda/cuda_gpu_executor.cc | 2 +- tensorflow/stream_executor/platform/port.h | 6 ------ 14 files changed, 23 insertions(+), 43 deletions(-) diff --git a/tensorflow/c/c_api.h b/tensorflow/c/c_api.h index fe85f8ee0e..c859434745 100644 --- a/tensorflow/c/c_api.h +++ b/tensorflow/c/c_api.h @@ -72,7 +72,7 @@ limitations under the License. #ifdef SWIG #define TF_CAPI_EXPORT #else -#if defined(COMPILER_MSVC) +#if defined(_WIN32) #ifdef TF_COMPILE_LIBRARY #define TF_CAPI_EXPORT __declspec(dllexport) #else @@ -80,7 +80,7 @@ limitations under the License. #endif // TF_COMPILE_LIBRARY #else #define TF_CAPI_EXPORT __attribute__((visibility("default"))) -#endif // COMPILER_MSVC +#endif // _WIN32 #endif // SWIG #ifdef __cplusplus diff --git a/tensorflow/c/c_api_experimental.h b/tensorflow/c/c_api_experimental.h index 666342974e..88cb173cd2 100644 --- a/tensorflow/c/c_api_experimental.h +++ b/tensorflow/c/c_api_experimental.h @@ -35,7 +35,7 @@ limitations under the License. #ifdef SWIG #define TF_CAPI_EXPORT #else -#if defined(COMPILER_MSVC) +#if defined(_WIN32) #ifdef TF_COMPILE_LIBRARY #define TF_CAPI_EXPORT __declspec(dllexport) #else @@ -43,7 +43,7 @@ limitations under the License. #endif // TF_COMPILE_LIBRARY #else #define TF_CAPI_EXPORT __attribute__((visibility("default"))) -#endif // COMPILER_MSVC +#endif // _WIN32 #endif // SWIG #ifdef __cplusplus diff --git a/tensorflow/c/eager/c_api.h b/tensorflow/c/eager/c_api.h index 3926c22ce1..c06ce84a8c 100644 --- a/tensorflow/c/eager/c_api.h +++ b/tensorflow/c/eager/c_api.h @@ -30,7 +30,7 @@ limitations under the License. #ifdef SWIG #define TF_CAPI_EXPORT #else -#if defined(COMPILER_MSVC) +#if defined(_WIN32) #ifdef TF_COMPILE_LIBRARY #define TF_CAPI_EXPORT __declspec(dllexport) #else @@ -38,7 +38,7 @@ limitations under the License. #endif // TF_COMPILE_LIBRARY #else #define TF_CAPI_EXPORT __attribute__((visibility("default"))) -#endif // COMPILER_MSVC +#endif // _WIN32 #endif // SWIG #ifdef __cplusplus diff --git a/tensorflow/compiler/aot/runtime.cc b/tensorflow/compiler/aot/runtime.cc index 5772776666..5e74079fc1 100644 --- a/tensorflow/compiler/aot/runtime.cc +++ b/tensorflow/compiler/aot/runtime.cc @@ -31,7 +31,7 @@ namespace { inline void* aligned_malloc(size_t size, int minimum_alignment) { #if defined(__ANDROID__) || defined(OS_ANDROID) || defined(OS_CYGWIN) return memalign(minimum_alignment, size); -#elif defined(COMPILER_MSVC) +#elif defined(_WIN32) return _aligned_malloc(size, minimum_alignment); #else // !__ANDROID__ && !OS_ANDROID && !OS_CYGWIN void* ptr = nullptr; @@ -48,7 +48,7 @@ inline void* aligned_malloc(size_t size, int minimum_alignment) { } inline void aligned_free(void* aligned_memory) { -#if defined(COMPILER_MSVC) +#if defined(_WIN32) _aligned_free(aligned_memory); #else free(aligned_memory); diff --git a/tensorflow/contrib/cmake/CMakeLists.txt b/tensorflow/contrib/cmake/CMakeLists.txt index 23b31ae1dc..a7944ea74a 100644 --- a/tensorflow/contrib/cmake/CMakeLists.txt +++ b/tensorflow/contrib/cmake/CMakeLists.txt @@ -124,7 +124,7 @@ endif() add_definitions(-DEIGEN_AVOID_STL_ARRAY) if(WIN32) - add_definitions(-DNOMINMAX -D_WIN32_WINNT=0x0A00 -DLANG_CXX11 -DCOMPILER_MSVC) + add_definitions(-DNOMINMAX -D_WIN32_WINNT=0x0A00 -DLANG_CXX11) add_definitions(-DWIN32 -DOS_WIN -D_MBCS -DWIN64 -DWIN32_LEAN_AND_MEAN -DNOGDI -DPLATFORM_WINDOWS) add_definitions(-DTENSORFLOW_USE_EIGEN_THREADPOOL -DEIGEN_HAS_C99_MATH) add_definitions(-DTF_COMPILE_LIBRARY) diff --git a/tensorflow/core/framework/numeric_types.h b/tensorflow/core/framework/numeric_types.h index dab53cba3e..b1d0127809 100644 --- a/tensorflow/core/framework/numeric_types.h +++ b/tensorflow/core/framework/numeric_types.h @@ -111,7 +111,7 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE tensorflow::bfloat16 abs( } // namespace numext } // namespace Eigen -#if defined(COMPILER_MSVC) && !defined(__clang__) +#if defined(_MSC_VER) && !defined(__clang__) namespace std { template <> struct hash { @@ -120,6 +120,6 @@ struct hash { } }; } // namespace std -#endif // COMPILER_MSVC +#endif // _MSC_VER #endif // TENSORFLOW_FRAMEWORK_NUMERIC_TYPES_H_ diff --git a/tensorflow/core/lib/gtl/manual_constructor.h b/tensorflow/core/lib/gtl/manual_constructor.h index 0a76e0962e..0176cdc94d 100644 --- a/tensorflow/core/lib/gtl/manual_constructor.h +++ b/tensorflow/core/lib/gtl/manual_constructor.h @@ -53,7 +53,7 @@ template struct AlignType<0, size> { typedef char result[size]; }; -#if defined(COMPILER_MSVC) +#if defined(_MSC_VER) #define TF_LIB_GTL_ALIGN_ATTRIBUTE(X) __declspec(align(X)) #define TF_LIB_GTL_ALIGN_OF(T) __alignof(T) #elif defined(COMPILER_GCC3) || __GNUC__ >= 3 || defined(__APPLE__) || \ diff --git a/tensorflow/core/lib/strings/stringprintf.cc b/tensorflow/core/lib/strings/stringprintf.cc index 03eba4c851..bbffa062a9 100644 --- a/tensorflow/core/lib/strings/stringprintf.cc +++ b/tensorflow/core/lib/strings/stringprintf.cc @@ -22,12 +22,6 @@ limitations under the License. namespace tensorflow { namespace strings { -#ifdef COMPILER_MSVC -enum { IS_COMPILER_MSVC = 1 }; -#else -enum { IS_COMPILER_MSVC = 0 }; -#endif - void Appendv(string* dst, const char* format, va_list ap) { // First try with a small fixed size buffer static const int kSpaceLength = 1024; @@ -48,13 +42,13 @@ void Appendv(string* dst, const char* format, va_list ap) { return; } - if (IS_COMPILER_MSVC) { +#ifdef _MSC_VER // Error or MSVC running out of space. MSVC 8.0 and higher // can be asked about space needed with the special idiom below: va_copy(backup_ap, ap); result = vsnprintf(nullptr, 0, format, backup_ap); va_end(backup_ap); - } +#endif if (result < 0) { // Just an error. diff --git a/tensorflow/core/lib/strings/stringprintf_test.cc b/tensorflow/core/lib/strings/stringprintf_test.cc index d61a1a945a..02cf4cbcad 100644 --- a/tensorflow/core/lib/strings/stringprintf_test.cc +++ b/tensorflow/core/lib/strings/stringprintf_test.cc @@ -30,9 +30,9 @@ TEST(PrintfTest, Empty) { TEST(PrintfTest, Misc) { // MSVC does not support $ format specifier. -#if !defined(COMPILER_MSVC) +#if !defined(_MSC_VER) EXPECT_EQ("123hello w", Printf("%3$d%2$s %1$c", 'w', "hello", 123)); -#endif // !COMPILER_MSVC +#endif // !_MSC_VER } TEST(AppendfTest, Empty) { diff --git a/tensorflow/core/util/memmapped_file_system.cc b/tensorflow/core/util/memmapped_file_system.cc index 1fa6b8bec0..d3439cbc93 100644 --- a/tensorflow/core/util/memmapped_file_system.cc +++ b/tensorflow/core/util/memmapped_file_system.cc @@ -185,7 +185,7 @@ const void* MemmappedFileSystem::GetMemoryWithOffset(uint64 offset) const { return reinterpret_cast(mapped_memory_->data()) + offset; } -#if defined(COMPILER_MSVC) +#if defined(_MSC_VER) constexpr char* MemmappedFileSystem::kMemmappedPackagePrefix; constexpr char* MemmappedFileSystem::kMemmappedPackageDefaultGraphDef; #else diff --git a/tensorflow/core/util/memmapped_file_system.h b/tensorflow/core/util/memmapped_file_system.h index 76cc4911f5..958e23d28e 100644 --- a/tensorflow/core/util/memmapped_file_system.h +++ b/tensorflow/core/util/memmapped_file_system.h @@ -53,7 +53,7 @@ class MemmappedFileSystem : public FileSystem { public: // Memmapped regions use this prefix to distinguish from // the filesystem. -#if defined(COMPILER_MSVC) +#if defined(_MSC_VER) static constexpr char* kMemmappedPackagePrefix = #else static constexpr char kMemmappedPackagePrefix[] = @@ -61,7 +61,7 @@ class MemmappedFileSystem : public FileSystem { "memmapped_package://"; // The default graphdef in the package. -#if defined(COMPILER_MSVC) +#if defined(_MSC_VER) static constexpr char* kMemmappedPackageDefaultGraphDef = #else static constexpr char kMemmappedPackageDefaultGraphDef[] = diff --git a/tensorflow/stream_executor/cuda/cuda_driver.cc b/tensorflow/stream_executor/cuda/cuda_driver.cc index 58e1e58c59..b06be69b64 100644 --- a/tensorflow/stream_executor/cuda/cuda_driver.cc +++ b/tensorflow/stream_executor/cuda/cuda_driver.cc @@ -37,14 +37,6 @@ limitations under the License. #include "tensorflow/stream_executor/platform/port.h" #include "tensorflow/stream_executor/lib/inlined_vector.h" -#if defined(PLATFORM_WINDOWS) -// TODO: in windows ARRAYSIZE is defined in winnt.h but including it -// here creates a conflict with cuda.h - for now define it here. -#define ARRAYSIZE(a) \ - ((sizeof(a) / sizeof(*(a))) / \ - static_cast(!(sizeof(a) % sizeof(*(a))))) -#endif - bool FLAGS_gpuexec_cuda_driver_inject_init_error = false; bool FLAGS_gpuexec_cuda_sync_around_driver_calls = false; bool FLAGS_gpuexec_cuda_device_0_only = false; @@ -720,15 +712,15 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) { port::bit_cast(uintptr_t(info_log_buffer_bytes)), port::bit_cast(info_log_buffer.data()), port::bit_cast(uintptr_t(log_verbose))}; - CHECK(ARRAYSIZE(options) == ARRAYSIZE(option_values)); + CHECK(TF_ARRAYSIZE(options) == TF_ARRAYSIZE(option_values)); CUresult res; { // TODO(leary) Need to see if NVIDIA can expunge the leakiness in their // module loading: see http://b/13248943 - res = cuModuleLoadDataEx(module, ptx_data, ARRAYSIZE(options), options, - option_values); + res = cuModuleLoadDataEx(module, ptx_data, TF_ARRAYSIZE(options), + options, option_values); } // The PTX JIT mutates the values in the option values array to reflect the diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc index 5ecaf46b8c..58ca0d3a97 100644 --- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc +++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc @@ -1127,7 +1127,7 @@ DeviceDescription *CUDAExecutor::PopulateDeviceDescription() const { builder.set_name(device_name); } - for (size_t i = 0; i < ARRAYSIZE(kAllUnqueryableDeviceParams); i++) { + for (size_t i = 0; i < TF_ARRAYSIZE(kAllUnqueryableDeviceParams); i++) { const auto ¶ms = kAllUnqueryableDeviceParams[i]; if (params.cc_major == cc_major_ && params.cc_minor == cc_minor_) { builder.set_blocks_per_core_limit(params.blocks_per_core_limit); diff --git a/tensorflow/stream_executor/platform/port.h b/tensorflow/stream_executor/platform/port.h index 6603df4878..db62100435 100644 --- a/tensorflow/stream_executor/platform/port.h +++ b/tensorflow/stream_executor/platform/port.h @@ -39,12 +39,6 @@ using tensorflow::uint64; using std::string; #endif -#if !defined(COMPILER_MSVC) -#define ARRAYSIZE(a) \ - ((sizeof(a) / sizeof(*(a))) / \ - static_cast(!(sizeof(a) % sizeof(*(a))))) -#endif - using tensorflow::LinkerInitialized; using tensorflow::LINKER_INITIALIZED; -- GitLab From f95906527e92a151a424b60a109d2361e20d610b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 12:39:48 -0700 Subject: [PATCH 258/791] Fix comment of bucket_by_sequence_length about return type of element_length_func. Current code requires tf.int32 in order to compare with buckets_min which is int32. PiperOrigin-RevId: 192652917 --- tensorflow/contrib/data/python/ops/grouping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/data/python/ops/grouping.py b/tensorflow/contrib/data/python/ops/grouping.py index 36591c055a..0531f9cbb9 100644 --- a/tensorflow/contrib/data/python/ops/grouping.py +++ b/tensorflow/contrib/data/python/ops/grouping.py @@ -108,7 +108,7 @@ def bucket_by_sequence_length(element_length_func, fraction of padding in a batch which increases training step efficiency. Args: - element_length_func: function from element in `Dataset` to `tf.int64`, + element_length_func: function from element in `Dataset` to `tf.int32`, determines the length of the element, which will determine the bucket it goes into. bucket_boundaries: `list`, upper length boundaries of the buckets. -- GitLab From 3add17c999e7a50442fb5c97d2bb2d88597d5039 Mon Sep 17 00:00:00 2001 From: Loo Rong Jie Date: Fri, 13 Apr 2018 03:57:26 +0800 Subject: [PATCH 259/791] [MSVC] Remove -D__VERSION__ flag and implement tf_compiler_version properly (#18445) --- tensorflow/contrib/cmake/tf_core_framework.cmake | 6 ------ tensorflow/tensorflow.bzl | 1 - tensorflow/tools/git/gen_git_source.py | 10 +++++++++- tensorflow/tools/git/gen_git_source.sh | 10 +++++++++- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/tensorflow/contrib/cmake/tf_core_framework.cmake b/tensorflow/contrib/cmake/tf_core_framework.cmake index bcfb4f0819..f7cb186c7c 100644 --- a/tensorflow/contrib/cmake/tf_core_framework.cmake +++ b/tensorflow/contrib/cmake/tf_core_framework.cmake @@ -341,9 +341,3 @@ add_dependencies(tf_core_framework tf_core_lib proto_text ) - -if(WIN32) - # Cmake > 3.6 will quote this as -D"__VERSION__=\"MSVC\"" which nvcc fails on. - # Instead of defining this global, limit it to tf_core_framework where its used. - target_compile_definitions(tf_core_framework PRIVATE __VERSION__="MSVC") -endif() diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index 528f811b40..bfb28d22a9 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -163,7 +163,6 @@ def if_override_eigen_strong_inline(a): def get_win_copts(is_external=False): WINDOWS_COPTS = [ - "/D__VERSION__=\\\"MSVC\\\"", "/DPLATFORM_WINDOWS", "/DEIGEN_HAS_C99_MATH", "/DTENSORFLOW_USE_EIGEN_THREADPOOL", diff --git a/tensorflow/tools/git/gen_git_source.py b/tensorflow/tools/git/gen_git_source.py index 6a1f126131..372329b70c 100755 --- a/tensorflow/tools/git/gen_git_source.py +++ b/tensorflow/tools/git/gen_git_source.py @@ -178,7 +178,15 @@ def write_version_info(filename, git_version): contents = """/* Generated by gen_git_source.py */ #include const char* tf_git_version() {return "%s";} -const char* tf_compiler_version() {return __VERSION__;} +const char* tf_compiler_version() { +#ifdef _MSC_VER +#define STRINGIFY(x) #x +#define TOSTRING(x) STRINGIFY(x) + return "MSVC " TOSTRING(_MSC_FULL_VER); +#else + return __VERSION__; +#endif +} const int tf_cxx11_abi_flag() { #ifdef _GLIBCXX_USE_CXX11_ABI return _GLIBCXX_USE_CXX11_ABI; diff --git a/tensorflow/tools/git/gen_git_source.sh b/tensorflow/tools/git/gen_git_source.sh index db20bb00e8..cd128af6b3 100755 --- a/tensorflow/tools/git/gen_git_source.sh +++ b/tensorflow/tools/git/gen_git_source.sh @@ -28,7 +28,15 @@ fi cat < ${OUTPUT_FILENAME} #include const char* tf_git_version() {return "${GIT_VERSION}";} -const char* tf_compiler_version() {return __VERSION__;} +const char* tf_compiler_version() { +#ifdef _MSC_VER +#define STRINGIFY(x) #x +#define TOSTRING(x) STRINGIFY(x) + return "MSVC " TOSTRING(_MSC_FULL_VER); +#else + return __VERSION__; +#endif +} const int tf_cxx11_abi_flag() { #ifdef _GLIBCXX_USE_CXX11_ABI return _GLIBCXX_USE_CXX11_ABI; -- GitLab From 393a65caac76f5b4a3fa4c3edc98000a4a62b2e4 Mon Sep 17 00:00:00 2001 From: Rholais Lii Date: Fri, 13 Apr 2018 03:57:39 +0800 Subject: [PATCH 260/791] Reorder section `Using SavedModel with Estimators` (#18412) Outputs should be specified before performing an export. --- .../docs_src/programmers_guide/saved_model.md | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/tensorflow/docs_src/programmers_guide/saved_model.md b/tensorflow/docs_src/programmers_guide/saved_model.md index 55ee42dd64..c6ef87c54a 100644 --- a/tensorflow/docs_src/programmers_guide/saved_model.md +++ b/tensorflow/docs_src/programmers_guide/saved_model.md @@ -485,31 +485,7 @@ portion of the signature. That is, when writing a to expect and how to map them to your model's expected inputs. By contrast, the *output* portion of the signature is determined by the model. - -### Perform the export - -To export your trained Estimator, call -@{tf.estimator.Estimator.export_savedmodel} with the export base path and -the `serving_input_receiver_fn`. - -```py -estimator.export_savedmodel(export_dir_base, serving_input_receiver_fn, - strip_default_attrs=True) -``` - -This method builds a new graph by first calling the -`serving_input_receiver_fn()` to obtain feature `Tensor`s, and then calling -this `Estimator`'s `model_fn()` to generate the model graph based on those -features. It starts a fresh `Session`, and, by default, restores the most recent -checkpoint into it. (A different checkpoint may be passed, if needed.) -Finally it creates a time-stamped export directory below the given -`export_dir_base` (i.e., `export_dir_base/`), and writes a -SavedModel into it containing a single `MetaGraphDef` saved from this -Session. - -> Note: It is your responsibility to garbage-collect old exports. -> Otherwise, successive exports will accumulate under `export_dir_base`. - + ### Specify the outputs of a custom model When writing a custom `model_fn`, you must populate the `export_outputs` element @@ -541,6 +517,30 @@ using [`signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY`](https://www.tens indicating which `SignatureDef` will be served when an inference request does not specify one. + +### Perform the export + +To export your trained Estimator, call +@{tf.estimator.Estimator.export_savedmodel} with the export base path and +the `serving_input_receiver_fn`. + +```py +estimator.export_savedmodel(export_dir_base, serving_input_receiver_fn, + strip_default_attrs=True) +``` + +This method builds a new graph by first calling the +`serving_input_receiver_fn()` to obtain feature `Tensor`s, and then calling +this `Estimator`'s `model_fn()` to generate the model graph based on those +features. It starts a fresh `Session`, and, by default, restores the most recent +checkpoint into it. (A different checkpoint may be passed, if needed.) +Finally it creates a time-stamped export directory below the given +`export_dir_base` (i.e., `export_dir_base/`), and writes a +SavedModel into it containing a single `MetaGraphDef` saved from this +Session. + +> Note: It is your responsibility to garbage-collect old exports. +> Otherwise, successive exports will accumulate under `export_dir_base`. ### Serve the exported model locally -- GitLab From 9e3077475cf86d8ed615a478984818d84b37d29c Mon Sep 17 00:00:00 2001 From: brett koonce Date: Thu, 12 Apr 2018 12:57:48 -0700 Subject: [PATCH 261/791] contrib: minor spelling tweaks (#18330) * contrib: minor spelling tweaks * Fix lint error --- .../estimator/python/estimator/replicate_model_fn.py | 4 ++-- .../python/ops/fused_conv2d_bias_activation_op.py | 2 +- .../python/ops/fused_conv2d_bias_activation_op_test.py | 10 +++++----- .../kernel_tests/sparse_feature_cross_op_test.py | 2 +- .../contrib/layers/python/layers/feature_column.py | 2 +- .../contrib/layers/python/layers/feature_column_ops.py | 4 ++-- tensorflow/contrib/layers/python/layers/layers.py | 4 ++-- .../meta_graph_transform/meta_graph_transform.py | 2 +- tensorflow/contrib/optimizer_v2/optimizer_v2.py | 2 +- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py b/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py index fa2697800e..a8774d6dab 100644 --- a/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py +++ b/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py @@ -456,7 +456,7 @@ def _get_local_devices(device_type): def _split_batch(features, labels, number_of_shards, device): - """Split input features and labes into batches.""" + """Split input features and labels into batches.""" def ensure_divisible_by_shards(sequence): batch_size = ops_lib.convert_to_tensor(sequence).get_shape()[0] @@ -602,7 +602,7 @@ def _local_device_setter(worker_device, ps_devices, ps_strategy): def _scale_tower_loss(tower_spec, loss_reduction, number_of_towers): - """Produce an EstimatorSpec with approproriately scaled loss.""" + """Produce an EstimatorSpec with appropriately scaled loss.""" if tower_spec.loss is None: return tower_spec diff --git a/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op.py b/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op.py index a97adf622e..983b6dc8e5 100644 --- a/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op.py +++ b/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op.py @@ -65,7 +65,7 @@ def fused_conv2d_bias_activation(conv_input, side_input_scale: A scalar `float32` that will be multiplied by side_input. This is optional and defaults to 0. side_input: A `Tensor` of the format specified by `data_format`. - This is useful for imlementing ResNet blocks. + This is useful for implementing ResNet blocks. activation_mode: (optional) currently must be the default "Relu". Note that in qint8 mode, it also clips to 127, so acts like ReluX. data_format: Specifies the data format. diff --git a/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test.py b/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test.py index bb155aa249..3d0ed89932 100644 --- a/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test.py +++ b/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test.py @@ -566,7 +566,7 @@ def GetInceptionFwdTest(input_size, filter_size, stride, padding, return Test -def CalculateCovolvedOutputDim(input_dim, filter_dim, stride, padding_type): +def CalculateConvolvedOutputDim(input_dim, filter_dim, stride, padding_type): """Calculates the size of an output dimension of a strided convolution. Given the sizes of the corresponding dimension of the input and filter shapes, @@ -827,10 +827,10 @@ class FusedConvInt8Tests(test.TestCase): maxval=1.0, dtype=dtypes.float32), -1.0, 1.0, dtypes.qint8) - output_height = CalculateCovolvedOutputDim(input_height, filter_height, - vertical_stride, padding_type) - output_width = CalculateCovolvedOutputDim(input_width, filter_width, - horizontal_stride, padding_type) + output_height = CalculateConvolvedOutputDim(input_height, filter_height, + vertical_stride, padding_type) + output_width = CalculateConvolvedOutputDim(input_width, filter_width, + horizontal_stride, padding_type) print("output_height=", output_height, ", output_width=", output_width) side_input, _, _ = gen_array_ops.quantize_v2( diff --git a/tensorflow/contrib/layers/python/kernel_tests/sparse_feature_cross_op_test.py b/tensorflow/contrib/layers/python/kernel_tests/sparse_feature_cross_op_test.py index f701647c2b..28ddaa69a1 100644 --- a/tensorflow/contrib/layers/python/kernel_tests/sparse_feature_cross_op_test.py +++ b/tensorflow/contrib/layers/python/kernel_tests/sparse_feature_cross_op_test.py @@ -200,7 +200,7 @@ class SparseCrossOpTest(test.TestCase): self._assert_sparse_tensor_equals(expected_out, sess.run(op)) def test_large_batch(self): - """Tests with large batch size to force multithreding. + """Tests with large batch size to force multithreading. """ batch_size = 5000 col1 = [] diff --git a/tensorflow/contrib/layers/python/layers/feature_column.py b/tensorflow/contrib/layers/python/layers/feature_column.py index 9ccb589d69..3ae07cedab 100644 --- a/tensorflow/contrib/layers/python/layers/feature_column.py +++ b/tensorflow/contrib/layers/python/layers/feature_column.py @@ -48,7 +48,7 @@ you should choose depends on (1) the feature type and (2) the model type. recommended. embedded_dept_column = embedding_column( - sparse_column_with_keys("department", ["math", "philosphy", ...]), + sparse_column_with_keys("department", ["math", "philosophy", ...]), dimension=10) * Wide (aka linear) models (`LinearClassifier`, `LinearRegressor`). diff --git a/tensorflow/contrib/layers/python/layers/feature_column_ops.py b/tensorflow/contrib/layers/python/layers/feature_column_ops.py index 78affea44c..06060b99e7 100644 --- a/tensorflow/contrib/layers/python/layers/feature_column_ops.py +++ b/tensorflow/contrib/layers/python/layers/feature_column_ops.py @@ -815,7 +815,7 @@ class _Transformer(object): """ def __init__(self, columns_to_tensors): - """Initializes transfomer. + """Initializes transformer. Args: columns_to_tensors: A mapping from feature columns to tensors. 'string' @@ -908,7 +908,7 @@ def _gather_feature_columns(feature_columns): def _check_forbidden_sequence_columns(feature_columns): - """Recursively cecks `feature_columns` for `_FORBIDDEN_SEQUENCE_COLUMNS`.""" + """Recursively checks `feature_columns` for `_FORBIDDEN_SEQUENCE_COLUMNS`.""" all_feature_columns = _gather_feature_columns(feature_columns) for feature_column in all_feature_columns: if isinstance(feature_column, _FORBIDDEN_SEQUENCE_COLUMNS): diff --git a/tensorflow/contrib/layers/python/layers/layers.py b/tensorflow/contrib/layers/python/layers/layers.py index 949e73deff..151fc7a0d7 100644 --- a/tensorflow/contrib/layers/python/layers/layers.py +++ b/tensorflow/contrib/layers/python/layers/layers.py @@ -1542,7 +1542,7 @@ def dense_to_sparse(tensor, eos_token=0, outputs_collections=None, scope=None): Args: tensor: An `int` `Tensor` to be converted to a `Sparse`. eos_token: An integer. - It is part of the target label that signfies the end of a sentence. + It is part of the target label that signifies the end of a sentence. outputs_collections: Collection to add the outputs. scope: Optional scope for name_scope. """ @@ -1686,7 +1686,7 @@ def _inner_flatten(inputs, new_rank, output_collections=None, scope=None): output_collections: Collection to which the outputs will be added. scope: Optional scope for `name_scope`. Returns: - A `Tensor` or `SparseTensor` conataining the same values as `inputs`, but + A `Tensor` or `SparseTensor` containing the same values as `inputs`, but with innermost dimensions flattened to obtain rank `new_rank`. Raises: diff --git a/tensorflow/contrib/meta_graph_transform/meta_graph_transform.py b/tensorflow/contrib/meta_graph_transform/meta_graph_transform.py index ff88b4fa84..4fe4e8d044 100644 --- a/tensorflow/contrib/meta_graph_transform/meta_graph_transform.py +++ b/tensorflow/contrib/meta_graph_transform/meta_graph_transform.py @@ -348,7 +348,7 @@ def _freeze_graph_with_def_protos(input_graph_def, output_node_names, input_saver_def, input_checkpoint): """Converts all variables in a graph and checkpoint into constants. - During this process, we need to retain certain initialzer nodes (e.g. table + During this process, we need to retain certain initializer nodes (e.g. table initializer nodes). Instead of determining which dependencies of the shared initializer node (e.g. group_deps) to keep, we reconstruct the connections between the individual initializer nodes and diff --git a/tensorflow/contrib/optimizer_v2/optimizer_v2.py b/tensorflow/contrib/optimizer_v2/optimizer_v2.py index 25d19578ea..ce15db6f1e 100644 --- a/tensorflow/contrib/optimizer_v2/optimizer_v2.py +++ b/tensorflow/contrib/optimizer_v2/optimizer_v2.py @@ -579,7 +579,7 @@ class OptimizerV2(optimizer_v1.Optimizer): ### State - Internal methods apre passed a `state` argument with the correct + Internal methods are passed a `state` argument with the correct values to use for the slot and non-slot variables, and the hyper parameters. """ -- GitLab From 5592a96a5195dc4e5f49a1e3ca4243faa094ff85 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 12 Apr 2018 12:58:04 -0700 Subject: [PATCH 262/791] Fix WARNING in BatchNormalization (#18315) The keep_dims for reduce_mean has been deprecated and replaced with keepdims. This casues the following WARNING in BatchNormalization: ``` normalization.py:584: calling reduce_mean (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version. Instructions for updating: keep_dims is deprecated, use keepdims instead ``` This fix fixes the warning in BatchNormalization. Signed-off-by: Yong Tang --- tensorflow/python/keras/_impl/keras/layers/normalization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/keras/_impl/keras/layers/normalization.py b/tensorflow/python/keras/_impl/keras/layers/normalization.py index b73025a5a8..69332c21e1 100644 --- a/tensorflow/python/keras/_impl/keras/layers/normalization.py +++ b/tensorflow/python/keras/_impl/keras/layers/normalization.py @@ -592,9 +592,9 @@ class BatchNormalization(Layer): # used during evaluation, it is more efficient to just update in one # step and should not make a significant difference in the result. new_mean = math_ops.reduce_mean(new_mean, - axis=1, keep_dims=True) + axis=1, keepdims=True) new_variance = math_ops.reduce_mean(new_variance, - axis=1, keep_dims=True) + axis=1, keepdims=True) def _do_update(var, value): if in_eager_mode and not self.trainable: -- GitLab From 9efffac056fd2e01755a0bc1059f20ff6448f35d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Fri, 13 Apr 2018 03:58:22 +0800 Subject: [PATCH 263/791] remove the misleading n_class information (#18305) * DOC: modify the misleading n_class info * DOC: add suggested fix --- tensorflow/python/estimator/canned/head.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/estimator/canned/head.py b/tensorflow/python/estimator/canned/head.py index 189b81aeea..5e61c30ea2 100644 --- a/tensorflow/python/estimator/canned/head.py +++ b/tensorflow/python/estimator/canned/head.py @@ -263,9 +263,12 @@ def _check_dense_labels_match_logits_and_reshape( if (dim1 is not None) and (dim1 != expected_labels_dimension): raise ValueError( 'Mismatched label shape. ' - 'Classifier configured with n_classes=%s. Received %s. ' - 'Suggested Fix: check your n_classes argument to the estimator ' - 'and/or the shape of your label.' % + 'Expected labels dimension=%s. Received %s. ' + 'Suggested Fix:' + 'If your classifier expects one-hot encoding label,' + 'check your n_classes argument to the estimator' + 'and/or the shape of your label.' + 'Otherwise, check the shape of your label.' % (expected_labels_dimension, dim1)) expected_labels_shape = array_ops.concat( [logits_shape[:-1], [expected_labels_dimension]], axis=0) -- GitLab From 12da1017c6182afefd53d707dadd0ea76ce658a1 Mon Sep 17 00:00:00 2001 From: brett koonce Date: Thu, 12 Apr 2018 12:58:36 -0700 Subject: [PATCH 264/791] contrib/autograph: minor spelling tweaks (#18284) --- tensorflow/contrib/autograph/converters/call_trees.py | 2 +- .../contrib/autograph/converters/call_trees_test.py | 2 +- .../contrib/autograph/converters/decorators_test.py | 2 +- tensorflow/contrib/autograph/impl/api.py | 4 ++-- tensorflow/contrib/autograph/impl/conversion.py | 2 +- .../contrib/autograph/pyct/static_analysis/activity.py | 6 +++--- .../autograph/pyct/static_analysis/activity_test.py | 2 +- .../contrib/autograph/pyct/static_analysis/annos.py | 8 ++++---- tensorflow/contrib/autograph/utils/builtins.py | 2 +- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tensorflow/contrib/autograph/converters/call_trees.py b/tensorflow/contrib/autograph/converters/call_trees.py index 61f6bfd7e7..e22895ed6a 100644 --- a/tensorflow/contrib/autograph/converters/call_trees.py +++ b/tensorflow/contrib/autograph/converters/call_trees.py @@ -147,7 +147,7 @@ class CallTreeTransformer(transformer.Base): # Inspect the target function decorators. If any include a @convert # or @graph_ready annotation, then they must be called as they are. # TODO(mdan): This may be quite heavy. - # To parse and re-analize each function for every call site could be quite + # To parse and re-analyze each function for every call site could be quite # wasteful. Maybe we could cache the parsed AST? try: target_node, _ = parser.parse_entity(target_entity) diff --git a/tensorflow/contrib/autograph/converters/call_trees_test.py b/tensorflow/contrib/autograph/converters/call_trees_test.py index c666dcb73b..303dd54a4e 100644 --- a/tensorflow/contrib/autograph/converters/call_trees_test.py +++ b/tensorflow/contrib/autograph/converters/call_trees_test.py @@ -34,7 +34,7 @@ class CallTreesTest(converter_test_base.TestCase): def test_basic(self): def test_fn_1(_): - raise ValueError('This should not be called in the compiled verison.') + raise ValueError('This should not be called in the compiled version.') def renamed_test_fn_1(a): return a + 1 diff --git a/tensorflow/contrib/autograph/converters/decorators_test.py b/tensorflow/contrib/autograph/converters/decorators_test.py index e67ab1cd6a..9c01f68912 100644 --- a/tensorflow/contrib/autograph/converters/decorators_test.py +++ b/tensorflow/contrib/autograph/converters/decorators_test.py @@ -28,7 +28,7 @@ from tensorflow.python.platform import test # The Python parser only briefly captures decorators into the AST. # The interpreter desugars them on load, and the decorated function loses any -# trace of the decorator (which is notmally what you would expect, since +# trace of the decorator (which is normally what you would expect, since # they are meant to be transparent). # However, decorators are still visible when you analyze the function # from inside a decorator, before it was applied - as is the case diff --git a/tensorflow/contrib/autograph/impl/api.py b/tensorflow/contrib/autograph/impl/api.py index dce994e50d..b1731480be 100644 --- a/tensorflow/contrib/autograph/impl/api.py +++ b/tensorflow/contrib/autograph/impl/api.py @@ -49,7 +49,7 @@ def convert(recursive=False, verbose=False, arg_types=None): function is called. This means the parameter values are known at compilation. Args: - recursive: Whether to recusrively convert any functions that the decorator + recursive: Whether to recursively convert any functions that the decorator function may call. verbose: Whether to output the compiled code in the logs. arg_types: See to_graph. @@ -215,7 +215,7 @@ def to_graph(e, Args: e: A Python entity. - recursive: Whether to recusrively convert any functions that the decorator + recursive: Whether to recursively convert any functions that the decorator function may call. verbose: Whether to output the compiled code in the logs. arg_values: A dict containing value hints for symbols like function diff --git a/tensorflow/contrib/autograph/impl/conversion.py b/tensorflow/contrib/autograph/impl/conversion.py index 3bacc94300..240e070368 100644 --- a/tensorflow/contrib/autograph/impl/conversion.py +++ b/tensorflow/contrib/autograph/impl/conversion.py @@ -56,7 +56,7 @@ class ConversionMap(object): This object is mutable, and is updated as functions are converted. Attributes: - recursive: Whether to recusrively convert any functions that the decorator + recursive: Whether to recursively convert any functions that the decorator function may call. nocompile_decorators: tuple of decorator functions that toggle compilation off. diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/activity.py b/tensorflow/contrib/autograph/pyct/static_analysis/activity.py index 6dd53091fa..b6817e9d75 100644 --- a/tensorflow/contrib/autograph/pyct/static_analysis/activity.py +++ b/tensorflow/contrib/autograph/pyct/static_analysis/activity.py @@ -162,11 +162,11 @@ class Scope(object): self.parent.mark_returned(name) -class ActivityAnalizer(transformer.Base): +class ActivityAnalyzer(transformer.Base): """Annotates nodes with local scope information. See Scope.""" def __init__(self, context, parent_scope): - super(ActivityAnalizer, self).__init__(context) + super(ActivityAnalyzer, self).__init__(context) self.scope = Scope(parent_scope) self._in_return_statement = False @@ -323,4 +323,4 @@ class ActivityAnalizer(transformer.Base): def resolve(node, context, parent_scope=None): - return ActivityAnalizer(context, parent_scope).visit(node) + return ActivityAnalyzer(context, parent_scope).visit(node) diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/activity_test.py b/tensorflow/contrib/autograph/pyct/static_analysis/activity_test.py index 1e6c686b01..65e1a8f0ea 100644 --- a/tensorflow/contrib/autograph/pyct/static_analysis/activity_test.py +++ b/tensorflow/contrib/autograph/pyct/static_analysis/activity_test.py @@ -108,7 +108,7 @@ class ScopeTest(test.TestCase): self.assertFalse(QN('a') in child.referenced) -class ActivityAnalizerTest(test.TestCase): +class ActivityAnalyzerTest(test.TestCase): def _parse_and_analyze(self, test_fn): node, source = parser.parse_entity(test_fn) diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/annos.py b/tensorflow/contrib/autograph/pyct/static_analysis/annos.py index d6d9f7e1a6..b929b35b79 100644 --- a/tensorflow/contrib/autograph/pyct/static_analysis/annos.py +++ b/tensorflow/contrib/autograph/pyct/static_analysis/annos.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Annotations used by the static analizer.""" +"""Annotations used by the static analyzer.""" from __future__ import absolute_import from __future__ import division @@ -28,15 +28,15 @@ class NoValue(Enum): class NodeAnno(NoValue): - """Additionnal annotations used by the static analyzer. + """Additional annotations used by the static analyzer. These are in addition to the basic annotations declared in anno.py. """ # Symbols # These flags are boolean. - IS_LOCAL = 'Symbol is local to the function scope being analized.' - IS_PARAM = 'Symbol is a parameter to the function being analized.' + IS_LOCAL = 'Symbol is local to the function scope being analyzed.' + IS_PARAM = 'Symbol is a parameter to the function being analyzed.' IS_MODIFIED_SINCE_ENTRY = ( 'Symbol has been explicitly replaced in the current function scope.') diff --git a/tensorflow/contrib/autograph/utils/builtins.py b/tensorflow/contrib/autograph/utils/builtins.py index 7fbb7c09d8..0a0e72d70e 100644 --- a/tensorflow/contrib/autograph/utils/builtins.py +++ b/tensorflow/contrib/autograph/utils/builtins.py @@ -84,7 +84,7 @@ def is_tf_print_compatible(value): def dynamic_print(*values): - """Implementartion of print using dynamic dispatch. + """Implementation of print using dynamic dispatch. The function attempts to use tf.Print if all the values are compatible. Otherwise, it will fall back to py_func. -- GitLab From 462b5d28061d7983aa852f09c9ee94e5957f58dd Mon Sep 17 00:00:00 2001 From: Wai Hon Law Date: Thu, 12 Apr 2018 12:58:44 -0700 Subject: [PATCH 265/791] Change --output_png to --output_image (#18273) The argument is incorrect. When running the given command, we get ``` E tensorflow/examples/wav_to_spectrogram/main.cc:54] Unknown argument --output_png=/tmp/spectrogram.png ``` TESTED:Rerun the updated command and verify that the flag is correct. ``` bazel run tensorflow/examples/wav_to_spectrogram:wav_to_spectrogram -- --input_wav=/tmp/speech_dataset/happy/ab00c4b2_nohash_0.wav --output_image=/tmp/spectrogram.png ``` --- tensorflow/docs_src/tutorials/audio_recognition.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/docs_src/tutorials/audio_recognition.md b/tensorflow/docs_src/tutorials/audio_recognition.md index 7d79f433c4..372ab47df7 100644 --- a/tensorflow/docs_src/tutorials/audio_recognition.md +++ b/tensorflow/docs_src/tutorials/audio_recognition.md @@ -280,7 +280,7 @@ tool: ``` bazel run tensorflow/examples/wav_to_spectrogram:wav_to_spectrogram -- \ --input_wav=/tmp/speech_dataset/happy/ab00c4b2_nohash_0.wav \ ---output_png=/tmp/spectrogram.png +--output_image=/tmp/spectrogram.png ``` If you open up `/tmp/spectrogram.png` you should see something like this: -- GitLab From 5c237ddfcc0e54427e4fc31cccff809d65e66321 Mon Sep 17 00:00:00 2001 From: Shaoning Zeng Date: Fri, 13 Apr 2018 03:58:59 +0800 Subject: [PATCH 266/791] give some writing updates to tensorflow/contrib/slim/README.md (#18259) * add missed right bracket in ### Scopes * change one , to . in ### Scopes * refine one sentence --- tensorflow/contrib/slim/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/contrib/slim/README.md b/tensorflow/contrib/slim/README.md index 40f484fd78..746b955642 100644 --- a/tensorflow/contrib/slim/README.md +++ b/tensorflow/contrib/slim/README.md @@ -290,9 +290,9 @@ slim.stack(x, slim.conv2d, [(32, [3, 3]), (32, [1, 1]), (64, [3, 3]), (64, [1, 1 In addition to the types of scope mechanisms in TensorFlow ([name_scope](https://www.tensorflow.org/api_docs/python/tf/name_scope), -[variable_scope](https://www.tensorflow.org/api_docs/python/tf/variable_scope), +[variable_scope](https://www.tensorflow.org/api_docs/python/tf/variable_scope)), TF-Slim adds a new scoping mechanism called -[arg_scope](https://www.tensorflow.org/api_docs/python/tf/contrib/framework/arg_scope), +[arg_scope](https://www.tensorflow.org/api_docs/python/tf/contrib/framework/arg_scope). This new scope allows a user to specify one or more operations and a set of arguments which will be passed to each of the operations defined in the `arg_scope`. This functionality is best illustrated by example. Consider the @@ -761,8 +761,8 @@ parts: 3. Finalization: (optionally) perform any final operation to compute metric values. For example, computing means, mins, maxes, etc. -For example, to compute `mean_absolute_error`, two variables, a `count` and -`total` variable are *initialized* to zero. During *aggregation*, we observed +For example, to compute `mean_absolute_error`, two variables (`count` and +`total`) are *initialized* to zero. During *aggregation*, we observed some set of predictions and labels, compute their absolute differences and add the total to `total`. Each time we observe another value, `count` is incremented. Finally, during *finalization*, `total` is divided -- GitLab From 4c7fe9e83f206fc177dd6deaa6a1338b6192f263 Mon Sep 17 00:00:00 2001 From: Quanlong Date: Fri, 13 Apr 2018 03:59:26 +0800 Subject: [PATCH 267/791] Latest nngraph cannot build with Hexagon SDK 3.0 (#17963) * fix: latest nngraph cannot build with Hexagon SDK 3.0 * Fix typo --- tensorflow/contrib/hvx/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/hvx/README.md b/tensorflow/contrib/hvx/README.md index 163993a3f6..68e34f3b09 100644 --- a/tensorflow/contrib/hvx/README.md +++ b/tensorflow/contrib/hvx/README.md @@ -42,11 +42,12 @@ If you've finished walking through the quick start guide, you may want to try bu ### Build libhexagon\_nn\_skel.so -Download Hexagon NN library from codeaurora.org and build it. +Download Hexagon NN library from codeaurora.org and build it. For Hexagon SDK 3.0, we need use the compatible version([721b2d58f](https://source.codeaurora.org/quic/hexagon_nn/nnlib/commit/?id=721b2d58f0f4e2d5b182f41e6b7c4db5356bf0fb)) of nnlib. ```shell git clone https://source.codeaurora.org/quic/hexagon_nn/nnlib cd nnlib +git reset 721b2d58f --hard ``` Just follow the instructions in `README.HOW_TO_BUILD`. You can find the file `libhexagon_nn_skel.so` in `hexagon_Release_dynamic_toolv72_v60/ship`. -- GitLab From ace33c76636ed908958888243131524091085f96 Mon Sep 17 00:00:00 2001 From: Yihong Wang Date: Thu, 12 Apr 2018 12:59:48 -0700 Subject: [PATCH 268/791] Link to gcc_s and gcc if compiler is GCC version 5 (#17849) When using cmake and GCC 5.4 to build tensorflow in Ubuntu 16.04, the following error message would show up when loading _pywrap_tensorflow_internal.so: ``` _pywrap_tensorflow_internal.so: undefined symbol: __cpu_model ``` The root cause is the same to this issue: https://github.com/tensorflow/tensorflow/issues/9593 Signed-off-by: Yihong Wang --- tensorflow/contrib/cmake/tf_python.cmake | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake index ded15b4b66..1c3206f1a2 100755 --- a/tensorflow/contrib/cmake/tf_python.cmake +++ b/tensorflow/contrib/cmake/tf_python.cmake @@ -586,6 +586,12 @@ add_library(pywrap_tensorflow_internal SHARED ${pywrap_tensorflow_deffile} ) +# There is a bug in GCC 5 resulting in undefined reference to a __cpu_model function when +# linking to the tensorflow library. Adding the following libraries fixes it. +if(CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 5.0) + target_link_libraries(pywrap_tensorflow_internal PRIVATE gcc_s gcc) +endif() + if(WIN32) add_dependencies(pywrap_tensorflow_internal pywrap_tensorflow_internal_static) endif(WIN32) -- GitLab From d68ef84dc9bc99bb4d06a48ad847f13f0c8d0396 Mon Sep 17 00:00:00 2001 From: David Norman Date: Thu, 12 Apr 2018 21:00:12 +0100 Subject: [PATCH 269/791] Allow for devices which have F16, no F64, no Complex (#17473) --- tensorflow/compiler/xla/tests/dot_operation_test.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tensorflow/compiler/xla/tests/dot_operation_test.cc b/tensorflow/compiler/xla/tests/dot_operation_test.cc index 7b994a4c17..c4031dfee5 100644 --- a/tensorflow/compiler/xla/tests/dot_operation_test.cc +++ b/tensorflow/compiler/xla/tests/dot_operation_test.cc @@ -50,6 +50,13 @@ using TypesF16F32 = ::testing::Types; using TypesF16F32F64 = ::testing::Types; using TypesF16F32F64CF64 = ::testing::Types; +#elif !defined(XLA_BACKEND_DOES_NOT_SUPPORT_FLOAT16) && \ + defined(XLA_BACKEND_DOES_NOT_SUPPORT_FLOAT64) && \ + defined(XLA_BACKEND_DOES_NOT_SUPPORT_COMPLEX) +using TypesF16F32 = ::testing::Types; +using TypesF16F32F64 = ::testing::Types; +using TypesF16F32F64CF64 = + ::testing::Types; #else #error "Situation not handled yet" #endif -- GitLab From 0253b68db7ccb0537b46052cbcac7715861ac22b Mon Sep 17 00:00:00 2001 From: Seyed Majid Azimi Date: Thu, 12 Apr 2018 22:00:33 +0200 Subject: [PATCH 270/791] Update nn.py (#17247) adding missing quantized_relu which was missing before. --- tensorflow/python/ops/nn.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/python/ops/nn.py b/tensorflow/python/ops/nn.py index 244702d13b..1d0d9a52a1 100644 --- a/tensorflow/python/ops/nn.py +++ b/tensorflow/python/ops/nn.py @@ -98,6 +98,7 @@ See the @{$python/nn} guide. @@fixed_unigram_candidate_sampler @@compute_accidental_hits @@quantized_conv2d +@@quantized_relu @@quantized_relu_x @@quantized_max_pool @@quantized_avg_pool -- GitLab From e40fec4a9563cfe021243f63beda51afcc6d13ef Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Thu, 12 Apr 2018 12:58:05 -0700 Subject: [PATCH 271/791] Upgrade libjpeg-turbo NOTE: libjpeg-turbo 1.5.90 also exists, which adds AVX2 SIMD support. However it also comes with a CMake build rewrite and 35 micro-architecture specialized nasm files for x86_64 alone. We do not have the cycles to update jpeg.BUILD to incorporate those changes at this time. If anyone wants to try, please note we'd need predicates such as the following: config_setting( name = "haswell_opt", values = { "cpu": "haswell", # First Intel chip with AVX2 "compilation_mode": "opt", }, visibility = ["//visibility:public"], ) config_setting( name = "excavator_opt", values = { "cpu": "excavator", # First AMD chip with AVX2 "compilation_mode": "opt", }, visibility = ["//visibility:public"], ) PiperOrigin-RevId: 192655533 --- tensorflow/workspace.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 52168a89c5..72f446d359 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -210,11 +210,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""): tf_http_archive( name = "jpeg", urls = [ - "https://mirror.bazel.build/github.com/libjpeg-turbo/libjpeg-turbo/archive/1.5.1.tar.gz", - "https://github.com/libjpeg-turbo/libjpeg-turbo/archive/1.5.1.tar.gz", + "https://mirror.bazel.build/github.com/libjpeg-turbo/libjpeg-turbo/archive/1.5.3.tar.gz", + "https://github.com/libjpeg-turbo/libjpeg-turbo/archive/1.5.3.tar.gz", ], - sha256 = "c15a9607892113946379ccea3ca8b85018301b200754f209453ab21674268e77", - strip_prefix = "libjpeg-turbo-1.5.1", + sha256 = "1a17020f859cb12711175a67eab5c71fc1904e04b587046218e36106e07eabde", + strip_prefix = "libjpeg-turbo-1.5.3", build_file = clean_dep("//third_party/jpeg:jpeg.BUILD"), ) -- GitLab From 9a9a90e9f170045e752805b390064c25fcc69573 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 12 Apr 2018 13:01:05 -0700 Subject: [PATCH 272/791] Add tensor support for num_spectrogram_bins in linear_to_mel_weight_matrix (#17404) * Add tensor support for num_spectrogram_bins in linear_to_mel_weight_matrix This fix tries to address the issue raised in 16553 where it was not possible to provide num_spectrogram_bins as a tensor or placeholder for linear_to_mel_weight_matrix. The reason comes from the implementation of `_validate_arguments` which requires num_spectrogram_bins to be a python. However, the validation here is not necessary as `num_spectrogram_bins` will be passed to `math_ops.linspace`, which performs the validation anyway. The validation in `math_ops.linspace` is done in shape function and in kernel's `Compute()`. For that it makes sense to remove the validation of `num_spectrogram_bins` in `_validate_arguments` so that the issue raised in 16553 could be addressed. This fix adds a test case to cover the changes. Also, the error case of `num_spectrogram_bins < 0` has already been covered in the existing test case: https://github.com/tensorflow/tensorflow/blob/013a6c7b3112573ba4d932c8a22bfaf45f648c77/tensorflow/contrib/signal/python/kernel_tests/mel_ops_test.py#L149-L165 This fix fixes 16553. Signed-off-by: Yong Tang * Add test case for tensor support of num_spectrogram_bins in mel_ops.linear_to_mel_weight_matrix Signed-off-by: Yong Tang * Add comment for removing validation of num_spectrogram_bins Signed-off-by: Yong Tang * Update docstring Signed-off-by: Yong Tang * Update test case for num_spectrogram_bins Signed-off-by: Yong Tang * Remove unused constant_op import to pass sanity check Signed-off-by: Yong Tang --- .../signal/python/kernel_tests/mel_ops_test.py | 13 +++++++++++++ tensorflow/contrib/signal/python/ops/mel_ops.py | 16 ++++++++-------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/tensorflow/contrib/signal/python/kernel_tests/mel_ops_test.py b/tensorflow/contrib/signal/python/kernel_tests/mel_ops_test.py index 35c4b5bec1..345eb6cfaa 100644 --- a/tensorflow/contrib/signal/python/kernel_tests/mel_ops_test.py +++ b/tensorflow/contrib/signal/python/kernel_tests/mel_ops_test.py @@ -24,6 +24,7 @@ from tensorflow.contrib.signal.python.kernel_tests import test_util from tensorflow.contrib.signal.python.ops import mel_ops from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops from tensorflow.python.platform import test # mel spectrum constants and functions. @@ -173,6 +174,18 @@ class LinearToMelTest(test.TestCase): rewritten_graph = test_util.grappler_optimize(g, [mel_matrix]) self.assertEqual(1, len(rewritten_graph.node)) + def test_num_spectrogram_bins_dynamic(self): + with self.test_session(use_gpu=True): + num_spectrogram_bins = array_ops.placeholder(shape=(), + dtype=dtypes.int32) + mel_matrix_np = spectrogram_to_mel_matrix( + 20, 129, 8000.0, 125.0, 3800.0) + mel_matrix = mel_ops.linear_to_mel_weight_matrix( + 20, num_spectrogram_bins, 8000.0, 125.0, 3800.0) + self.assertAllClose( + mel_matrix_np, + mel_matrix.eval(feed_dict={num_spectrogram_bins: 129}), atol=3e-6) + if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/signal/python/ops/mel_ops.py b/tensorflow/contrib/signal/python/ops/mel_ops.py index d1a36548d9..1e84006116 100644 --- a/tensorflow/contrib/signal/python/ops/mel_ops.py +++ b/tensorflow/contrib/signal/python/ops/mel_ops.py @@ -64,14 +64,11 @@ def _hertz_to_mel(frequencies_hertz, name=None): 1.0 + (frequencies_hertz / _MEL_BREAK_FREQUENCY_HERTZ)) -def _validate_arguments(num_mel_bins, num_spectrogram_bins, sample_rate, +def _validate_arguments(num_mel_bins, sample_rate, lower_edge_hertz, upper_edge_hertz, dtype): """Checks the inputs to linear_to_mel_weight_matrix.""" if num_mel_bins <= 0: raise ValueError('num_mel_bins must be positive. Got: %s' % num_mel_bins) - if num_spectrogram_bins <= 0: - raise ValueError('num_spectrogram_bins must be positive. Got: %s' % - num_spectrogram_bins) if sample_rate <= 0.0: raise ValueError('sample_rate must be positive. Got: %s' % sample_rate) if lower_edge_hertz < 0.0: @@ -122,9 +119,9 @@ def linear_to_mel_weight_matrix(num_mel_bins=20, Args: num_mel_bins: Python int. How many bands in the resulting mel spectrum. - num_spectrogram_bins: Python int. How many bins there are in the source - spectrogram data, which is understood to be `fft_size // 2 + 1`, i.e. the - spectrogram only contains the nonredundant FFT bins. + num_spectrogram_bins: An integer `Tensor`. How many bins there are in the + source spectrogram data, which is understood to be `fft_size // 2 + 1`, + i.e. the spectrogram only contains the nonredundant FFT bins. sample_rate: Python float. Samples per second of the input signal used to create the spectrogram. We need this to figure out the actual frequencies for each spectrogram bin, which dictates how they are mapped into the mel @@ -148,7 +145,10 @@ def linear_to_mel_weight_matrix(num_mel_bins=20, [mel]: https://en.wikipedia.org/wiki/Mel_scale """ with ops.name_scope(name, 'linear_to_mel_weight_matrix') as name: - _validate_arguments(num_mel_bins, num_spectrogram_bins, sample_rate, + # Note: As num_spectrogram_bins is passed to `math_ops.linspace` + # and the validation is already done in linspace (both in shape function + # and in kernel), there is no need to validate num_spectrogram_bins here. + _validate_arguments(num_mel_bins, sample_rate, lower_edge_hertz, upper_edge_hertz, dtype) # To preserve accuracy, we compute the matrix at float64 precision and then -- GitLab From 18f8568ca2e2efedd482e1120d4a5b73aab7841c Mon Sep 17 00:00:00 2001 From: Russell Klopfer Date: Thu, 12 Apr 2018 16:01:25 -0400 Subject: [PATCH 273/791] crf_decode fails when sequence_length is 0 (#17755) * updating documentation * crf_decode fails when sequence_length is 0 * fixing line length * more pylint fixes --- .../contrib/crf/python/kernel_tests/crf_test.py | 15 +++++++++++++++ tensorflow/contrib/crf/python/ops/crf.py | 8 +++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/crf/python/kernel_tests/crf_test.py b/tensorflow/contrib/crf/python/kernel_tests/crf_test.py index 721dc4d080..a5e065b93a 100644 --- a/tensorflow/contrib/crf/python/kernel_tests/crf_test.py +++ b/tensorflow/contrib/crf/python/kernel_tests/crf_test.py @@ -281,6 +281,21 @@ class CrfTest(test.TestCase): self.assertEqual(list(tf_actual_max_sequence[:sequence_lengths]), expected_max_sequence[:sequence_lengths]) + def testCrfDecodeZeroSeqLength(self): + """ + Test that crf_decode works when sequence_length contains one or more zeros. + """ + with self.test_session() as sess: + inputs = constant_op.constant(np.ones([2, 10, 5], + dtype=np.float32)) + transition_params = constant_op.constant(np.ones([5, 5], + dtype=np.float32)) + sequence_lengths = constant_op.constant(np.zeros([2], + dtype=np.int32)) + values = crf.crf_decode(inputs, transition_params, sequence_lengths) + tags, scores = sess.run(values) + self.assertEqual(len(tags.shape), 2) + self.assertEqual(len(scores.shape), 1) if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/crf/python/ops/crf.py b/tensorflow/contrib/crf/python/ops/crf.py index 1233c8f251..e37c029ceb 100644 --- a/tensorflow/contrib/crf/python/ops/crf.py +++ b/tensorflow/contrib/crf/python/ops/crf.py @@ -479,15 +479,17 @@ def crf_decode(potentials, transition_params, sequence_length): initial_state = array_ops.slice(potentials, [0, 0, 0], [-1, 1, -1]) initial_state = array_ops.squeeze(initial_state, axis=[1]) # [B, O] inputs = array_ops.slice(potentials, [0, 1, 0], [-1, -1, -1]) # [B, T-1, O] + # sequence length is not allowed to be less than zero + sequence_length_less_one = math_ops.maximum(0, sequence_length - 1) backpointers, last_score = rnn.dynamic_rnn( # [B, T - 1, O], [B, O] crf_fwd_cell, inputs=inputs, - sequence_length=sequence_length - 1, + sequence_length=sequence_length_less_one, initial_state=initial_state, time_major=False, dtype=dtypes.int32) backpointers = gen_array_ops.reverse_sequence( # [B, T - 1, O] - backpointers, sequence_length - 1, seq_dim=1) + backpointers, sequence_length_less_one, seq_dim=1) # Computes backward decoding. Extract tag indices from backpointers. crf_bwd_cell = CrfDecodeBackwardRnnCell(num_tags) @@ -497,7 +499,7 @@ def crf_decode(potentials, transition_params, sequence_length): decode_tags, _ = rnn.dynamic_rnn( # [B, T - 1, 1] crf_bwd_cell, inputs=backpointers, - sequence_length=sequence_length - 1, + sequence_length=sequence_length_less_one, initial_state=initial_state, time_major=False, dtype=dtypes.int32) -- GitLab From 64eb9b445a79d571c26c3e63920402d3b0940c12 Mon Sep 17 00:00:00 2001 From: Priya Gupta Date: Thu, 12 Apr 2018 13:06:28 -0700 Subject: [PATCH 274/791] Separate out distribute dependency out of training, as it needs to be used in summary utils (which training depends on, thus causing circular dependency). PiperOrigin-RevId: 192656997 --- tensorflow/contrib/distribute/python/BUILD | 12 ++++++-- tensorflow/contrib/optimizer_v2/BUILD | 1 + tensorflow/python/BUILD | 33 +++++++++++++++++++++- tensorflow/python/estimator/BUILD | 5 ++++ tensorflow/python/keras/BUILD | 1 + 5 files changed, 48 insertions(+), 4 deletions(-) diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD index 78b2b0054a..51b4fbacd1 100644 --- a/tensorflow/contrib/distribute/python/BUILD +++ b/tensorflow/contrib/distribute/python/BUILD @@ -27,6 +27,8 @@ py_library( "//tensorflow/python:array_ops", "//tensorflow/python:checkpointable", "//tensorflow/python:control_flow_ops", + "//tensorflow/python:device_util", + "//tensorflow/python:distribute", "//tensorflow/python:framework_ops", "//tensorflow/python:training", "//tensorflow/python:util", @@ -51,6 +53,7 @@ cuda_py_test( "//tensorflow/python:training", "//tensorflow/python:variable_scope", "//tensorflow/python/eager:context", + "//tensorflow/python:device_util", "//tensorflow/python/eager:test", "//tensorflow/python/estimator:model_fn", ], @@ -66,6 +69,8 @@ py_library( ":values", "//tensorflow/python:array_ops", "//tensorflow/python:device", + "//tensorflow/python:device_util", + "//tensorflow/python:distribute", "//tensorflow/python:framework_ops", "//tensorflow/python:pywrap_tensorflow", "//tensorflow/python:training", @@ -84,9 +89,9 @@ py_library( ":values", "//tensorflow/contrib/eager/python:datasets", "//tensorflow/python:array_ops", + "//tensorflow/python:distribute", "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", - "//tensorflow/python:training", "//tensorflow/python/eager:context", "@six_archive//:six", ], @@ -104,6 +109,7 @@ py_library( "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", "//tensorflow/python:constant_op", + "//tensorflow/python:distribute", "//tensorflow/python:framework_ops", "//tensorflow/python:layers", "//tensorflow/python:training", @@ -156,8 +162,8 @@ py_test( deps = [ ":mirrored_strategy", ":strategy_test_lib", + "//tensorflow/python:distribute", "//tensorflow/python:framework_test_lib", - "//tensorflow/python:training", "//tensorflow/python:variable_scope", "//tensorflow/python/eager:context", "//tensorflow/python/eager:test", @@ -186,10 +192,10 @@ cuda_py_test( ":mirrored_strategy", ":values", ":strategy_test_lib", + "//tensorflow/python:distribute", "//tensorflow/core:protos_all_py", "//tensorflow/python:constant_op", "//tensorflow/python:layers", - "//tensorflow/python:training", "//tensorflow/python:variable_scope", "//tensorflow/python:array_ops", "//tensorflow/python:framework_test_lib", diff --git a/tensorflow/contrib/optimizer_v2/BUILD b/tensorflow/contrib/optimizer_v2/BUILD index 26ea9135f5..86e5f4a437 100644 --- a/tensorflow/contrib/optimizer_v2/BUILD +++ b/tensorflow/contrib/optimizer_v2/BUILD @@ -48,6 +48,7 @@ py_library( srcs_version = "PY2AND3", deps = [ "//tensorflow/python:control_flow_ops", + "//tensorflow/python:distribute", "//tensorflow/python:framework", "//tensorflow/python:math_ops", "//tensorflow/python:resource_variable_ops", diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 559926d415..72284fd50b 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -2955,6 +2955,7 @@ py_library( ":framework_ops", ":gradients", ":init_ops", + ":distribute", ":io_ops", ":io_ops_gen", ":layers_base", @@ -3012,6 +3013,35 @@ py_test( ], ) +py_library( + name = "device_util", + srcs = ["training/device_util.py"], + srcs_version = "PY2AND3", + deps = [ + ":device", + ":framework_ops", + "//tensorflow/python/eager:context", + ], +) + +py_library( + name = "distribute", + srcs = ["training/distribute.py"], + srcs_version = "PY2AND3", + deps = [ + ":array_ops", + ":control_flow_ops", + ":device_util", + ":framework_ops", + ":platform", + ":resource_variable_ops", + ":state_ops", + ":util", + ":variable_scope", + "//tensorflow/python/ops/losses", + ], +) + py_test( name = "checkpointable_utils_test", srcs = ["training/checkpointable_utils_test.py"], @@ -3052,7 +3082,7 @@ py_test( srcs_version = "PY2AND3", deps = [ ":client_testlib", - ":training", + ":distribute", ":variable_scope", ], ) @@ -4316,6 +4346,7 @@ py_library( srcs_version = "PY2AND3", visibility = ["//visibility:public"], deps = [ + ":distribute", ":framework", ":framework_for_generated_wrappers", ":platform", diff --git a/tensorflow/python/estimator/BUILD b/tensorflow/python/estimator/BUILD index 5d8b19223f..a34405c702 100644 --- a/tensorflow/python/estimator/BUILD +++ b/tensorflow/python/estimator/BUILD @@ -251,6 +251,7 @@ py_library( "//tensorflow/python:array_ops", "//tensorflow/python:boosted_trees_ops", "//tensorflow/python:data_flow_ops", + "//tensorflow/python:distribute", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", "//tensorflow/python:lookup_ops", @@ -327,6 +328,7 @@ py_library( "//tensorflow/python:client_testlib", "//tensorflow/python:constant_op", "//tensorflow/python:control_flow_ops", + "//tensorflow/python:distribute", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", @@ -383,6 +385,7 @@ py_library( ":model_fn", ":optimizers", "//tensorflow/python:control_flow_ops", + "//tensorflow/python:distribute", "//tensorflow/python:framework_ops", "//tensorflow/python:init_ops", "//tensorflow/python:layers", @@ -466,6 +469,7 @@ py_library( "//tensorflow/core:protos_all_py", "//tensorflow/python:client", "//tensorflow/python:control_flow_ops", + "//tensorflow/python:distribute", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:metrics", "//tensorflow/python:platform", @@ -743,6 +747,7 @@ py_library( "//tensorflow/python:client_testlib", "//tensorflow/python:control_flow_ops", "//tensorflow/python:data_flow_ops", + "//tensorflow/python:distribute", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD index da5bc3e6f1..024a8cd3d1 100755 --- a/tensorflow/python/keras/BUILD +++ b/tensorflow/python/keras/BUILD @@ -205,6 +205,7 @@ py_library( deps = [ ":engine", "//tensorflow/python:array_ops", + "//tensorflow/python:distribute", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", "//tensorflow/python:logging_ops", -- GitLab From 322580a5b704f0db72cd2bfa4e5a08f6b8c3b664 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Thu, 12 Apr 2018 13:24:32 -0700 Subject: [PATCH 275/791] Fix build breakage on metagraph exporting when caching_device is set PiperOrigin-RevId: 192659701 --- tensorflow/python/ops/resource_variable_ops.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py index c51d1e467d..49dd7f9948 100644 --- a/tensorflow/python/ops/resource_variable_ops.py +++ b/tensorflow/python/ops/resource_variable_ops.py @@ -522,12 +522,13 @@ class ResourceVariable(variables.Variable): else: self._initial_value = None if variable_def.snapshot_name: - self._cached_value = g.as_graph_element( + snapshot = g.as_graph_element( ops.prepend_name_scope( variable_def.snapshot_name, import_scope=import_scope)) - self._graph_element = g.as_graph_element( - ops.prepend_name_scope(variable_def.snapshot_name, - import_scope=import_scope)) + self._cached_value = snapshot + while snapshot.op.type != "ReadVariableOp": + snapshot = snapshot.op.inputs[0] + self._graph_element = snapshot else: self._cached_value = None # Legacy case for protos without the snapshot name; assume it's the -- GitLab From 111ee9ba4c7bcc736db9b79f967f380052a091e0 Mon Sep 17 00:00:00 2001 From: James Wexler Date: Thu, 12 Apr 2018 13:24:51 -0700 Subject: [PATCH 276/791] Make new build target public. PiperOrigin-RevId: 192659759 --- tensorflow/core/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 97e0095e05..c461f9ed2f 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -255,6 +255,7 @@ closure_js_proto_library( "example/example.proto", "example/feature.proto", ], + visibility = ["//visibility:public"], ) exports_files([ -- GitLab From 7c0172e0853f3262e1d85aa6bc37cf70d718cca0 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 14:31:08 -0700 Subject: [PATCH 277/791] ResolveConstantReshape transformation and fix for ResolveConstantTranspose. PiperOrigin-RevId: 192670991 --- tensorflow/contrib/lite/toco/BUILD | 1 + .../graph_transformations.h | 1 + .../remove_trivial_reshape.cc | 5 +- .../resolve_constant_reshape.cc | 124 ++++++++++++++++++ .../resolve_constant_transpose.cc | 6 + tensorflow/contrib/lite/toco/toco_tooling.cc | 1 + tensorflow/contrib/lite/toco/tooling_util.cc | 48 +++---- tensorflow/contrib/lite/toco/tooling_util.h | 17 +++ 8 files changed, 171 insertions(+), 32 deletions(-) create mode 100644 tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_reshape.cc diff --git a/tensorflow/contrib/lite/toco/BUILD b/tensorflow/contrib/lite/toco/BUILD index a05d71985f..4c8652d62e 100644 --- a/tensorflow/contrib/lite/toco/BUILD +++ b/tensorflow/contrib/lite/toco/BUILD @@ -266,6 +266,7 @@ cc_library( "graph_transformations/resolve_constant_gather.cc", "graph_transformations/resolve_constant_random_uniform.cc", "graph_transformations/resolve_constant_range.cc", + "graph_transformations/resolve_constant_reshape.cc", "graph_transformations/resolve_constant_shape_or_rank.cc", "graph_transformations/resolve_constant_stack.cc", "graph_transformations/resolve_constant_strided_slice.cc", diff --git a/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h b/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h index 80463ce8f8..384bd85b81 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h +++ b/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h @@ -165,6 +165,7 @@ DECLARE_GRAPH_TRANSFORMATION(ResolveTensorFlowSwitch) DECLARE_GRAPH_TRANSFORMATION(ResolveTensorFlowTile) DECLARE_GRAPH_TRANSFORMATION(ResolveConstantFakeQuant) DECLARE_GRAPH_TRANSFORMATION(ResolveConstantConcatenation) +DECLARE_GRAPH_TRANSFORMATION(ResolveConstantReshape) DECLARE_GRAPH_TRANSFORMATION(ResolveConstantTranspose) DECLARE_GRAPH_TRANSFORMATION(DropFakeQuant) DECLARE_GRAPH_TRANSFORMATION(UnfuseActivationFunctions) diff --git a/tensorflow/contrib/lite/toco/graph_transformations/remove_trivial_reshape.cc b/tensorflow/contrib/lite/toco/graph_transformations/remove_trivial_reshape.cc index 61477d59ae..e28d8cf01e 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/remove_trivial_reshape.cc +++ b/tensorflow/contrib/lite/toco/graph_transformations/remove_trivial_reshape.cc @@ -41,8 +41,8 @@ bool IsReshapeTrivial(const Model& model, const Operator& op, ShapesAgreeUpToExtending(input_array.shape(), output_array.shape())) { transformation->AddMessageF( "%s is trivial because its input and output shapes are equal up to " - "extending " - "by 1's, and we are told to aggressively discard such Reshape ops.", + "extending by 1's, and we are told to aggressively discard such " + "Reshape ops.", LogName(op)); return true; } @@ -80,6 +80,7 @@ bool RemoveTrivialReshape::Run(Model* model, std::size_t op_index) { } if (!IsReshapeTrivial(*model, *reshape_op, this)) { + AddMessageF("%s is not trivial", LogName(*reshape_op)); return false; } diff --git a/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_reshape.cc b/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_reshape.cc new file mode 100644 index 0000000000..7e7ad383e7 --- /dev/null +++ b/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_reshape.cc @@ -0,0 +1,124 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include + +#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h" +#include "tensorflow/contrib/lite/toco/model.h" +#include "tensorflow/contrib/lite/toco/tooling_util.h" +#include "tensorflow/core/platform/logging.h" + +namespace toco { + +// Resolves a constant reshape operation by copying the buffer. +bool ResolveConstantReshape::Run(Model* model, std::size_t op_index) { + auto it = model->operators.begin() + op_index; + const auto* base_op = it->get(); + if (base_op->type != OperatorType::kTensorFlowReshape) { + return false; + } + const auto* op = static_cast(base_op); + + CHECK_EQ(op->inputs.size(), 2); + CHECK_EQ(op->outputs.size(), 1); + + // We require constant inputs. + if (!IsConstantParameterArray(*model, op->inputs[0]) || + !IsConstantParameterArray(*model, op->inputs[1])) { + return false; + } + + auto& output_array = model->GetArray(op->outputs[0]); + if (output_array.data_type == ArrayDataType::kNone) { + // Yield until the output type has been set by PropagateArrayDataTypes. + return false; + } + if (!output_array.has_shape()) { + // Yield until the output shape has been set by PropagateFixedShapes. + return false; + } + + const Array& input_array = model->GetArray(op->inputs[0]); + if (!ShapesAgreeUpToExtending(input_array.shape(), output_array.shape())) { + AddMessageF("Constant reshape is non-trivial (%s -> %s)", + ShapeToString(input_array.shape()), + ShapeToString(output_array.shape())); + return false; + } + + CHECK(!output_array.buffer); + switch (input_array.data_type) { + case ArrayDataType::kBool: + CopyArrayBuffer(input_array, &output_array); + break; + case ArrayDataType::kFloat: + CopyArrayBuffer(input_array, &output_array); + break; + case ArrayDataType::kInt8: + CopyArrayBuffer(input_array, &output_array); + break; + case ArrayDataType::kUint8: + CopyArrayBuffer(input_array, &output_array); + break; + case ArrayDataType::kInt16: + CopyArrayBuffer(input_array, &output_array); + break; + case ArrayDataType::kUint16: + CopyArrayBuffer(input_array, &output_array); + break; + case ArrayDataType::kInt32: + CopyArrayBuffer(input_array, &output_array); + break; + case ArrayDataType::kUint32: + CopyArrayBuffer(input_array, &output_array); + break; + case ArrayDataType::kInt64: + CopyArrayBuffer(input_array, &output_array); + break; + case ArrayDataType::kUint64: + CopyArrayBuffer(input_array, &output_array); + break; + case ArrayDataType::kString: + CopyArrayBuffer(input_array, &output_array); + break; + default: + LOG(FATAL) << "Unsupported data type: " + << ArrayDataTypeName(input_array.data_type); + return false; + } + + AddMessageF("Resolving constant reshape of %s", LogName(*op)); + + if (input_array.minmax) { + output_array.GetOrCreateMinMax() = input_array.GetMinMax(); + } + if (input_array.quantization_params) { + output_array.GetOrCreateQuantizationParams() = + input_array.GetQuantizationParams(); + } + + // Erase input arrays if no longer used. + for (const auto& input : op->inputs) { + if (IsDiscardableArray(*model, input) && + CountOpsWithInput(*model, input) == 1) { + model->EraseArray(input); + } + } + + // Erase the operator. + model->operators.erase(it); + return true; +} + +} // namespace toco diff --git a/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_transpose.cc b/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_transpose.cc index 4f984bfde5..1fd20314b1 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_transpose.cc +++ b/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_transpose.cc @@ -131,6 +131,10 @@ bool ResolveConstantTranspose::Run(Model* model, std::size_t op_index) { if (input_array.minmax) { output_array.GetOrCreateMinMax() = input_array.GetMinMax(); } + if (input_array.quantization_params) { + output_array.GetOrCreateQuantizationParams() = + input_array.GetQuantizationParams(); + } if (op->perm.empty()) { // Yield until perm has been populated by ResolveTransposeAttributes. @@ -164,6 +168,8 @@ bool ResolveConstantTranspose::Run(Model* model, std::size_t op_index) { break; } + AddMessageF("Resolving constant transpose of %s", LogName(*op)); + // Erase input arrays if no longer used. for (const auto& input : op->inputs) { if (IsDiscardableArray(*model, input) && diff --git a/tensorflow/contrib/lite/toco/toco_tooling.cc b/tensorflow/contrib/lite/toco/toco_tooling.cc index 1ab0a6f058..5ba093a830 100644 --- a/tensorflow/contrib/lite/toco/toco_tooling.cc +++ b/tensorflow/contrib/lite/toco/toco_tooling.cc @@ -83,6 +83,7 @@ void MakeGeneralGraphTransformationsSet( transformations->Add(new ResolveConstantGather); transformations->Add(new ResolveConstantRandomUniform); transformations->Add(new ResolveConstantRange); + transformations->Add(new ResolveConstantReshape); transformations->Add(new ResolveConstantStack); transformations->Add(new ResolveConstantStridedSlice); transformations->Add(new ResolveConstantTranspose); diff --git a/tensorflow/contrib/lite/toco/tooling_util.cc b/tensorflow/contrib/lite/toco/tooling_util.cc index bd2d5f7df0..224df9973e 100644 --- a/tensorflow/contrib/lite/toco/tooling_util.cc +++ b/tensorflow/contrib/lite/toco/tooling_util.cc @@ -1084,23 +1084,30 @@ void InsertCopyOperator(Model* model, const string& source_array_name, model->operators.emplace_back(copy_op); } -namespace { -template -void CopyArrayBuffer(const Array& source_array, Array* target_array) { - if (source_array.buffer) { - const auto& source_buffer = source_array.GetBuffer(); - auto& target_buffer = target_array->GetMutableBuffer(); - target_buffer.data = source_buffer.data; - } -} -} // namespace - void CloneArray(Model* model, const string& source_array_name, const string& target_array_name) { CHECK(!model->HasArray(target_array_name)); const Array& source_array = model->GetArray(source_array_name); Array& target_array = model->GetOrCreateArray(target_array_name); + if (source_array.minmax) { + const auto& smm = source_array.GetMinMax(); + auto& tmm = target_array.GetOrCreateMinMax(); + tmm.min = smm.min; + tmm.max = smm.max; + } + + if (source_array.quantization_params) { + const auto& sqp = source_array.GetQuantizationParams(); + auto& tqp = target_array.GetOrCreateQuantizationParams(); + tqp.zero_point = sqp.zero_point; + tqp.scale = sqp.scale; + } + + target_array.data_type = source_array.data_type; + target_array.final_data_type = source_array.final_data_type; + target_array.copy_shape(source_array.shape()); + switch (source_array.data_type) { case ArrayDataType::kBool: CopyArrayBuffer(source_array, &target_array); @@ -1140,25 +1147,6 @@ void CloneArray(Model* model, const string& source_array_name, << ArrayDataTypeName(source_array.data_type); return; } - - if (source_array.minmax) { - const auto& smm = source_array.GetMinMax(); - auto& tmm = target_array.GetOrCreateMinMax(); - tmm.min = smm.min; - tmm.max = smm.max; - } - - if (source_array.quantization_params) { - const auto& sqp = source_array.GetQuantizationParams(); - auto& tqp = target_array.GetOrCreateQuantizationParams(); - tqp.zero_point = sqp.zero_point; - tqp.scale = sqp.scale; - } - - target_array.data_type = source_array.data_type; - target_array.final_data_type = source_array.final_data_type; - - target_array.copy_shape(source_array.shape()); } void MakeArrayDims(int num_dims, int batch, int height, int width, int depth, diff --git a/tensorflow/contrib/lite/toco/tooling_util.h b/tensorflow/contrib/lite/toco/tooling_util.h index dfd81173c3..ed0ecd4d0f 100644 --- a/tensorflow/contrib/lite/toco/tooling_util.h +++ b/tensorflow/contrib/lite/toco/tooling_util.h @@ -147,6 +147,23 @@ void FixNoOrphanedArray(Model* model); // Fixes input/output arrays that may have issues during export or inference. void FixEdgeArrays(Model* model); +// Copies the contents of an array into another. +// Expects that the shape and data type match. +template +void CopyArrayBuffer(const Array& source_array, Array* target_array) { + int source_buffer_size = RequiredBufferSizeForShape(source_array.shape()); + int target_buffer_size = RequiredBufferSizeForShape(target_array->shape()); + CHECK_EQ(source_buffer_size, target_buffer_size) + << "Buffer sizes must match in element count"; + CHECK(source_array.data_type == target_array->data_type) + << "Data types must match"; + if (source_array.buffer) { + const auto& source_buffer = source_array.GetBuffer(); + auto& target_buffer = target_array->GetMutableBuffer(); + target_buffer.data = source_buffer.data; + } +} + // Inserts a no-op reshape operator between the source array and the target // array. This effectively just copies the data. void InsertCopyOperator(Model* model, const string& source_array_name, -- GitLab From 0161bb77accc64d3742098feb7f438752a83ff32 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 14:33:16 -0700 Subject: [PATCH 278/791] K-FAC: Deprecate tf.contrib.kfac. As LayerCollections are required to instantiate KfacOptimizer and FisherEstimator, a deprecation warning is printed upon instantiating LayerCollection. PiperOrigin-RevId: 192671370 --- tensorflow/contrib/kfac/python/ops/layer_collection.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection.py b/tensorflow/contrib/kfac/python/ops/layer_collection.py index 411da033c3..366e2a82d5 100644 --- a/tensorflow/contrib/kfac/python/ops/layer_collection.py +++ b/tensorflow/contrib/kfac/python/ops/layer_collection.py @@ -28,6 +28,7 @@ from collections import defaultdict from collections import OrderedDict from contextlib import contextmanager from functools import partial +import warnings import math import six @@ -171,6 +172,9 @@ class LayerCollection(object): def __init__(self, graph=None, name="LayerCollection"): + warnings.warn( + "tf.contrib.kfac is deprecated and will be removed by 2018-11-01. " + "Use https://pypi.python.org/pypi/kfac instead.") self.fisher_blocks = LayerParametersDict() self.fisher_factors = OrderedDict() self._linked_parameters = dict( -- GitLab From 69edcec4746cc4260fd40079f1d72c2b23cdc297 Mon Sep 17 00:00:00 2001 From: Sami Kama Date: Thu, 12 Apr 2018 15:04:35 -0700 Subject: [PATCH 279/791] Merge libraries back --- tensorflow/contrib/tensorrt/BUILD | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD index 2ee0c4589c..8dc6e8fae6 100644 --- a/tensorflow/contrib/tensorrt/BUILD +++ b/tensorflow/contrib/tensorrt/BUILD @@ -193,28 +193,11 @@ tf_py_wrap_cc( ], ) -tf_cuda_library( - name = "trt_resource_manager_impl", - srcs = [ - "resources/trt_resource_manager.cc", - ], - hdrs = [ - "resources/trt_resource_manager.h", - ], - deps = [ - ":trt_logging", - "//tensorflow/core:framework_headers_lib", - "//tensorflow/core:framework_lite", - "//tensorflow/core:lib_proto_parsing", - ] + if_tensorrt([ - "@local_config_tensorrt//:nv_infer", - ]), -) - tf_cuda_library( name = "trt_resources", srcs = [ "resources/trt_int8_calibrator.cc", + "resources/trt_resource_manager.cc", ], hdrs = [ "resources/trt_int8_calibrator.h", @@ -228,8 +211,6 @@ tf_cuda_library( "//tensorflow/core:lib_proto_parsing", ] + if_tensorrt([ "@local_config_tensorrt//:nv_infer", - ]) + if_static([ - ":trt_resource_manager_impl", ]), ) @@ -248,7 +229,6 @@ tf_cuda_library( ":segment", ":trt_logging", ":trt_resources", - ":trt_resource_manager_impl", "//tensorflow/core/grappler:grappler_item", "//tensorflow/core/grappler:utils", "//tensorflow/core:framework", -- GitLab From 504a2ee3d82ac04a813b0bf18b0f972ce6bab2db Mon Sep 17 00:00:00 2001 From: Sami Kama Date: Thu, 12 Apr 2018 15:17:03 -0700 Subject: [PATCH 280/791] Remove if_static import --- tensorflow/contrib/tensorrt/BUILD | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD index 8dc6e8fae6..c792587733 100644 --- a/tensorflow/contrib/tensorrt/BUILD +++ b/tensorflow/contrib/tensorrt/BUILD @@ -27,10 +27,6 @@ load( "if_tensorrt", ) -load( - "//tensorflow/core:platform/default/build_config_root.bzl", - "if_static", -) tf_cuda_cc_test( name = "tensorrt_test_cc", -- GitLab From fffbe5a26da2d6fab5a3eb648cefef49db4d38de Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Thu, 12 Apr 2018 15:20:18 -0700 Subject: [PATCH 281/791] Check if the session has been deleted before releasing a callable. In some versions of Python, the Session._session field may be cleared (in `Session.__del__()`) before a callable that has a reference to that Session is deleted. Add a defensive check in the `Session._Callable.__del__()` method. PiperOrigin-RevId: 192679796 --- tensorflow/python/client/session.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/client/session.py b/tensorflow/python/client/session.py index 4c84d78f2e..5507d011bb 100644 --- a/tensorflow/python/client/session.py +++ b/tensorflow/python/client/session.py @@ -1454,7 +1454,10 @@ class BaseSession(SessionInterface): self._session._session, self._handle, args, status, None) def __del__(self): - if self._handle is not None: + # NOTE(mrry): It is possible that `self._session.__del__()` could be + # called before this destructor, in which case `self._session._session` + # will be `None`. + if self._handle is not None and self._session._session is not None: with errors.raise_exception_on_not_ok_status() as status: if self._session._created_with_new_api: tf_session.TF_SessionReleaseCallable( -- GitLab From d49cbc232ed5cd8c14159b7f3760348d10aa6206 Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Thu, 12 Apr 2018 15:20:34 -0700 Subject: [PATCH 282/791] [tf.data] Clean up //tensorflow/contrib/data/python/ops/BUILD. Create spearate targets for each submodule, so that each test can depend on the appropriate subset of Python files. PiperOrigin-RevId: 192679856 --- tensorflow/contrib/data/BUILD | 6 - tensorflow/contrib/data/__init__.py | 2 - .../contrib/data/python/kernel_tests/BUILD | 58 +++-- tensorflow/contrib/data/python/ops/BUILD | 214 +++++++++++++++--- tensorflow/contrib/distribute/python/BUILD | 2 +- tensorflow/contrib/eager/python/BUILD | 4 +- tensorflow/contrib/tpu/BUILD | 3 +- 7 files changed, 218 insertions(+), 71 deletions(-) diff --git a/tensorflow/contrib/data/BUILD b/tensorflow/contrib/data/BUILD index 7bb0dc1c0f..8bdbba83ef 100644 --- a/tensorflow/contrib/data/BUILD +++ b/tensorflow/contrib/data/BUILD @@ -22,13 +22,7 @@ py_library( deps = [ "//tensorflow/contrib/data/python/ops:dataset_ops", "//tensorflow/contrib/data/python/ops:iterator_ops", - "//tensorflow/contrib/data/python/ops:prefetching_ops", - "//tensorflow/contrib/data/python/ops:readers", - "//tensorflow/contrib/data/python/ops:shuffle_ops", - "//tensorflow/contrib/data/python/ops:transformation_ops", - "//tensorflow/python:parsing_ops", "//tensorflow/python:util", - "//tensorflow/python/data/ops:iterator_ops", ], ) diff --git a/tensorflow/contrib/data/__init__.py b/tensorflow/contrib/data/__init__.py index 17048314a4..f58e5ec1f0 100644 --- a/tensorflow/contrib/data/__init__.py +++ b/tensorflow/contrib/data/__init__.py @@ -78,8 +78,6 @@ from tensorflow.contrib.data.python.ops.resampling import rejection_resample from tensorflow.contrib.data.python.ops.scan_ops import scan from tensorflow.contrib.data.python.ops.shuffle_ops import shuffle_and_repeat from tensorflow.contrib.data.python.ops.sliding import sliding_window_batch -from tensorflow.python.data.ops.iterator_ops import Iterator -from tensorflow.python.ops.parsing_ops import parse_single_example_v2 as parse_single_example # pylint: enable=unused-import from tensorflow.python.util.all_util import remove_undocumented diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD index 5d6dbdcbdf..a8481dc90a 100644 --- a/tensorflow/contrib/data/python/kernel_tests/BUILD +++ b/tensorflow/contrib/data/python/kernel_tests/BUILD @@ -14,8 +14,7 @@ py_test( tags = ["no_pip"], deps = [ ":dataset_serialization_test", - "//tensorflow/contrib/data/python/ops:dataset_ops", - "//tensorflow/contrib/data/python/ops:transformation_ops", + "//tensorflow/contrib/data/python/ops:batching", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:constant_op", @@ -37,8 +36,7 @@ py_test( srcs_version = "PY2AND3", deps = [ ":dataset_serialization_test", - "//tensorflow/contrib/data/python/ops:dataset_ops", - "//tensorflow/contrib/data/python/ops:transformation_ops", + "//tensorflow/contrib/data/python/ops:grouping", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:constant_op", @@ -59,10 +57,10 @@ py_test( srcs_version = "PY2AND3", deps = [ ":dataset_serialization_test", - "//tensorflow/contrib/data/python/ops:dataset_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:errors", "//tensorflow/python:tensor_shape", + "//tensorflow/python/data/ops:dataset_ops", "//tensorflow/python/data/util:nest", "//third_party/py/numpy", ], @@ -79,8 +77,7 @@ py_test( ], deps = [ ":dataset_serialization_test", - "//tensorflow/contrib/data/python/ops:dataset_ops", - "//tensorflow/contrib/data/python/ops:transformation_ops", + "//tensorflow/contrib/data/python/ops:batching", "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -127,13 +124,13 @@ py_test( tags = ["no_pip"], deps = [ ":dataset_serialization_test", - "//tensorflow/contrib/data/python/ops:dataset_ops", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:dtypes", "//tensorflow/python:errors", "//tensorflow/python:functional_ops", "//tensorflow/python:math_ops", + "//tensorflow/python/data/ops:dataset_ops", "//third_party/py/numpy", ], ) @@ -145,7 +142,7 @@ tf_py_test( additional_deps = [ ":dataset_serialization_test", "//third_party/py/numpy", - "//tensorflow/contrib/data/python/ops:dataset_ops", + "//tensorflow/python/data/ops:dataset_ops", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:constant_op", @@ -175,8 +172,7 @@ py_test( ], deps = [ ":dataset_serialization_test", - "//tensorflow/contrib/data/python/ops:dataset_ops", - "//tensorflow/contrib/data/python/ops:transformation_ops", + "//tensorflow/contrib/data/python/ops:interleave_ops", "//tensorflow/python:array_ops", "//tensorflow/python:client", "//tensorflow/python:client_testlib", @@ -187,6 +183,7 @@ py_test( "//tensorflow/python:sparse_ops", "//tensorflow/python:sparse_tensor", "//tensorflow/python:training", + "//tensorflow/python/data/ops:dataset_ops", "//third_party/py/numpy", ], ) @@ -197,7 +194,8 @@ tf_py_test( srcs = ["get_single_element_test.py"], additional_deps = [ "//third_party/py/numpy", - "//tensorflow/contrib/data/python/ops:dataset_ops", + "//tensorflow/contrib/data/python/ops:get_single_element", + "//tensorflow/python/data/ops:dataset_ops", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:constant_op", @@ -215,8 +213,7 @@ py_test( tags = ["no_pip"], deps = [ ":dataset_serialization_test", - "//tensorflow/contrib/data/python/ops:dataset_ops", - "//tensorflow/contrib/data/python/ops:transformation_ops", + "//tensorflow/contrib/data/python/ops:error_ops", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:constant_op", @@ -261,8 +258,8 @@ py_test( srcs_version = "PY2AND3", deps = [ ":dataset_serialization_test", - "//tensorflow/contrib/data/python/ops:dataset_ops", - "//tensorflow/contrib/data/python/ops:transformation_ops", + "//tensorflow/contrib/data/python/ops:counter", + "//tensorflow/contrib/data/python/ops:enumerate_ops", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:constant_op", @@ -274,6 +271,7 @@ py_test( "//tensorflow/python:parsing_ops", "//tensorflow/python:tensor_shape", "//tensorflow/python:variables", + "//tensorflow/python/data/ops:dataset_ops", ], ) @@ -309,12 +307,12 @@ py_test( srcs_version = "PY2AND3", tags = ["noasan"], deps = [ - "//tensorflow/contrib/data/python/ops:dataset_ops", - "//tensorflow/contrib/data/python/ops:transformation_ops", + "//tensorflow/contrib/data/python/ops:resampling", "//tensorflow/python:client_testlib", "//tensorflow/python:errors", "//tensorflow/python:string_ops", "//tensorflow/python:util", + "//tensorflow/python/data/ops:dataset_ops", "//third_party/py/numpy", ], ) @@ -327,7 +325,7 @@ py_test( tags = ["no_pip"], deps = [ ":dataset_serialization_test", - "//tensorflow/contrib/data/python/ops:transformation_ops", + "//tensorflow/contrib/data/python/ops:scan_ops", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:constant_op", @@ -346,11 +344,11 @@ py_test( tags = ["no_pip"], deps = [ ":dataset_serialization_test", - "//tensorflow/contrib/data/python/ops:dataset_ops", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:dtypes", "//tensorflow/python:errors", + "//tensorflow/python/data/ops:dataset_ops", "//third_party/py/numpy", ], ) @@ -378,7 +376,6 @@ py_test( tags = ["no_pip"], deps = [ ":dataset_serialization_test", - "//tensorflow/contrib/data/python/ops:dataset_ops", "//tensorflow/contrib/data/python/ops:shuffle_ops", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -415,10 +412,10 @@ py_test( tags = ["no_pip"], deps = [ ":dataset_serialization_test", - "//tensorflow/contrib/data/python/ops:dataset_ops", - "//tensorflow/contrib/data/python/ops:transformation_ops", + "//tensorflow/contrib/data/python/ops:stats_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:errors", + "//tensorflow/python/data/ops:dataset_ops", ], ) @@ -429,10 +426,11 @@ py_test( srcs_version = "PY2AND3", tags = ["no_pip"], deps = [ - "//tensorflow/contrib/data/python/ops:dataset_ops", - "//tensorflow/contrib/data/python/ops:transformation_ops", + "//tensorflow/contrib/data/python/ops:threadpool", + "//tensorflow/contrib/data/python/ops:unique", "//tensorflow/python:client_testlib", "//tensorflow/python:errors", + "//tensorflow/python/data/ops:dataset_ops", ], ) @@ -444,13 +442,13 @@ py_test( tags = ["no_pip"], deps = [ ":dataset_serialization_test", - "//tensorflow/contrib/data/python/ops:dataset_ops", - "//tensorflow/contrib/data/python/ops:transformation_ops", + "//tensorflow/contrib/data/python/ops:unique", "//tensorflow/contrib/stateless", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:dtypes", "//tensorflow/python:errors", + "//tensorflow/python/data/ops:dataset_ops", "//third_party/py/numpy", ], ) @@ -463,11 +461,11 @@ py_test( tags = ["no_pip"], deps = [ ":dataset_serialization_test", - "//tensorflow/contrib/data/python/ops:dataset_ops", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:dtypes", "//tensorflow/python:errors", + "//tensorflow/python/data/ops:dataset_ops", "//third_party/py/numpy", ], ) @@ -497,8 +495,8 @@ tf_py_test( size = "small", srcs = ["slide_dataset_op_test.py"], additional_deps = [ - "//tensorflow/contrib/data/python/ops:dataset_ops", - "//tensorflow/contrib/data/python/ops:transformation_ops", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/contrib/data/python/ops:sliding", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:dtypes", diff --git a/tensorflow/contrib/data/python/ops/BUILD b/tensorflow/contrib/data/python/ops/BUILD index 236792bb98..7c28d1f005 100644 --- a/tensorflow/contrib/data/python/ops/BUILD +++ b/tensorflow/contrib/data/python/ops/BUILD @@ -12,18 +12,26 @@ load( load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") py_library( - name = "dataset_ops", - srcs = [ - "counter.py", - "get_single_element.py", + name = "counter", + srcs = ["counter.py"], + srcs_version = "PY2AND3", + deps = [ + ":scan_ops", + "//tensorflow/python:dtypes", + "//tensorflow/python:framework_ops", + "//tensorflow/python/data/ops:dataset_ops", ], +) + +py_library( + name = "get_single_element", + srcs = ["get_single_element.py"], srcs_version = "PY2AND3", deps = [ - ":transformation_ops", "//tensorflow/python:dataset_ops_gen", - "//tensorflow/python:util", "//tensorflow/python/data/ops:dataset_ops", "//tensorflow/python/data/util:nest", + "//tensorflow/python/data/util:sparse", ], ) @@ -66,7 +74,8 @@ py_library( ], srcs_version = "PY2AND3", deps = [ - ":dataset_ops", + ":batching", + ":interleave_ops", ":shuffle_ops", "//tensorflow/python:constant_op", "//tensorflow/python:dataset_ops_gen", @@ -94,50 +103,169 @@ py_library( ], srcs_version = "PY2AND3", deps = [ - ":random_ops", - ":transformation_ops", "//tensorflow/python/data/ops:dataset_ops", ], ) py_library( - name = "transformation_ops", - srcs = [ - "batching.py", - "enumerate_ops.py", - "error_ops.py", - "grouping.py", - "interleave_ops.py", - "resampling.py", - "scan_ops.py", - "sliding.py", - "stats_ops.py", - "threadpool.py", - "unique.py", + name = "batching", + srcs = ["batching.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:array_ops", + "//tensorflow/python:dataset_ops_gen", + "//tensorflow/python:dtypes", + "//tensorflow/python:framework_ops", + "//tensorflow/python:math_ops", + "//tensorflow/python:tensor_shape", + "//tensorflow/python:tensor_util", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/data/util:nest", + "//tensorflow/python/data/util:sparse", + ], +) + +py_library( + name = "enumerate_ops", + srcs = ["enumerate_ops.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:dtypes", + "//tensorflow/python/data/ops:dataset_ops", ], +) + +py_library( + name = "error_ops", + srcs = ["error_ops.py"], srcs_version = "PY2AND3", deps = [ ":contrib_op_loader", ":gen_dataset_ops", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/data/util:nest", + "//tensorflow/python/data/util:sparse", + ], +) + +py_library( + name = "grouping", + srcs = ["grouping.py"], + srcs_version = "PY2AND3", + deps = [ "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:dataset_ops_gen", + "//tensorflow/python:check_ops", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", "//tensorflow/python:function", + "//tensorflow/python:math_ops", + "//tensorflow/python:tensor_shape", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/data/util:nest", + "//tensorflow/python/data/util:sparse", + ], +) + +py_library( + name = "interleave_ops", + srcs = ["interleave_ops.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:util", + "//tensorflow/python/data/ops:readers", + ], +) + +py_library( + name = "resampling", + srcs = ["resampling.py"], + srcs_version = "PY2AND3", + deps = [ + ":batching", + ":scan_ops", + "//tensorflow/python:array_ops", + "//tensorflow/python:control_flow_ops", + "//tensorflow/python:dtypes", + "//tensorflow/python:framework_ops", "//tensorflow/python:logging_ops", "//tensorflow/python:math_ops", "//tensorflow/python:random_ops", + "//tensorflow/python/data/ops:dataset_ops", + ], +) + +py_library( + name = "scan_ops", + srcs = ["scan_ops.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:dataset_ops_gen", + "//tensorflow/python:framework_ops", + "//tensorflow/python:function", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/data/util:nest", + "//tensorflow/python/data/util:sparse", + ], +) + +py_library( + name = "sliding", + srcs = ["sliding.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:dataset_ops_gen", + "//tensorflow/python:dtypes", + "//tensorflow/python:framework_ops", + "//tensorflow/python:function", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/data/util:nest", + "//tensorflow/python/data/util:sparse", + ], +) + +py_library( + name = "stats_ops", + srcs = ["stats_ops.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:dataset_ops_gen", + "//tensorflow/python:dtypes", + "//tensorflow/python:framework_ops", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/data/ops:iterator_ops", + "//tensorflow/python/data/util:nest", + "//tensorflow/python/data/util:sparse", + ], +) + +py_library( + name = "threadpool", + srcs = ["threadpool.py"], + srcs_version = "PY2AND3", + deps = [ + ":contrib_op_loader", + ":gen_dataset_ops", "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:tensor_shape", - "//tensorflow/python:tensor_util", - "//tensorflow/python:util", "//tensorflow/python/data/ops:dataset_ops", - "//tensorflow/python/data/ops:readers", - "//tensorflow/python/data/util:convert", "//tensorflow/python/data/util:nest", "//tensorflow/python/data/util:sparse", - "//third_party/py/numpy", + "//tensorflow/python/eager:context", + ], +) + +py_library( + name = "unique", + srcs = [ + "unique.py", + ], + srcs_version = "PY2AND3", + deps = [ + ":contrib_op_loader", + ":gen_dataset_ops", + "//tensorflow/python:dtypes", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/data/util:nest", + "//tensorflow/python/data/util:sparse", ], ) @@ -183,3 +311,29 @@ py_library( "//tensorflow/python/data/util:sparse", ], ) + +py_library( + name = "dataset_ops", + deps = [ + ":batching", + ":counter", + ":enumerate_ops", + ":error_ops", + ":get_single_element", + ":grouping", + ":interleave_ops", + ":prefetching_ops", + ":readers", + ":resampling", + ":scan_ops", + ":shuffle_ops", + ":sliding", + ":stats_ops", + ":threadpool", + ":unique", + "//tensorflow/python:dataset_ops_gen", + "//tensorflow/python:util", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/data/util:nest", + ], +) diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD index 51b4fbacd1..5aad21cccd 100644 --- a/tensorflow/contrib/distribute/python/BUILD +++ b/tensorflow/contrib/distribute/python/BUILD @@ -22,7 +22,7 @@ py_library( visibility = ["//tensorflow:internal"], deps = [ ":prefetching_ops_v2", - "//tensorflow/contrib/data/python/ops:transformation_ops", + "//tensorflow/contrib/data/python/ops:batching", "//tensorflow/contrib/eager/python:datasets", "//tensorflow/python:array_ops", "//tensorflow/python:checkpointable", diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD index 04e2d99048..e2744a430d 100644 --- a/tensorflow/contrib/eager/python/BUILD +++ b/tensorflow/contrib/eager/python/BUILD @@ -71,7 +71,9 @@ cuda_py_test( additional_deps = [ ":datasets", ":checkpointable_utils", - "//tensorflow/contrib/data/python/ops:transformation_ops", + "//tensorflow/contrib/data/python/ops:prefetching_ops", + "//tensorflow/contrib/data/python/ops:threadpool", + "//tensorflow/contrib/data/python/ops:unique", "//tensorflow/contrib/lookup:lookup_py", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD index 3e489d38b6..9646d15486 100644 --- a/tensorflow/contrib/tpu/BUILD +++ b/tensorflow/contrib/tpu/BUILD @@ -198,7 +198,8 @@ py_library( ], srcs_version = "PY2AND3", deps = [ - "//tensorflow/contrib/data/python/ops:transformation_ops", + "//tensorflow/contrib/data/python/ops:batching", + "//tensorflow/contrib/data/python/ops:interleave_ops", "//tensorflow/python:dtypes", "//tensorflow/python:function", "//tensorflow/python:functional_ops", -- GitLab From f768aa0bb3d16edfdd1ac11733fac09c97c48f74 Mon Sep 17 00:00:00 2001 From: Sami Kama Date: Thu, 12 Apr 2018 15:33:09 -0700 Subject: [PATCH 283/791] Fix buildifier issues --- tensorflow/contrib/tensorrt/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD index c792587733..fd3582e175 100644 --- a/tensorflow/contrib/tensorrt/BUILD +++ b/tensorflow/contrib/tensorrt/BUILD @@ -27,7 +27,6 @@ load( "if_tensorrt", ) - tf_cuda_cc_test( name = "tensorrt_test_cc", size = "small", -- GitLab From 0195d6b4fbbe948914d0045d19eec8fcef1211f5 Mon Sep 17 00:00:00 2001 From: Benoit Steiner Date: Thu, 12 Apr 2018 15:41:41 -0700 Subject: [PATCH 284/791] Added a utility to compute a topo ordering of a graph PiperOrigin-RevId: 192683166 --- .../core/grappler/utils/topological_sort.cc | 35 ++++++++++++++----- .../core/grappler/utils/topological_sort.h | 4 +++ .../grappler/utils/topological_sort_test.cc | 24 +++++++++++-- 3 files changed, 52 insertions(+), 11 deletions(-) diff --git a/tensorflow/core/grappler/utils/topological_sort.cc b/tensorflow/core/grappler/utils/topological_sort.cc index 8d8ff4da3a..a8e464d09d 100644 --- a/tensorflow/core/grappler/utils/topological_sort.cc +++ b/tensorflow/core/grappler/utils/topological_sort.cc @@ -26,24 +26,24 @@ namespace grappler { // Kahn's algorithm is implemented. // For details, see https://en.wikipedia.org/wiki/Topological_sorting -Status TopologicalSort(GraphDef* graph) { +Status ComputeTopologicalOrder(const GraphDef& graph, + std::vector* ready_nodes) { SimpleGraphView graph_view; - TF_RETURN_IF_ERROR(graph_view.Initialize(*graph)); + TF_RETURN_IF_ERROR(graph_view.Initialize(graph)); - std::vector ready_nodes; - ready_nodes.reserve(graph_view.num_nodes()); + ready_nodes->reserve(graph_view.num_nodes()); int front = 0; int back = 0; std::vector num_ready_inputs(graph_view.num_nodes(), 0); for (int i = 0; i < graph_view.num_nodes(); i++) { if (graph_view.inputs(i).empty()) { - ready_nodes.push_back(i); + ready_nodes->push_back(i); back++; } - if (IsMerge(graph->node(i))) { + if (IsMerge(graph.node(i))) { for (int input : graph_view.inputs(i)) { - if (IsNextIteration(graph->node(input))) { + if (IsNextIteration(graph.node(input))) { num_ready_inputs[i]++; } } @@ -51,11 +51,11 @@ Status TopologicalSort(GraphDef* graph) { } while (front != back) { - int ready_node = ready_nodes[front]; + int ready_node = (*ready_nodes)[front]; for (int fanout : graph_view.outputs(ready_node)) { ++num_ready_inputs[fanout]; if (num_ready_inputs[fanout] == graph_view.inputs(fanout).size()) { - ready_nodes.push_back(fanout); + ready_nodes->push_back(fanout); ++back; } } @@ -66,7 +66,24 @@ Status TopologicalSort(GraphDef* graph) { return errors::InvalidArgument( "The graph couldn't be sorted in topological order."); } + return Status::OK(); +} +Status ComputeTopologicalOrder( + const GraphDef& graph, + std::unordered_map* topo_order) { + std::vector ready_nodes; + TF_RETURN_IF_ERROR(ComputeTopologicalOrder(graph, &ready_nodes)); + topo_order->reserve(graph.node_size()); + for (int i = 0; i < ready_nodes.size(); ++i) { + (*topo_order)[&graph.node(ready_nodes[i])] = i; + } + return Status::OK(); +} + +Status TopologicalSort(GraphDef* graph) { + std::vector ready_nodes; + TF_RETURN_IF_ERROR(ComputeTopologicalOrder(*graph, &ready_nodes)); PermuteNodesInPlace(graph, &ready_nodes, /*invert_permutation=*/true); return Status::OK(); } diff --git a/tensorflow/core/grappler/utils/topological_sort.h b/tensorflow/core/grappler/utils/topological_sort.h index 7700fe41e4..668c88dc75 100644 --- a/tensorflow/core/grappler/utils/topological_sort.h +++ b/tensorflow/core/grappler/utils/topological_sort.h @@ -22,6 +22,10 @@ limitations under the License. namespace tensorflow { namespace grappler { +// Compute a topological ordering for the graph nodes. +Status ComputeTopologicalOrder( + const GraphDef& graph, std::unordered_map* topo_order); + // Sort a graph in topological order. Status TopologicalSort(GraphDef* graph); diff --git a/tensorflow/core/grappler/utils/topological_sort_test.cc b/tensorflow/core/grappler/utils/topological_sort_test.cc index c96f15b0e8..f5c95009d2 100644 --- a/tensorflow/core/grappler/utils/topological_sort_test.cc +++ b/tensorflow/core/grappler/utils/topological_sort_test.cc @@ -52,8 +52,19 @@ TEST_F(TopologicalSortTest, NoLoop) { *graph.add_node() = CreateNode("5", {}); *graph.add_node() = CreateNode("4", {}); + std::unordered_map topo_order; + TF_EXPECT_OK(ComputeTopologicalOrder(graph, &topo_order)); + + const std::vector order = {"5", "4", "2", "0", "3", "1"}; + for (const auto& topo : topo_order) { + const string& node_name = topo.first->name(); + const int topo_order = topo.second; + std::cout << "Node " << node_name << " at order " << topo_order + << std::endl; + EXPECT_EQ(node_name, order[topo_order]); + } + TF_EXPECT_OK(TopologicalSort(&graph)); - std::vector order = {"5", "4", "2", "0", "3", "1"}; for (int i = 0; i < order.size(); i++) { EXPECT_EQ(graph.node(i).name(), order[i]); } @@ -68,8 +79,17 @@ TEST_F(TopologicalSortTest, WithLoop) { *graph.add_node() = CreateNode("5", "NextIteration", {"4"}); *graph.add_node() = CreateNode("1", {}); + std::unordered_map topo_order; + TF_EXPECT_OK(ComputeTopologicalOrder(graph, &topo_order)); + + const std::vector order = {"1", "2", "3", "4", "5"}; + for (const auto& topo : topo_order) { + const string& node_name = topo.first->name(); + const int topo_order = topo.second; + EXPECT_EQ(node_name, order[topo_order]); + } + TF_EXPECT_OK(TopologicalSort(&graph)); - std::vector order = {"1", "2", "3", "4", "5"}; for (int i = 0; i < order.size(); i++) { EXPECT_EQ(graph.node(i).name(), order[i]); } -- GitLab From cc108a73af35b407bf9bf51e679e5884b309964b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 16:26:13 -0700 Subject: [PATCH 285/791] Add support for RNN state array of type tf.identity. PiperOrigin-RevId: 192689747 --- .../lite/toco/graph_transformations/remove_unused_op.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/lite/toco/graph_transformations/remove_unused_op.cc b/tensorflow/contrib/lite/toco/graph_transformations/remove_unused_op.cc index aa2c293382..8e6aaf544a 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/remove_unused_op.cc +++ b/tensorflow/contrib/lite/toco/graph_transformations/remove_unused_op.cc @@ -47,7 +47,8 @@ bool RemoveUnusedOp::Run(Model* model, std::size_t op_index) { bool found_output_as_rnn_state_array = false; for (const auto& rnn_state : model->flags.rnn_states()) { if (output == rnn_state.state_array()) { - CHECK(op->type == OperatorType::kFill); + CHECK(op->type == OperatorType::kFill || + op->type == OperatorType::kTensorFlowIdentity); found_output_as_rnn_state_array = true; break; } -- GitLab From dde6aaf321d7f73fb31578fb044b783fb449d017 Mon Sep 17 00:00:00 2001 From: Jiri Simsa Date: Thu, 12 Apr 2018 16:35:47 -0700 Subject: [PATCH 286/791] Exposing tensorflow.contrib.proto in the pip package. PiperOrigin-RevId: 192691078 --- tensorflow/contrib/BUILD | 1 + tensorflow/contrib/__init__.py | 1 + tensorflow/contrib/cmake/tf_python.cmake | 6 ++++-- .../python/kernel_tests/decode_proto_fail_test.py | 4 ++-- .../python/kernel_tests/decode_proto_op_test.py | 4 ++-- .../python/kernel_tests/encode_proto_op_test.py | 15 ++++++++------- 6 files changed, 18 insertions(+), 13 deletions(-) diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD index 9bef0d8b61..ae68f4aec4 100644 --- a/tensorflow/contrib/BUILD +++ b/tensorflow/contrib/BUILD @@ -77,6 +77,7 @@ py_library( "//tensorflow/contrib/optimizer_v2:optimizer_v2_py", "//tensorflow/contrib/periodic_resample:init_py", "//tensorflow/contrib/predictor", + "//tensorflow/contrib/proto", "//tensorflow/contrib/quantization:quantization_py", "//tensorflow/contrib/quantize:quantize_graph", "//tensorflow/contrib/autograph", diff --git a/tensorflow/contrib/__init__.py b/tensorflow/contrib/__init__.py index aaddb06fa0..e27ece8fa5 100644 --- a/tensorflow/contrib/__init__.py +++ b/tensorflow/contrib/__init__.py @@ -64,6 +64,7 @@ from tensorflow.contrib import nn from tensorflow.contrib import opt from tensorflow.contrib import periodic_resample from tensorflow.contrib import predictor +from tensorflow.contrib import proto from tensorflow.contrib import quantization from tensorflow.contrib import quantize from tensorflow.contrib import recurrent diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake index ded15b4b66..21f59d2563 100755 --- a/tensorflow/contrib/cmake/tf_python.cmake +++ b/tensorflow/contrib/cmake/tf_python.cmake @@ -330,8 +330,10 @@ GENERATE_PYTHON_OP_LIB("ctc_ops") GENERATE_PYTHON_OP_LIB("cudnn_rnn_ops") GENERATE_PYTHON_OP_LIB("data_flow_ops") GENERATE_PYTHON_OP_LIB("dataset_ops") -GENERATE_PYTHON_OP_LIB("decode_proto_ops") -GENERATE_PYTHON_OP_LIB("encode_proto_ops") +GENERATE_PYTHON_OP_LIB("decode_proto_ops" + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/proto/python/ops/gen_decode_proto_op.py) +GENERATE_PYTHON_OP_LIB("encode_proto_ops" + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/proto/python/ops/gen_encode_proto_op.py) GENERATE_PYTHON_OP_LIB("image_ops") GENERATE_PYTHON_OP_LIB("io_ops") GENERATE_PYTHON_OP_LIB("linalg_ops") diff --git a/tensorflow/contrib/proto/python/kernel_tests/decode_proto_fail_test.py b/tensorflow/contrib/proto/python/kernel_tests/decode_proto_fail_test.py index f019833905..f8969b0bd5 100644 --- a/tensorflow/contrib/proto/python/kernel_tests/decode_proto_fail_test.py +++ b/tensorflow/contrib/proto/python/kernel_tests/decode_proto_fail_test.py @@ -21,7 +21,7 @@ from __future__ import print_function import numpy as np -from tensorflow.contrib import proto +from tensorflow.contrib.proto import decode_proto from tensorflow.contrib.proto.python.kernel_tests import test_case from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors @@ -46,7 +46,7 @@ class DecodeProtoFailTest(test_case.ProtoOpTestCase): field_types = [dtypes.int32] with self.test_session() as sess: - ctensor, vtensor = proto.decode_proto( + ctensor, vtensor = decode_proto( batch, message_type=msg_type, field_names=field_names, diff --git a/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test.py b/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test.py index 30ceac5f5f..cd5121cdba 100644 --- a/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test.py +++ b/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test.py @@ -27,7 +27,7 @@ import numpy as np from google.protobuf import text_format -from tensorflow.contrib import proto +from tensorflow.contrib.proto import decode_proto from tensorflow.contrib.proto.python.kernel_tests import test_case from tensorflow.contrib.proto.python.kernel_tests import test_example_pb2 from tensorflow.python.framework import dtypes @@ -175,7 +175,7 @@ class DecodeProtoOpTest(test_case.ProtoOpTestCase): output_types = [f.dtype for f in fields] with self.test_session() as sess: - sizes, vtensor = proto.decode_proto( + sizes, vtensor = decode_proto( batch, message_type=message_type, field_names=field_names, diff --git a/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test.py b/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test.py index 2a24c3b8ce..a289ff290a 100644 --- a/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test.py +++ b/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test.py @@ -30,7 +30,8 @@ import numpy as np from google.protobuf import text_format -from tensorflow.contrib import proto +from tensorflow.contrib.proto import decode_proto +from tensorflow.contrib.proto import encode_proto from tensorflow.contrib.proto.python.kernel_tests import test_case from tensorflow.contrib.proto.python.kernel_tests import test_example_pb2 from tensorflow.python.framework import dtypes @@ -50,7 +51,7 @@ class EncodeProtoOpTest(test_case.ProtoOpTestCase): # Invalid field name with self.test_session(): with self.assertRaisesOpError('Unknown field: non_existent_field'): - proto.encode_proto( + encode_proto( sizes=[[1]], values=[np.array([[0.0]], dtype=np.int32)], message_type='tensorflow.contrib.proto.RepeatedPrimitiveValue', @@ -60,7 +61,7 @@ class EncodeProtoOpTest(test_case.ProtoOpTestCase): with self.test_session(): with self.assertRaisesOpError( 'Incompatible type for field double_value.'): - proto.encode_proto( + encode_proto( sizes=[[1]], values=[np.array([[0.0]], dtype=np.int32)], message_type='tensorflow.contrib.proto.RepeatedPrimitiveValue', @@ -72,7 +73,7 @@ class EncodeProtoOpTest(test_case.ProtoOpTestCase): r'sizes should be batch_size \+ \[len\(field_names\)\]'): sizes = array_ops.placeholder(dtypes.int32) values = array_ops.placeholder(dtypes.float64) - proto.encode_proto( + encode_proto( sizes=sizes, values=[values], message_type='tensorflow.contrib.proto.RepeatedPrimitiveValue', @@ -88,7 +89,7 @@ class EncodeProtoOpTest(test_case.ProtoOpTestCase): sizes = array_ops.placeholder(dtypes.int32) values1 = array_ops.placeholder(dtypes.float64) values2 = array_ops.placeholder(dtypes.int32) - (proto.encode_proto( + (encode_proto( sizes=[[1, 1]], values=[values1, values2], message_type='tensorflow.contrib.proto.RepeatedPrimitiveValue', @@ -103,13 +104,13 @@ class EncodeProtoOpTest(test_case.ProtoOpTestCase): out_types = [f.dtype for f in fields] with self.test_session() as sess: - sizes, field_tensors = proto.decode_proto( + sizes, field_tensors = decode_proto( in_bufs, message_type=message_type, field_names=field_names, output_types=out_types) - out_tensors = proto.encode_proto( + out_tensors = encode_proto( sizes, field_tensors, message_type=message_type, -- GitLab From 9908cb16746a2c1a5b4c28950debc0b5964447ad Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 16:51:43 -0700 Subject: [PATCH 287/791] Change assertions to use the tensor 'x' rather than 'x.op.name'. This enables eager execution in validate_args=True contexts. PiperOrigin-RevId: 192693458 --- .../python/ops/bijectors/reshape.py | 14 +++++++------- tensorflow/python/ops/distributions/util.py | 17 ++++++++--------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/reshape.py b/tensorflow/contrib/distributions/python/ops/bijectors/reshape.py index 82210cd6c9..f21b982ba6 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/reshape.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/reshape.py @@ -138,7 +138,7 @@ class Reshape(bijector_lib.Bijector): """Check that a shape Tensor is int-type and otherwise sane.""" if not shape.dtype.is_integer: raise TypeError("{} dtype ({}) should be `int`-like.".format( - shape.op.name, shape.dtype.name)) + shape, shape.dtype.name)) assertions = [] @@ -146,10 +146,10 @@ class Reshape(bijector_lib.Bijector): ndims_ = tensor_util.constant_value(ndims) if ndims_ is not None and ndims_ > 1: raise ValueError("`{}` rank ({}) should be <= 1.".format( - shape.op.name, ndims_)) + shape, ndims_)) elif validate_args: assertions.append(check_ops.assert_less_equal( - ndims, 1, message="`{}` rank should be <= 1.".format(shape.op.name))) + ndims, 1, message="`{}` rank should be <= 1.".format(shape))) shape_ = tensor_util.constant_value_as_shape(shape) if shape_.is_fully_defined(): @@ -157,12 +157,12 @@ class Reshape(bijector_lib.Bijector): if sum(es == -1) > 1: raise ValueError( "`{}` must have at most one `-1` (given {})" - .format(shape.op.name, es)) + .format(shape, es)) if np.any(es < -1): raise ValueError( "`{}` elements must be either positive integers or `-1`" "(given {})." - .format(shape.op.name, es)) + .format(shape, es)) elif validate_args: assertions.extend([ check_ops.assert_less_equal( @@ -170,11 +170,11 @@ class Reshape(bijector_lib.Bijector): math_ops.cast(math_ops.equal(shape, -1), dtypes.int32)), 1, message="`{}` elements must have at most one `-1`." - .format(shape.op.name)), + .format(shape)), check_ops.assert_greater_equal( shape, -1, message="`{}` elements must be either positive integers or `-1`." - .format(shape.op.name)), + .format(shape)), ]) return assertions diff --git a/tensorflow/python/ops/distributions/util.py b/tensorflow/python/ops/distributions/util.py index 0fe6aa30f9..2e067eab45 100644 --- a/tensorflow/python/ops/distributions/util.py +++ b/tensorflow/python/ops/distributions/util.py @@ -58,8 +58,7 @@ def assert_close( if data is None: data = [ message, - "Condition x ~= y did not hold element-wise: x = ", x.name, x, "y = ", - y.name, y + "Condition x ~= y did not hold element-wise: x = ", x, "y = ", y ] if x.dtype.is_integer: @@ -95,7 +94,7 @@ def assert_integer_form( x = ops.convert_to_tensor(x, name="x") if x.dtype.is_integer: return control_flow_ops.no_op() - message = message or "{} has non-integer components".format(x.op.name) + message = message or "{} has non-integer components".format(x) if int_dtype is None: try: int_dtype = { @@ -123,13 +122,13 @@ def embed_check_nonnegative_integer_form( x = ops.convert_to_tensor(x, name="x") assertions = [ check_ops.assert_non_negative( - x, message="'{}' must be non-negative.".format(x.op.name)), + x, message="'{}' must be non-negative.".format(x)), ] if not x.dtype.is_integer: assertions += [ assert_integer_form( x, message="'{}' cannot contain fractional components.".format( - x.op.name)), + x)), ] return control_flow_ops.with_dependencies(assertions, x) @@ -434,7 +433,7 @@ def embed_check_integer_casting_closed( and not _is_integer_like_by_dtype(target_dtype)): raise TypeError("At least one of {}.dtype ({}) and target_dtype ({}) " "must be integer-type.".format( - x.op.name, x.dtype.name, target_dtype.name)) + x, x.dtype.name, target_dtype.name)) assertions = [] if assert_nonnegative: @@ -683,7 +682,7 @@ def pick_vector(cond, cond = ops.convert_to_tensor(cond, name="cond") if cond.dtype != dtypes.bool: raise TypeError("%s.dtype=%s which is not %s" % - (cond.name, cond.dtype, dtypes.bool)) + (cond, cond.dtype, dtypes.bool)) cond_value_static = tensor_util.constant_value(cond) if cond_value_static is not None: return true_vector if cond_value_static else false_vector @@ -692,8 +691,8 @@ def pick_vector(cond, if true_vector.dtype != false_vector.dtype: raise TypeError( "%s.dtype=%s does not match %s.dtype=%s" - % (true_vector.name, true_vector.dtype, - false_vector.name, false_vector.dtype)) + % (true_vector, true_vector.dtype, + false_vector, false_vector.dtype)) n = array_ops.shape(true_vector)[0] return array_ops.slice( array_ops.concat([true_vector, false_vector], 0), -- GitLab From 5d442bea19fd8f7f945248fb55f1ca2a6f6205c5 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 16:55:30 -0700 Subject: [PATCH 288/791] Propagate sharding of the source instruction to the copies added by layout assignment. PiperOrigin-RevId: 192693972 --- .../compiler/xla/service/hlo_instruction.cc | 15 +- .../compiler/xla/service/hlo_instruction.h | 7 + .../compiler/xla/service/hlo_sharding.cc | 10 ++ .../compiler/xla/service/hlo_sharding.h | 4 + .../compiler/xla/service/layout_assignment.cc | 163 ++++++++++-------- .../compiler/xla/service/layout_assignment.h | 23 +++ 6 files changed, 148 insertions(+), 74 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index 5d2d7a9727..56cb241087 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -838,6 +838,16 @@ static string FusionNodeName(HloInstruction::FusionKind fusion_kind) { return instruction; } +void HloInstruction::SetupDerivedInstruction( + HloInstruction* derived_instruction) const { + if (sharding_ != nullptr) { + derived_instruction->set_sharding(*sharding_); + } else { + derived_instruction->clear_sharding(); + } + derived_instruction->set_metadata(metadata_); +} + HloInstruction* HloInstruction::AddFusionOperand(HloInstruction* new_operand) { CHECK_EQ(opcode(), HloOpcode::kFusion); CHECK_EQ(operand_count(), @@ -1480,10 +1490,7 @@ std::unique_ptr HloInstruction::CloneWithNewOperands( case HloOpcode::kTrace: LOG(FATAL) << "Not yet implemented, clone: " << HloOpcodeString(opcode_); } - clone->set_metadata(metadata_); - if (has_sharding()) { - clone->set_sharding(sharding()); - } + SetupDerivedInstruction(clone.get()); clone->set_parent(parent_); return clone; } diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h index 9a9de07883..49aa075029 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.h +++ b/tensorflow/compiler/xla/service/hlo_instruction.h @@ -949,6 +949,13 @@ class HloInstruction { // Return true if this operator has a sharding assigned. bool has_sharding() const { return sharding_ != nullptr; } + // When creating a new instruction which either replaces, or shifts up (kCopy + // insertion case), another instruction, we need to make sure the certain + // properties of the new instruction are copied into the derived one. As of + // today, the metadata and sharding will be propagated to the derived + // instruction. + void SetupDerivedInstruction(HloInstruction* derived_instruction) const; + // Adds a new operand the fusion instruction. HloInstruction* AddFusionOperand(HloInstruction* new_operand); diff --git a/tensorflow/compiler/xla/service/hlo_sharding.cc b/tensorflow/compiler/xla/service/hlo_sharding.cc index e8e45f1ee9..1b42349b0b 100644 --- a/tensorflow/compiler/xla/service/hlo_sharding.cc +++ b/tensorflow/compiler/xla/service/hlo_sharding.cc @@ -376,6 +376,16 @@ HloSharding HloSharding::TransformShardedTileShape( return HloSharding::Tile(new_tile_shape, tile_assignment()); } +HloSharding HloSharding::GetSubSharding(const Shape& shape, + const ShapeIndex& index) const { + CHECK(IsTuple()); + + ShapeTree sub_shape_tree(ShapeUtil::GetSubshape(shape, index), + Replicate()); + sub_shape_tree.CopySubtreeFrom(GetAsShapeTree(shape), index, {}); + return Tuple(sub_shape_tree); +} + std::ostream& operator<<(std::ostream& out, const HloSharding& sharding) { out << sharding.ToString(); return out; diff --git a/tensorflow/compiler/xla/service/hlo_sharding.h b/tensorflow/compiler/xla/service/hlo_sharding.h index 06204acbca..2b8e757f42 100644 --- a/tensorflow/compiler/xla/service/hlo_sharding.h +++ b/tensorflow/compiler/xla/service/hlo_sharding.h @@ -175,6 +175,10 @@ class HloSharding { } } + // Retrieves the sub sharding at a given index, out of a tuple sharding. + // REQUIRES: IsTuple() + HloSharding GetSubSharding(const Shape& shape, const ShapeIndex& index) const; + bool operator==(const HloSharding& other) const { return replicated_ == other.replicated_ && maximal_ == other.maximal_ && ShapeUtil::Compatible(tile_shape_, other.tile_shape_) && diff --git a/tensorflow/compiler/xla/service/layout_assignment.cc b/tensorflow/compiler/xla/service/layout_assignment.cc index 39f9120e55..2494569db5 100644 --- a/tensorflow/compiler/xla/service/layout_assignment.cc +++ b/tensorflow/compiler/xla/service/layout_assignment.cc @@ -57,76 +57,6 @@ namespace xla { // anonymous namespace, instead of three or four spread all over this file. namespace { -// Creates and returns a copy of the given instruction with a different -// layout. Tuple-shaped instructions will be deep-copied, and the last Tuple -// instruction producing the copy is returned. -StatusOr CreateCopyWithNewLayout( - const Shape& shape_with_layout, HloInstruction* instruction) { - TF_RET_CHECK(LayoutUtil::HasLayout(shape_with_layout)); - DCHECK(ShapeUtil::Compatible(shape_with_layout, instruction->shape())) - << ShapeUtil::HumanString(shape_with_layout) << " " - << ShapeUtil::HumanString(instruction->shape()) - << " instruction: " << instruction->ToString(); - - if (ShapeUtil::IsTuple(instruction->shape())) { - // Deep-copy tuples. - std::vector element_copies; - for (int64 i = 0; i < ShapeUtil::TupleElementCount(instruction->shape()); - ++i) { - HloInstruction* gte = instruction->parent()->AddInstruction( - HloInstruction::CreateGetTupleElement( - ShapeUtil::GetSubshape(instruction->shape(), {i}), instruction, - i)); - - // Recurse to copy each elements. - TF_ASSIGN_OR_RETURN( - HloInstruction * element_copy, - CreateCopyWithNewLayout( - ShapeUtil::GetSubshape(shape_with_layout, {i}), gte)); - element_copies.push_back(element_copy); - } - // Gather element copies into a tuple with a new Tuple instruction. - HloInstruction* tuple_copy = instruction->parent()->AddInstruction( - HloInstruction::CreateTuple(element_copies)); - LayoutUtil::ClearLayout(tuple_copy->mutable_shape()); - TF_RETURN_IF_ERROR(LayoutUtil::CopyLayoutBetweenShapes( - shape_with_layout, tuple_copy->mutable_shape())); - return tuple_copy; - } else if (ShapeUtil::IsArray(instruction->shape())) { - HloInstruction* copy = - instruction->parent()->AddInstruction(HloInstruction::CreateUnary( - instruction->shape(), HloOpcode::kCopy, instruction)); - LayoutUtil::ClearLayout(copy->mutable_shape()); - TF_RETURN_IF_ERROR(LayoutUtil::CopyLayoutBetweenShapes( - shape_with_layout, copy->mutable_shape())); - - return copy; - } else { - return FailedPrecondition( - "Can only copy array and tuple shaped instructions"); - } -} - -// Creates a copy of the given operand if the operand's layout does not match -// the given layout. This copy replaces the use in the given instruction. Tuple -// operands will be deep-copied. -Status CopyOperandIfLayoutsDiffer(const ShapeLayout& operand_layout, - HloInstruction* instruction, - int64 operand_no) { - HloInstruction* operand = instruction->mutable_operand(operand_no); - TF_RET_CHECK(operand_layout.LayoutIsSet()); - TF_RET_CHECK(LayoutUtil::HasLayout(operand->shape())); - - if (ShapeUtil::Equal(operand_layout.shape(), operand->shape())) { - // Operand layout already matches our constraint. Nothing to do. - return Status::OK(); - } - - TF_ASSIGN_OR_RETURN(HloInstruction * operand_copy, - CreateCopyWithNewLayout(operand_layout.shape(), operand)); - - return instruction->ReplaceOperandWith(operand_no, operand_copy); -} } // namespace @@ -793,6 +723,99 @@ Status CheckConstantLayout(HloInstruction* constant) { } // namespace +StatusOr LayoutAssignment::CreateCopyWithNewLayout( + const Shape& shape_with_layout, HloInstruction* instruction) { + TF_RET_CHECK(LayoutUtil::HasLayout(shape_with_layout)); + DCHECK(ShapeUtil::Compatible(shape_with_layout, instruction->shape())) + << ShapeUtil::HumanString(shape_with_layout) << " " + << ShapeUtil::HumanString(instruction->shape()) + << " instruction: " << instruction->ToString(); + + if (ShapeUtil::IsTuple(instruction->shape())) { + // Deep-copy tuples. + std::vector element_copies; + for (int64 i = 0; i < ShapeUtil::TupleElementCount(instruction->shape()); + ++i) { + HloInstruction* gte = instruction->parent()->AddInstruction( + HloInstruction::CreateGetTupleElement( + ShapeUtil::GetSubshape(instruction->shape(), {i}), instruction, + i)); + SetupCopiedInstruction(*instruction, gte, {i}); + // Recurse to copy each elements. + TF_ASSIGN_OR_RETURN( + HloInstruction * element_copy, + CreateCopyWithNewLayout( + ShapeUtil::GetSubshape(shape_with_layout, {i}), gte)); + element_copies.push_back(element_copy); + } + // Gather element copies into a tuple with a new Tuple instruction. + HloInstruction* tuple_copy = instruction->parent()->AddInstruction( + HloInstruction::CreateTuple(element_copies)); + SetupCopiedInstruction(*instruction, tuple_copy, {}); + LayoutUtil::ClearLayout(tuple_copy->mutable_shape()); + TF_RETURN_IF_ERROR(LayoutUtil::CopyLayoutBetweenShapes( + shape_with_layout, tuple_copy->mutable_shape())); + return tuple_copy; + } else if (ShapeUtil::IsArray(instruction->shape())) { + HloInstruction* copy = + instruction->parent()->AddInstruction(HloInstruction::CreateUnary( + instruction->shape(), HloOpcode::kCopy, instruction)); + SetupCopiedInstruction(*instruction, copy, {}); + LayoutUtil::ClearLayout(copy->mutable_shape()); + TF_RETURN_IF_ERROR(LayoutUtil::CopyLayoutBetweenShapes( + shape_with_layout, copy->mutable_shape())); + + return copy; + } else { + return FailedPrecondition( + "Can only copy array and tuple shaped instructions"); + } +} + +// Creates a copy of the given operand if the operand's layout does not match +// the given layout. This copy replaces the use in the given instruction. Tuple +// operands will be deep-copied. +Status LayoutAssignment::CopyOperandIfLayoutsDiffer( + const ShapeLayout& operand_layout, HloInstruction* instruction, + int64 operand_no) { + HloInstruction* operand = instruction->mutable_operand(operand_no); + TF_RET_CHECK(operand_layout.LayoutIsSet()); + TF_RET_CHECK(LayoutUtil::HasLayout(operand->shape())); + + if (ShapeUtil::Equal(operand_layout.shape(), operand->shape())) { + // Operand layout already matches our constraint. Nothing to do. + return Status::OK(); + } + + TF_ASSIGN_OR_RETURN(HloInstruction * operand_copy, + CreateCopyWithNewLayout(operand_layout.shape(), operand)); + + return instruction->ReplaceOperandWith(operand_no, operand_copy); +} + +void LayoutAssignment::SetupCopiedInstruction(const HloInstruction& instruction, + HloInstruction* copy, + const ShapeIndex& index) { + if (instruction.has_sharding()) { + // If the index is empty, we want to copy the whole sharding, in case the + // sharding is a tuple sharding. + HloSharding sharding = + !index.empty() && instruction.sharding().IsTuple() + ? instruction.sharding().GetSubSharding(instruction.shape(), index) + : instruction.sharding(); + // We propagate the sharding to the copied instruction only if it is a + // special sharding, like tiled ones, or special devices like the + // HostCompute module. + // Otherwise it is preferable to leave the new instruction without device, + // and let the automatic device placer to choose the best location. + if (!sharding.HasUniqueDevice() || + HloSharding::IsReservedDevice(sharding.UniqueDevice().ValueOrDie())) { + copy->set_sharding(sharding); + } + } + copy->set_metadata(instruction.metadata()); +} + Status LayoutAssignment::CheckLayouts(HloModule* module) { TF_ASSIGN_OR_RETURN(auto points_to_analysis, TuplePointsToAnalysis::Run(module)); diff --git a/tensorflow/compiler/xla/service/layout_assignment.h b/tensorflow/compiler/xla/service/layout_assignment.h index 680f88048a..ae4986d6ad 100644 --- a/tensorflow/compiler/xla/service/layout_assignment.h +++ b/tensorflow/compiler/xla/service/layout_assignment.h @@ -405,6 +405,29 @@ class LayoutAssignment : public HloPassInterface { ComputationLayout* entry_computation_layout_; protected: + // Sets up the copy instruction according to the characteristic (sharding, + // metadata, ...) of the reference instruction. The index argument is used + // when the instruction is a tuple, and in such case the index represents + // the location from where the copy instruction was created from. + // If the index is empty, the whole sharding will be propagated, even in case + // the intruction has a tuple sharding. + static void SetupCopiedInstruction(const HloInstruction& instruction, + HloInstruction* copy, + const ShapeIndex& index); + + // Creates and returns a copy of the given instruction with a different + // layout. Tuple-shaped instructions will be deep-copied, and the last Tuple + // instruction producing the copy is returned. + static StatusOr CreateCopyWithNewLayout( + const Shape& shape_with_layout, HloInstruction* instruction); + + // Creates a copy of the given operand if the operand's layout does not match + // the given layout. This copy replaces the use in the given instruction. + // Tuple operands will be deep-copied. + static Status CopyOperandIfLayoutsDiffer(const ShapeLayout& operand_layout, + HloInstruction* instruction, + int64 operand_no); + // Map containing the layouts of all computations assigned so // far. Computations are handled in a topological sort where computations are // handled before their caller instructions so the layouts of caller -- GitLab From 5f7929b8c340b579f859396677c17f010f94d984 Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Thu, 12 Apr 2018 16:56:45 -0700 Subject: [PATCH 289/791] [XLA:GPU] Pass all four args to custom-call convs when they're created. A custom-call-conv should have four arguments: lhs, rhs, algorithm, and use-tensor-cores. CudnnAlgorithmPicker did the right thing, and that path is exercised 99% of the time. But CudnnAlgorithmPicker can fail, and if it does, we're stuck with whatever we had before. What we had before only had three of the four args, which is bad. In addition to fixing it, added an e2e test that catches the bug. PiperOrigin-RevId: 192694119 --- .../xla/service/gpu/ir_emission_utils.cc | 13 +++++++---- .../compiler/xla/tests/convolution_test.cc | 23 +++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc b/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc index 32413f975a..532d436ee8 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc +++ b/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc @@ -160,14 +160,19 @@ static HloInstruction* CreateCudnnConv( Shape call_shape = ShapeUtil::MakeTupleShape({shape, ShapeUtil::MakeShape(U8, {0})}); - // Our CustomCall takes three arguments: The conv lhs and rhs, and the cudnn - // algorithm to use. It's up to a later pass to choose the algorithm, so to - // indicate that we haven't yet made a choice, we speicfy -1 for that arg. + // Our CustomCall takes four arguments: The conv lhs and rhs, the cudnn + // algorithm to use, and a boolean indicating whether to use tensor cores. + // + // It's up to a later pass to choose the algorithm and decide whether to use + // tensor cores, so to indicate that we haven't yet made a choice, we speicfy + // -1 and false for those args. HloInstruction* negative_one = computation->AddInstruction( HloInstruction::CreateConstant(Literal::CreateR0(-1))); + HloInstruction* false_constant = computation->AddInstruction( + HloInstruction::CreateConstant(Literal::CreateR0(false))); HloInstruction* custom_call = computation->AddInstruction(HloInstruction::CreateCustomCall( - call_shape, {lhs, rhs, negative_one}, call_target)); + call_shape, {lhs, rhs, negative_one, false_constant}, call_target)); custom_call->set_window(window); custom_call->set_convolution_dimension_numbers(dnums); return custom_call; diff --git a/tensorflow/compiler/xla/tests/convolution_test.cc b/tensorflow/compiler/xla/tests/convolution_test.cc index 5eb3136abe..947959beb1 100644 --- a/tensorflow/compiler/xla/tests/convolution_test.cc +++ b/tensorflow/compiler/xla/tests/convolution_test.cc @@ -745,5 +745,28 @@ XLA_TEST_F(ConvolutionTest, Convolve_bf16_1x1x1x2_1x1x1x2_Valid) { error_spec_); } +// Check that GPU convs still work if the CudnnAlgorithmPicker pass is disabled. +// (We run this test on all platforms, because, what the heck.) +XLA_TEST_F(ConvolutionTest, NoCudnnAlgorithmPicker) { + execution_options_.mutable_debug_options()->add_xla_disable_hlo_passes( + "cudnn-convolution-algorithm-picker"); + + XlaBuilder builder(TestName()); + Shape input_shape = ShapeUtil::MakeShape(F32, {1, 1, 1, 2}); + Shape filter_shape = ShapeUtil::MakeShape(F32, {1, 1, 1, 2}); + auto input = builder.Parameter(0, input_shape, "input"); + auto filter = builder.Parameter(1, filter_shape, "filter"); + builder.Conv(input, filter, {1, 1}, Padding::kValid); + + Array4D input_data(1, 1, 1, 2); + input_data.FillIota(0); + Array4D filter_data(1, 1, 1, 2); + filter_data.FillIota(10); + + ComputeAndCompare(&builder, + {std::move(*Literal::CreateFromArray(input_data)), + std::move(*Literal::CreateFromArray(filter_data))}); +} + } // namespace } // namespace xla -- GitLab From 59c828c5f0d040f6461534d7760e2ff6e89b3f1a Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Thu, 12 Apr 2018 16:57:40 -0700 Subject: [PATCH 290/791] Document support for boolean values in tf.contrib.training.HParams. PiperOrigin-RevId: 192694244 --- tensorflow/contrib/training/python/training/hparam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/training/python/training/hparam.py b/tensorflow/contrib/training/python/training/hparam.py index 185f70a86d..6c59b68053 100644 --- a/tensorflow/contrib/training/python/training/hparam.py +++ b/tensorflow/contrib/training/python/training/hparam.py @@ -315,7 +315,7 @@ class HParams(object): Hyperparameters have type, which is inferred from the type of their value passed at construction type. The currently supported types are: integer, - float, string, and list of integer, float, or string. + float, boolean, string, and list of integer, float, boolean, or string. You can override hyperparameter values by calling the [`parse()`](#HParams.parse) method, passing a string of comma separated -- GitLab From 4d568133aade026bfc3bcee3a444682a349058b6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 16:59:57 -0700 Subject: [PATCH 291/791] Misc. small optimizations in Grappler and shape inference code. Impact on time per optimizer on inception graph: model_pruner: 590 ms -> 550 ms (-7%) function_optimizer: 130 ms -> 130 ms (-0%) constant_folding: 7600 ms -> 7550 ms (-0.7%) arithmetic_optimizer: 1860 ms -> 1550 ms (-20%) loop_optimizer: 320 ms -> 320 ms (-0%) dependency_optimizer: 1300 ms -> 720 ms (-45%) layout: 1400 ms -> 1400 ms (-0%) memory_optimizer: 4200 ms -> 3540 ms (-16%) PiperOrigin-RevId: 192694528 --- tensorflow/core/framework/shape_inference.cc | 18 +++--- tensorflow/core/graph/graph_constructor.cc | 48 +++++++++------ .../core/grappler/costs/graph_memory.cc | 34 +++++++---- tensorflow/core/grappler/grappler_item.cc | 4 +- tensorflow/core/grappler/grappler_item.h | 6 +- .../optimizers/arithmetic_optimizer.cc | 15 +++-- .../grappler/optimizers/constant_folding.cc | 6 +- .../optimizers/dependency_optimizer.cc | 45 ++++++-------- .../grappler/optimizers/memory_optimizer.cc | 2 +- .../grappler/optimizers/meta_optimizer.cc | 61 ++++++++----------- tensorflow/core/grappler/utils.cc | 41 ++++--------- tensorflow/core/grappler/utils.h | 34 ++++++++++- 12 files changed, 169 insertions(+), 145 deletions(-) diff --git a/tensorflow/core/framework/shape_inference.cc b/tensorflow/core/framework/shape_inference.cc index cc1ec47a83..229b4a45fa 100644 --- a/tensorflow/core/framework/shape_inference.cc +++ b/tensorflow/core/framework/shape_inference.cc @@ -40,6 +40,7 @@ InferenceContext::InferenceContext( : graph_def_version_(graph_def_version), node_def_(CHECK_NOTNULL(node_def)) { std::vector input_tensors_as_shape_handles; + input_tensors_as_shape_handles.reserve(input_tensors_as_shapes.size()); for (const TensorShapeProto& p : input_tensors_as_shapes) { ShapeHandle shape; construction_status_.Update(MakeShapeFromShapeProto(p, &shape)); @@ -50,6 +51,7 @@ InferenceContext::InferenceContext( } PreInputInit(op_def, input_tensors, input_tensors_as_shape_handles); if (!construction_status_.ok()) return; + inputs_.reserve(input_shapes.size()); for (const TensorShapeProto& p : input_shapes) { ShapeHandle shape; construction_status_.Update(MakeShapeFromShapeProto(p, &shape)); @@ -93,6 +95,7 @@ InferenceContext::InferenceContext( : graph_def_version_(graph_def_version), node_def_(CHECK_NOTNULL(node_def)) { std::vector input_tensors_as_shape_handles; + input_tensors_as_shape_handles.reserve(input_tensors_as_shapes.size()); for (const PartialTensorShape& p : input_tensors_as_shapes) { ShapeHandle shape; construction_status_.Update(MakeShapeFromPartialTensorShape(p, &shape)); @@ -103,6 +106,7 @@ InferenceContext::InferenceContext( } PreInputInit(op_def, input_tensors, input_tensors_as_shape_handles); if (!construction_status_.ok()) return; + inputs_.reserve(input_shapes.size()); for (const PartialTensorShape& p : input_shapes) { ShapeHandle shape; construction_status_.Update(MakeShapeFromPartialTensorShape(p, &shape)); @@ -229,9 +233,7 @@ void InferenceContext::PreInputInit( for (const auto& e : output_name_map_) { num_outputs = std::max(num_outputs, e.second.second); } - for (int i = 0; i < num_outputs; ++i) { - outputs_.push_back(nullptr); - } + outputs_.assign(num_outputs, nullptr); output_handle_shapes_and_types_.resize(num_outputs); } @@ -469,13 +471,15 @@ Status InferenceContext::MergePrefix(ShapeHandle s, ShapeHandle prefix, TF_RETURN_IF_ERROR(WithRankAtLeast(s, rank, &s)); // Merge the prefix dims and create the new output shapes. + const int32 rank_s = Rank(s); std::vector dims; + dims.reserve(std::max(rank, rank_s)); dims.resize(rank); for (int i = 0; i < rank; ++i) { TF_RETURN_IF_ERROR(Merge(Dim(s, i), Dim(prefix, i), &dims[i])); } *prefix_out = MakeShape(dims); - for (int i = rank; i < Rank(s); ++i) dims.push_back(Dim(s, i)); + for (int i = rank; i < rank_s; ++i) dims.push_back(Dim(s, i)); *s_out = MakeShape(dims); return Status::OK(); } @@ -1105,6 +1109,7 @@ Status InferenceContext::Max(DimensionHandle first, DimensionOrConstant second, Status InferenceContext::AttachContext(const Status& status) { std::vector input_shapes; + input_shapes.reserve(inputs_.size()); for (const ShapeHandle& input_shape : inputs_) { input_shapes.emplace_back(DebugString(input_shape)); } @@ -1112,6 +1117,7 @@ Status InferenceContext::AttachContext(const Status& status) { // Add information about the input tensors and partial tensor shapes used. std::vector input_from_tensors_str; std::vector input_from_tensors_as_shape_str; + input_from_tensors_as_shape_str.reserve(inputs_.size()); for (int i = 0; i < inputs_.size(); ++i) { if (requested_input_tensor_as_partial_shape_[i] && i < input_tensors_as_shapes_.size() && @@ -1233,9 +1239,7 @@ bool InferenceContext::RelaxHandleShapesAndMergeTypes( if (!refined) { return false; } - for (int i = 0; i < new_values.size(); ++i) { - (*to_update)[i] = new_values[i]; - } + to_update->swap(new_values); return true; } diff --git a/tensorflow/core/graph/graph_constructor.cc b/tensorflow/core/graph/graph_constructor.cc index 250992fb7a..c678283fce 100644 --- a/tensorflow/core/graph/graph_constructor.cc +++ b/tensorflow/core/graph/graph_constructor.cc @@ -666,20 +666,17 @@ Status GraphConstructor::ModifyNodeDefForImport(NodeDef* node_def) { void RemoveInputs(const std::vector& inputs_to_remove, NodeDef* node_def, std::vector* input_already_exists) { // Remove 'inputs_to_remove' from 'node_def' - // TODO(skyewm): is there a better way to do this? - std::vector inputs; - inputs.reserve(node_def->input_size()); - for (int i = 0; i < node_def->input_size(); ++i) { - inputs.push_back(node_def->input(i)); - } - node_def->clear_input(); - for (int i = 0, j = 0; i < inputs.size(); ++i) { + NodeDef copy; + copy.mutable_input()->Reserve(node_def->input_size() - + inputs_to_remove.size()); + for (int i = 0, j = 0; i < node_def->input_size(); ++i) { if (j < inputs_to_remove.size() && i == inputs_to_remove[j]) { ++j; } else { - node_def->add_input(inputs[i]); + copy.add_input()->swap(*node_def->mutable_input(i)); } } + node_def->mutable_input()->Swap(copy.mutable_input()); // Remove 'inputs_to_remove' from 'input_already_exists' for (int idx : inputs_to_remove) { input_already_exists->erase(input_already_exists->begin() + idx); @@ -745,9 +742,21 @@ void GraphConstructor::AddControlDependencies( // dependencies for (const string& control_dep : opts_.control_dependencies) { string input = TensorId(control_dep, Graph::kControlSlot).ToString(); - const protobuf::RepeatedPtrField& inputs = node_def->input(); - if (std::find(inputs.begin(), inputs.end(), input) != inputs.end()) { - // Control dependency already exists + bool found = false; + for (int i = node_def->input_size() - 1; i >= 0; --i) { + const string& node_input = node_def->input(i); + if (node_input[0] != '^') { + // Control inputs are at the end. Break when we reach the non-control + // inputs. + break; + } + if (node_input == input) { + // Control dependency already exists + found = true; + break; + } + } + if (found) { continue; } node_def->add_input(input); @@ -761,10 +770,10 @@ void GraphConstructor::AddPrefixToNodeDef( node_def->set_name(strings::StrCat(prefix_, node_def->name())); // Update names of input nodes for (int i = 0; i < node_def->input_size(); ++i) { - StringPiece input(node_def->input(i)); // Skip remapped inputs (which already exist in g_ and are not being // imported). if (input_already_exists[i]) continue; + StringPiece input(node_def->input(i)); if (str_util::ConsumePrefix(&input, "^")) { node_def->set_input(i, strings::StrCat("^", prefix_, input)); } else { @@ -933,10 +942,10 @@ Status GraphConstructor::Convert() { } } - // TODO(ashankar): The line below means an additional copy of the NodeDef, - // which can be expensive if the NodeDef contains large tensors in it. - // Might make sense to change the API for ImportGraphDef to take a mutable - // GraphDef* and avoid the copying. + // TODO(ashankar): The line below means an additional copy of the + // NodeDef, which can be expensive if the NodeDef contains large tensors + // in it. Might make sense to change the API for ImportGraphDef to take + // a mutable GraphDef* and avoid the copying. imported_node_def = original_node_def; if (!opts_.input_map.empty()) { // Note that input_already_exists can shrink here @@ -980,7 +989,7 @@ Status GraphConstructor::Convert() { src_node->num_outputs(), " outputs"); } - inputs.push_back(InputInfo(id.first.ToString(), src_node, src_index)); + inputs.emplace_back(id.first.ToString(), src_node, src_index); } if (has_data_back_edge && !IsMerge(*node_def)) { @@ -1010,8 +1019,7 @@ Status GraphConstructor::Convert() { if (inputs[i].node == nullptr) { // Record this back edge, which will be added after all nodes // are created. - back_edges_.push_back( - EdgeInfo(inputs[i].name, inputs[i].index, node, i)); + back_edges_.emplace_back(inputs[i].name, inputs[i].index, node, i); } else if (inputs[i].index == Graph::kControlSlot) { g_->AddControlEdge(inputs[i].node, node); } else { diff --git a/tensorflow/core/grappler/costs/graph_memory.cc b/tensorflow/core/grappler/costs/graph_memory.cc index 3604de392f..a5736d40b1 100644 --- a/tensorflow/core/grappler/costs/graph_memory.cc +++ b/tensorflow/core/grappler/costs/graph_memory.cc @@ -14,7 +14,8 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/grappler/costs/graph_memory.h" -#include + +#include #include "tensorflow/core/framework/allocation_description.pb.h" #include "tensorflow/core/framework/attr_value.pb.h" #include "tensorflow/core/framework/node_def.pb.h" @@ -120,7 +121,7 @@ int64 GraphMemory::InferMemUsageForNeighbors( static GraphMemory::LiveTensor* FindOrCreateLiveTensor( const string& node_name, int output_id, std::unordered_map* live_tensors, - std::list* device_tensors) { + std::deque* device_tensors) { string name = strings::StrCat(node_name, ":", output_id); GraphMemory::LiveTensor* live; auto it = live_tensors->find(name); @@ -141,6 +142,10 @@ static GraphMemory::LiveTensor* FindOrCreateLiveTensor( namespace { struct Event { + Event(int64 _timestamp, bool _allocated, + const GraphMemory::LiveTensor* _tensor) + : timestamp(_timestamp), allocated(_allocated), tensor(_tensor) {} + int64 timestamp; bool allocated; const GraphMemory::LiveTensor* tensor; @@ -160,13 +165,15 @@ void GraphMemory::InferFromTrace(const StepStats& timeline) { } std::unordered_map live_tensors; - std::unordered_map> live_tensors_per_device; - - NodeMap node_map(&item_.graph); + std::unordered_map> live_tensors_per_device; + std::unordered_map node_map; + for (const NodeDef& node : item_.graph.node()) { + node_map[node.name()] = &node; + } for (const auto& dev_stats : timeline.dev_stats()) { const string& device_name = dev_stats.device(); const bool is_gpu = (device_name.find("GPU:") || device_name.find("gpu:")); - std::list& device_tensors = + std::deque& device_tensors = live_tensors_per_device[dev_stats.device()]; for (const auto& node_stats : dev_stats.node_stats()) { for (int i = 0; i < node_stats.output_size(); ++i) { @@ -191,12 +198,13 @@ void GraphMemory::InferFromTrace(const StepStats& timeline) { node_stats.op_end_rel_micros())); } - const NodeDef* node = node_map.GetNode(node_stats.node_name()); - if (!node) { + auto it = node_map.find(node_stats.node_name()); + if (it == node_map.end()) { // Skip nodes inserted by TF since they don't exist in the original // graph (e.g _Send/_Recv nodes). continue; } + const NodeDef* node = it->second; std::unordered_set swapped_inputs; if (is_gpu) { auto it = node->attr().find("_swap_to_host"); @@ -237,14 +245,16 @@ void GraphMemory::InferFromTrace(const StepStats& timeline) { std::vector events; events.reserve(2 * live_per_device.second.size()); for (const auto& live : live_per_device.second) { - events.push_back(Event{live.allocation_time.count(), true, &live}); - events.push_back(Event{live.deallocation_time.count(), false, &live}); + events.emplace_back(static_cast(live.allocation_time.count()), + true, &live); + events.emplace_back(static_cast(live.deallocation_time.count()), + false, &live); } std::stable_sort(events.begin(), events.end()); size_t peak = 0; - std::set live_at_peak; + std::unordered_set live_at_peak; size_t current = 0; - std::set currently_live; + std::unordered_set currently_live; for (int i = 0; i < events.size(); ++i) { const auto& event = events[i]; diff --git a/tensorflow/core/grappler/grappler_item.cc b/tensorflow/core/grappler/grappler_item.cc index ad86356504..bbc0fedd22 100644 --- a/tensorflow/core/grappler/grappler_item.cc +++ b/tensorflow/core/grappler/grappler_item.cc @@ -27,7 +27,7 @@ limitations under the License. namespace tensorflow { namespace grappler { -GrapplerItem::GrapplerItem(const GrapplerItem& other, GraphDef&& graphDef) { +GrapplerItem::GrapplerItem(const GrapplerItem& other, GraphDef* graph_def) { id = other.id; feed = other.feed; fetch = other.fetch; @@ -38,7 +38,7 @@ GrapplerItem::GrapplerItem(const GrapplerItem& other, GraphDef&& graphDef) { restore_op = other.restore_op; save_restore_loc_tensor = other.save_restore_loc_tensor; queue_runners = other.queue_runners; - graph.Swap(&graphDef); + graph.Swap(graph_def); } std::vector GrapplerItem::MainOpsFanin() const { diff --git a/tensorflow/core/grappler/grappler_item.h b/tensorflow/core/grappler/grappler_item.h index 45eed47b50..cd165ac3d4 100644 --- a/tensorflow/core/grappler/grappler_item.h +++ b/tensorflow/core/grappler/grappler_item.h @@ -33,10 +33,12 @@ namespace grappler { // A TensorFlow model to optimize. // Models are represented by the combination of a graph, one of more fetch // nodes, and potentially a set of nodes to feed. -// TODO(volunteer_needed): turn this struct into a class. struct GrapplerItem { GrapplerItem() = default; - GrapplerItem(const GrapplerItem& other, GraphDef&& graphDef); + GrapplerItem(const GrapplerItem& other, GraphDef&& graph_def) + : GrapplerItem(other, &graph_def) {} + // Swaps *graph_def with an empty GraphDef. + GrapplerItem(const GrapplerItem& other, GraphDef* graph_def); virtual ~GrapplerItem() = default; string id; // A unique id for this item diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc index 463c332858..60b1af48ec 100644 --- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc @@ -253,9 +253,8 @@ NodeDef* GetTailOfValuePreservingChain( const NodeDef& node, const NodeMap& node_map, const std::unordered_set& nodes_to_preserve) { auto is_value_preserving_non_branching = [&](const NodeDef& node) { - return IsValuePreserving(node) && - NumNonControlOutputs(node, node_map) == 1 && - nodes_to_preserve.count(node.name()) == 0; + return nodes_to_preserve.find(node.name()) == nodes_to_preserve.end() && + IsValuePreserving(node) && NumNonControlOutputs(node, node_map) == 1; }; return GetTailOfChain(node, node_map, /*follow_control_input=*/false, is_value_preserving_non_branching); @@ -2023,12 +2022,11 @@ Status ArithmeticOptimizer::SimplifyArithmeticOps(bool can_use_shapes) { Status ArithmeticOptimizer::Optimize(Cluster* /*cluster*/, const GrapplerItem& item, GraphDef* optimized_graph) { - GrapplerItem optimized_item(item); - optimized_graph_ = &optimized_item.graph; - // Set up helper data structures. nodes_to_preserve_ = item.NodesToPreserve(); fetch_nodes_known_ = !item.fetch.empty(); + *optimized_graph = item.graph; + optimized_graph_ = optimized_graph; node_map_.reset(new NodeMap(optimized_graph_)); DedupComputations(); @@ -2037,8 +2035,9 @@ Status ArithmeticOptimizer::Optimize(Cluster* /*cluster*/, // optimize larger subgraphs starting from the roots with more inputs. TF_RETURN_IF_ERROR(TopologicalSort(optimized_graph_)); - // Shapes are only needed in aggressive mode. - graph_properties_.reset(new GraphProperties(item)); + GrapplerItem optimized_item(item, optimized_graph); + optimized_graph_ = &optimized_item.graph; + graph_properties_.reset(new GraphProperties(optimized_item)); const Status status = graph_properties_->InferStatically(false); const bool can_use_shapes = status.ok(); if (!can_use_shapes) { diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc index b2a1ce6ab6..e29aaa25fe 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding.cc +++ b/tensorflow/core/grappler/optimizers/constant_folding.cc @@ -1004,7 +1004,7 @@ Status ConstantFolding::EvaluateOneFoldable(const NodeDef& node, for (const auto& input : node.input()) { int port = 0; - ParseNodeName(input, &port); + ParseNodeNameAsStringPiece(input, &port); if (port < 0) { // Control dependency break; @@ -2084,9 +2084,9 @@ Status ConstantFolding::SimplifyGraph(GraphDef* optimized_graph, left_child_is_constant ? left_child : right_child; // Make sure that it is safe to change the value of the child node-> if (op_child_node->input_size() < 2 || - NumNonControlOutputs(*op_child_node, *node_map_) > 1 || nodes_to_preserve_.find(op_child_node->name()) != - nodes_to_preserve_.end()) { + nodes_to_preserve_.end() || + NumNonControlOutputs(*op_child_node, *node_map_) > 1) { continue; } diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc index ed9bce439c..7b7fd81155 100644 --- a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc @@ -109,23 +109,12 @@ bool DependencyOptimizer::SafeToRemoveIdentity(const NodeDef& node) { } bool DependencyOptimizer::SafeToConvertToNoOp(const NodeDef& node) { - if (nodes_to_preserve_.find(node.name()) != nodes_to_preserve_.end()) { - return false; - } - if (!fetch_nodes_known_ || NumNonControlOutputs(node, *node_map_) > 0) { - // The output values of this node may be needed. - return false; - } - if (IsMerge(node) || IsSwitch(node)) { - return false; - } - if (ModifiesFrameInfo(node)) { - return false; - } - if (!IsFreeOfSideEffect(node)) { + if (!fetch_nodes_known_ || + nodes_to_preserve_.find(node.name()) != nodes_to_preserve_.end()) { return false; } - if (node.op() == "ControlTrigger") { + if (IsMerge(node) || IsSwitch(node) || ModifiesFrameInfo(node) || + !IsFreeOfSideEffect(node)) { return false; } if (node.op().rfind("Submodel", 0) == 0) { @@ -136,16 +125,21 @@ bool DependencyOptimizer::SafeToConvertToNoOp(const NodeDef& node) { if (!status.ok() || op_def->output_arg_size() == 0) { return false; } - + const std::unordered_set do_not_rewrite_ops{ + "Assert", "CheckNumerics", "_Retval", + "_Arg", "_ParallelConcatUpdate", "_TPUExecute", + "_TPUCompile", "ControlTrigger"}; + if (do_not_rewrite_ops.find(node.op()) != do_not_rewrite_ops.end()) { + return false; + } if (!SafeToRemoveIdentity(node)) { return false; } - - const std::unordered_set do_not_rewrite_ops{ - "Assert", "CheckNumerics", "_Retval", - "_Arg", "_ParallelConcatUpdate", "_TPUExecute", - "_TPUCompile"}; - return do_not_rewrite_ops.find(node.op()) == do_not_rewrite_ops.end(); + if (NumNonControlOutputs(node, *node_map_) > 0) { + // The output values of this node may be needed. + return false; + } + return true; } void DependencyOptimizer::OptimizeNode(int node_idx, @@ -164,7 +158,8 @@ void DependencyOptimizer::OptimizeNode(int node_idx, bool data_connection = false; for (int i = fanout->input_size() - 1; i >= 0; --i) { int pos; - string input_name = ParseNodeName(fanout->input(i), &pos); + StringPiece input_name = + ParseNodeNameAsStringPiece(fanout->input(i), &pos); if (input_name == node_name) { if (pos < 0) { fanout->mutable_input()->SwapElements(i, fanout->input_size() - 1); @@ -358,8 +353,8 @@ void DependencyOptimizer::OptimizeNode(int node_idx, for (int j = 0; j < consumer->input_size(); ++j) { const string& old_input = consumer->input(j); int old_input_pos; - string old_input_node_name = - ParseNodeName(old_input, &old_input_pos); + StringPiece old_input_node_name = + ParseNodeNameAsStringPiece(old_input, &old_input_pos); if (old_input_node_name == node_name) { if (old_input_pos >= 0) { // Regular input diff --git a/tensorflow/core/grappler/optimizers/memory_optimizer.cc b/tensorflow/core/grappler/optimizers/memory_optimizer.cc index 27e9d2c78d..c1fee0e993 100644 --- a/tensorflow/core/grappler/optimizers/memory_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/memory_optimizer.cc @@ -1227,7 +1227,7 @@ Status MemoryOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item, recomputation_targets_name_scope_, optimized_graph, item); - GrapplerItem optimized_item(item, std::move(*optimized_graph)); + GrapplerItem optimized_item(item, optimized_graph); std::unordered_set skip_list; // Bound the number of rewrite passes to avoid long processing times on graphs // that simply won't fit in memory. diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc index 5723e397ab..558b8a77e8 100644 --- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc @@ -178,45 +178,41 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item, cfg_.meta_optimizer_iterations() == RewriterConfig::DEFAULT_NUM_ITERS ? 1 : cfg_.meta_optimizer_iterations(); + GrapplerItem optimized_item = item; + optimized_graph->Swap(&optimized_item.graph); for (int iteration = 0; iteration < num_iterations; ++iteration) { VLOG(1) << "Starting optimization iteration " << iteration + 1; for (const auto& optimizer : optimizers) { + // Invariant: optimized_graph contains the most recently optimized + // version of the graph. if (iteration > 0 && run_once_optimizers.count(optimizer->name())) { continue; } - if (!already_optimized) { - Status status = optimizer->Optimize(cluster, item, optimized_graph); - string result; - if (!status.ok()) { - VLOG(1) << "Not able to apply optimizer " << optimizer->name() - << ". Return status: " << status.ToString(); - result = status.ToString(); - } else { - already_optimized = true; - result = strings::StrCat( - "OK. ", PrintSizesBeforeAfter(item.graph, *optimized_graph)); - } - result_.push_back(std::make_pair(optimizer->name(), result)); - VLOG(1) << "Optimizer " << optimizer->name() - << " return status: " << result; + uint64 start_us = Env::Default()->NowMicros(); + // This swaps the current optimized_graph into optimized item and + // resets optimized_graph to an empty graph. + optimized_graph->Swap(&optimized_item.graph); + *optimized_graph = GraphDef(); + Status status = + optimizer->Optimize(cluster, optimized_item, optimized_graph); + + uint64 end_us = Env::Default()->NowMicros(); + float duration_ms = (end_us - start_us) / 1000.0f; + string result; + if (!status.ok()) { + VLOG(1) << "Not able to apply optimizer " << optimizer->name() << ": " + << status.ToString(); + optimized_graph->Swap(&optimized_item.graph); + result = status.ToString(); } else { - GrapplerItem optimized_item(item, std::move(*optimized_graph)); - Status status = - optimizer->Optimize(cluster, optimized_item, optimized_graph); - string result; - if (!status.ok()) { - VLOG(1) << "Not able to apply optimizer " << optimizer->name() << ": " - << status.ToString(); - optimized_graph->Swap(&optimized_item.graph); - result = status.ToString(); - } else { - result = strings::StrCat( - optimizer->name(), ": ", - PrintSizesBeforeAfter(optimized_item.graph, *optimized_graph)); - } - result_.push_back(std::make_pair(optimizer->name(), result)); - VLOG(1) << result; + already_optimized = true; + result = strings::StrCat( + optimizer->name(), ": ", + PrintSizesBeforeAfter(optimized_item.graph, *optimized_graph), + ", time = ", duration_ms, "ms."); } + result_.emplace_back(optimizer->name(), result); + VLOG(1) << result; } } @@ -230,10 +226,7 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item, item.graph.library().gradient_size()); DCHECK_EQ(optimized_graph->versions().producer(), item.graph.versions().producer()); - } else { - *optimized_graph = item.graph; } - return Status::OK(); } diff --git a/tensorflow/core/grappler/utils.cc b/tensorflow/core/grappler/utils.cc index 534fe670e0..7398d2c896 100644 --- a/tensorflow/core/grappler/utils.cc +++ b/tensorflow/core/grappler/utils.cc @@ -142,38 +142,12 @@ bool IsSameInput(const string& name1, const string& name2) { return true; } int position1; - string node1 = ParseNodeName(name1, &position1); + StringPiece node1 = ParseNodeNameAsStringPiece(name1, &position1); int position2; - string node2 = ParseNodeName(name2, &position2); + StringPiece node2 = ParseNodeNameAsStringPiece(name2, &position2); return (position1 == position2) && (node1 == node2); } -string ParseNodeName(const string& name, int* position) { - // Strip the prefix '^' (if any), and strip the trailing ":{digits} (if any) - // to get a node name. - strings::Scanner scan(name); - scan.ZeroOrOneLiteral("^") - .RestartCapture() - .One(strings::Scanner::LETTER_DIGIT_DOT_UNDERSCORE) - .Any(strings::Scanner::LETTER_DIGIT_DASH_DOT_SLASH_UNDERSCORE); - StringPiece capture; - StringPiece remaining; - if (scan.Peek(':') != ':' || !scan.GetResult(&remaining, &capture)) { - *position = 0; - return ""; - } else { - if (name[0] == '^') { - *position = -1; - } else if (remaining.empty()) { - *position = 0; - } else { - // Skip the first ':' character. - CHECK(strings::safe_strto32(remaining.substr(1), position)); - } - return capture.ToString(); - } -} - bool IsControlInput(const string& name) { return !name.empty() && name[0] == '^'; } @@ -185,7 +159,7 @@ string NodeName(const string& name) { int NodePosition(const string& name) { int position; - ParseNodeName(name, &position); + ParseNodeNameAsStringPiece(name, &position); return position; } @@ -275,13 +249,20 @@ int NumNonControlInputs(const NodeDef& node) { int NumNonControlOutputs(const NodeDef& node, const NodeMap& node_map) { int num_outputs = 0; + int pos; for (const NodeDef* output : node_map.GetOutputs(node.name())) { for (const string& node_as_input : output->input()) { if (IsControlInput(node_as_input)) { break; } - if (NodeName(node_as_input) == node.name()) { + if (node_as_input == node.name()) { ++num_outputs; + } else { + const StringPiece name = + ParseNodeNameAsStringPiece(node_as_input, &pos); + if (name == node.name()) { + ++num_outputs; + } } } } diff --git a/tensorflow/core/grappler/utils.h b/tensorflow/core/grappler/utils.h index 11555d712a..b15667dca2 100644 --- a/tensorflow/core/grappler/utils.h +++ b/tensorflow/core/grappler/utils.h @@ -26,8 +26,10 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/core/stringpiece.h" #include "tensorflow/core/lib/core/threadpool.h" #include "tensorflow/core/lib/gtl/inlined_vector.h" +#include "tensorflow/core/lib/strings/scanner.h" namespace tensorflow { namespace grappler { @@ -107,8 +109,38 @@ string NodeName(const string& name); // Get the trailing position number ":{digits}" (if any) of a node name. int NodePosition(const string& name); +inline StringPiece ParseNodeNameAsStringPiece(const string& name, + int* position) { + // Strip the prefix '^' (if any), and strip the trailing ":{digits} (if any) + // to get a node name. + strings::Scanner scan(name); + scan.ZeroOrOneLiteral("^") + .RestartCapture() + .One(strings::Scanner::LETTER_DIGIT_DOT_UNDERSCORE) + .Any(strings::Scanner::LETTER_DIGIT_DASH_DOT_SLASH_UNDERSCORE); + StringPiece capture; + StringPiece remaining; + if (scan.Peek(':') != ':' || !scan.GetResult(&remaining, &capture)) { + *position = 0; + static const string empty; + return StringPiece(empty); + } else { + if (name[0] == '^') { + *position = -1; + } else if (remaining.empty()) { + *position = 0; + } else { + // Skip the first ':' character. + CHECK(strings::safe_strto32(remaining.substr(1), position)); + } + return capture; + } +} + // Returns the node name and position in a single call. -string ParseNodeName(const string& name, int* position); +inline string ParseNodeName(const string& name, int* position) { + return ParseNodeNameAsStringPiece(name, position).ToString(); +} // Add a prefix to a node name with a custom delimiter. string AddPrefixToNodeName(const string& name, const string& prefix, -- GitLab From 3755128f3a83fea84c5a90d71d5b684157a99ac7 Mon Sep 17 00:00:00 2001 From: Yuefeng Zhou Date: Thu, 12 Apr 2018 17:01:55 -0700 Subject: [PATCH 292/791] Fix a typo in cross_tower_ops. PiperOrigin-RevId: 192694794 --- tensorflow/contrib/distribute/python/cross_tower_ops.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/distribute/python/cross_tower_ops.py b/tensorflow/contrib/distribute/python/cross_tower_ops.py index bbe5e877d5..cff717db80 100644 --- a/tensorflow/contrib/distribute/python/cross_tower_ops.py +++ b/tensorflow/contrib/distribute/python/cross_tower_ops.py @@ -488,7 +488,8 @@ class AllReduceCrossTowerOps(CrossTowerOps): "agg_small_grads_max_group = %d", len(per_device_values), self.all_reduce_alg, self.agg_small_grads_max_bytes, self.agg_small_grads_max_group) - tensor_packer = AggregateSmallTensorPacker(100, 10) + tensor_packer = AggregateSmallTensorPacker( + self.agg_small_grads_max_bytes, self.agg_small_grads_max_group) device_grad_packs = tensor_packer.pack(grouped) else: logging.info( -- GitLab From fffd3ca4fcf1f54f97a7be6f225fe183ad82b0ea Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 17:07:35 -0700 Subject: [PATCH 293/791] Move dummy AssertOp and CheckNumericsOp to //third_party/tensorflow/compiler/tf2xla/kernels. Enable type DT_STRING for AssertOp and ConstOp, in order to make dummy Assert compile with a const string (assert message) as its input. PiperOrigin-RevId: 192695938 --- tensorflow/compiler/aot/BUILD | 1 + tensorflow/compiler/aot/tests/BUILD | 15 ++++++ .../compiler/aot/tests/make_test_graphs.py | 10 ++++ .../tests/test_graph_tfassert_eq.config.pbtxt | 16 ++++++ .../compiler/aot/tests/tfcompile_test.cc | 18 +++++++ .../compiler/jit/mark_for_compilation_pass.cc | 9 ++++ .../jit/mark_for_compilation_pass_test.cc | 24 +++++++++ tensorflow/compiler/tf2xla/kernels/BUILD | 17 +++++++ .../compiler/tf2xla/kernels/assert_op.cc | 49 ++++++++++++++++++ .../tf2xla/kernels/check_numerics_op.cc | 50 +++++++++++++++++++ tensorflow/compiler/tf2xla/tf2xla_util.cc | 9 ++++ tensorflow/compiler/tf2xla/tf2xla_util.h | 5 ++ tensorflow/compiler/tf2xla/xla_cpu_backend.cc | 7 +++ tensorflow/compiler/tf2xla/xla_gpu_backend.cc | 7 +++ 14 files changed, 237 insertions(+) create mode 100644 tensorflow/compiler/aot/tests/test_graph_tfassert_eq.config.pbtxt create mode 100644 tensorflow/compiler/tf2xla/kernels/assert_op.cc create mode 100644 tensorflow/compiler/tf2xla/kernels/check_numerics_op.cc diff --git a/tensorflow/compiler/aot/BUILD b/tensorflow/compiler/aot/BUILD index fa03b1f3c2..19e6bf68e7 100644 --- a/tensorflow/compiler/aot/BUILD +++ b/tensorflow/compiler/aot/BUILD @@ -60,6 +60,7 @@ cc_library( "//tensorflow/compiler/tf2xla:tf2xla_util", "//tensorflow/compiler/tf2xla:xla_compiler", "//tensorflow/compiler/tf2xla/kernels:xla_cpu_only_ops", + "//tensorflow/compiler/tf2xla/kernels:xla_dummy_ops", "//tensorflow/compiler/tf2xla/kernels:xla_ops", "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:statusor", diff --git a/tensorflow/compiler/aot/tests/BUILD b/tensorflow/compiler/aot/tests/BUILD index b053dad1b5..bb73cb19c5 100644 --- a/tensorflow/compiler/aot/tests/BUILD +++ b/tensorflow/compiler/aot/tests/BUILD @@ -14,6 +14,7 @@ test_suite( ":test_graph_tfadd_test", ":test_graph_tfadd_with_ckpt_saver_test", ":test_graph_tfadd_with_ckpt_test", + ":test_graph_tfassert_eq_test", ":test_graph_tffunction_test", ":test_graph_tfgather_test", ":test_graph_tfmatmul_test", @@ -33,6 +34,7 @@ py_binary( "//tensorflow/python", # TODO(b/34059704): remove when fixed "//tensorflow/python:array_ops", "//tensorflow/python:client", + "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:math_ops", "//tensorflow/python:platform", @@ -52,6 +54,7 @@ genrule( "test_graph_tfadd_with_ckpt_saver.ckpt", "test_graph_tfadd_with_ckpt_saver.pb", "test_graph_tfadd_with_ckpt_saver.saver", + "test_graph_tfassert_eq.pb", "test_graph_tffunction.pb", "test_graph_tfgather.pb", "test_graph_tfmatmul.pb", @@ -104,6 +107,17 @@ tf_library( ], ) +tf_library( + name = "test_graph_tfassert_eq", + testonly = 1, + config = "test_graph_tfassert_eq.config.pbtxt", + cpp_class = "AssertComp", + graph = "test_graph_tfassert_eq.pb", + tags = [ + "manual", + ], +) + tf_library( name = "test_graph_tffunction", testonly = 1, @@ -170,6 +184,7 @@ tf_cc_test( ":test_graph_tfadd", ":test_graph_tfadd_with_ckpt", ":test_graph_tfadd_with_ckpt_saver", + ":test_graph_tfassert_eq", ":test_graph_tffunction", ":test_graph_tfgather", ":test_graph_tfmatmul", diff --git a/tensorflow/compiler/aot/tests/make_test_graphs.py b/tensorflow/compiler/aot/tests/make_test_graphs.py index 89c7cd4507..67767f55da 100644 --- a/tensorflow/compiler/aot/tests/make_test_graphs.py +++ b/tensorflow/compiler/aot/tests/make_test_graphs.py @@ -29,6 +29,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import function from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import variables from tensorflow.python.platform import app @@ -125,6 +126,14 @@ def tfsplits(_): array_ops.identity(y, name='result') +def tfassert_eq(_): + x = array_ops.placeholder(dtypes.int32, name='x_hold') + y = array_ops.placeholder(dtypes.int32, name='y_hold') + control_flow_ops.Assert( + math_ops.equal(x, y), ['Expected x == y.'], name='assert_eq') + math_ops.add(x, math_ops.negative(y), name='x_y_diff') + + def write_graph(build_graph, out_dir): """Build a graph using build_graph and write it out.""" g = ops.Graph() @@ -144,6 +153,7 @@ def main(_): write_graph(tfmatmulandadd, FLAGS.out_dir) write_graph(tffunction, FLAGS.out_dir) write_graph(tfsplits, FLAGS.out_dir) + write_graph(tfassert_eq, FLAGS.out_dir) if __name__ == '__main__': diff --git a/tensorflow/compiler/aot/tests/test_graph_tfassert_eq.config.pbtxt b/tensorflow/compiler/aot/tests/test_graph_tfassert_eq.config.pbtxt new file mode 100644 index 0000000000..8732d1709e --- /dev/null +++ b/tensorflow/compiler/aot/tests/test_graph_tfassert_eq.config.pbtxt @@ -0,0 +1,16 @@ +# Text form of tensorflow.tf2xla.Config proto. +feed { + id { node_name: "x_hold" } + shape { + dim { size: 1 } + } +} +feed { + id { node_name: "y_hold" } + shape { + dim { size: 1 } + } +} +fetch { + id { node_name: "x_y_diff" } +} diff --git a/tensorflow/compiler/aot/tests/tfcompile_test.cc b/tensorflow/compiler/aot/tests/tfcompile_test.cc index 413efd9cea..67dbd643bf 100644 --- a/tensorflow/compiler/aot/tests/tfcompile_test.cc +++ b/tensorflow/compiler/aot/tests/tfcompile_test.cc @@ -20,6 +20,7 @@ limitations under the License. #include "tensorflow/compiler/aot/tests/test_graph_tfadd.h" #include "tensorflow/compiler/aot/tests/test_graph_tfadd_with_ckpt.h" #include "tensorflow/compiler/aot/tests/test_graph_tfadd_with_ckpt_saver.h" +#include "tensorflow/compiler/aot/tests/test_graph_tfassert_eq.h" #include "tensorflow/compiler/aot/tests/test_graph_tffunction.h" #include "tensorflow/compiler/aot/tests/test_graph_tfgather.h" #include "tensorflow/compiler/aot/tests/test_graph_tfmatmul.h" @@ -413,6 +414,23 @@ TEST(TFCompileTest, Splits) { EXPECT_NEAR(expected[3], fn.result0(1, 1), 1e4); } +TEST(TFCompileTest, AssertEqAndReturnDiff) { + // Assert is converted into a no-op in XLA, so there is no failure even if the + // two args are different. + AssertComp assert; + EXPECT_EQ(assert.arg0_data(), assert.args()[0]); + EXPECT_EQ(assert.arg1_data(), assert.args()[1]); + + assert.arg0() = 2; + assert.arg1() = 1; + const int32 expected_result = assert.arg0() - assert.arg1(); + EXPECT_TRUE(assert.Run()); + EXPECT_EQ(assert.error_msg(), ""); + EXPECT_EQ(assert.result0(), expected_result); + EXPECT_EQ(assert.result0_data()[0], expected_result); + EXPECT_EQ(assert.result0_data(), assert.results()[0]); +} + TEST(TFCompileTest, LookupNameIndex) { // add doesn't have any names defined in its config. AddComp add; diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc index f32c0f4ba8..0c9fbf3d54 100644 --- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc +++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc @@ -50,6 +50,15 @@ bool HasXLAKernel(const Node& node, const DeviceType& jit_device_type) { // is really a kind of function call and will be handled by // IsCompilableCall(). if (node.type_string() == "SymbolicGradient") return false; + if (node.type_string() == "Const") { + // Skip Const op with type DT_STRING, since XLA doesn't support it, but the + // registered Const KernelDef says that it does, to support no-op Assert for + // tfcompile. + const AttrValue* attr = node.attrs().Find("dtype"); + if (attr != nullptr && attr->type() == DT_STRING) { + return false; + } + } return FindKernelDef(jit_device_type, node.def(), nullptr, nullptr).ok(); } diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc index 80edaf28b8..703d8825d7 100644 --- a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc +++ b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc @@ -609,5 +609,29 @@ TEST(XlaCompilationTest, DontCountIdentityOpsWithLocalJit) { EXPECT_TRUE(clusters.empty()); } +TEST(XlaCompilationTest, ConstOp) { + // valid data type + { + std::unique_ptr graph(new Graph(OpRegistry::Global())); + Scope root = Scope::NewRootScope().ExitOnError(); + auto c = ops::Const(root.WithOpName("const"), 0.5f); + c.node()->AddAttr(kXlaCompileAttr, true); + TF_ASSERT_OK(root.ToGraph(graph.get())); + TF_ASSERT_OK(MarkForCompilation(&graph)); + EXPECT_EQ(1, GetClusters(*graph).size()); + } + + // invalid data type + { + std::unique_ptr graph(new Graph(OpRegistry::Global())); + Scope root = Scope::NewRootScope().ExitOnError(); + auto c = ops::Const(root.WithOpName("const"), string("string")); + c.node()->AddAttr(kXlaCompileAttr, true); + TF_ASSERT_OK(root.ToGraph(graph.get())); + TF_ASSERT_OK(MarkForCompilation(&graph)); + EXPECT_TRUE(GetClusters(*graph).empty()); + } +} + } // namespace } // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD index f1bc7d6af4..3ba37b0383 100644 --- a/tensorflow/compiler/tf2xla/kernels/BUILD +++ b/tensorflow/compiler/tf2xla/kernels/BUILD @@ -171,6 +171,23 @@ tf_kernel_library( ], ) +# Kernels that have a dummy (no-op) implementation. +tf_kernel_library( + name = "xla_dummy_ops", + srcs = [ + "assert_op.cc", + "check_numerics_op.cc", + ], + deps = [ + "//tensorflow/compiler/tf2xla:xla_compiler", + "//tensorflow/core:array_ops_op_lib", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:logging_ops_op_lib", + ], + alwayslink = 1, +) + # Kernels that only work on CPU, because they use XLA custom calls. # Only link this when using the CPU backend for XLA. tf_kernel_library( diff --git a/tensorflow/compiler/tf2xla/kernels/assert_op.cc b/tensorflow/compiler/tf2xla/kernels/assert_op.cc new file mode 100644 index 0000000000..af4ab5e8ef --- /dev/null +++ b/tensorflow/compiler/tf2xla/kernels/assert_op.cc @@ -0,0 +1,49 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/tf2xla/xla_op_kernel.h" +#include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/mutex.h" + +namespace tensorflow { + +namespace { + +// This TensorFlow op supports the Assert primitve. +class AssertOp : public XlaOpKernel { + public: + explicit AssertOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {} + ~AssertOp() override {} + + void Compile(XlaOpKernelContext* ctx) override { + static mutex mu(tensorflow::LINKER_INITIALIZED); + static int log_counter = 0; + + mutex_lock l(mu); + if (log_counter < 20) { + ++log_counter; + LOG(WARNING) << "Ignoring Assert operator " << name(); + } + } + + private: + TF_DISALLOW_COPY_AND_ASSIGN(AssertOp); +}; + +REGISTER_XLA_OP(Name("Assert"), AssertOp); + +} // anonymous namespace +} // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/kernels/check_numerics_op.cc b/tensorflow/compiler/tf2xla/kernels/check_numerics_op.cc new file mode 100644 index 0000000000..6061e822d8 --- /dev/null +++ b/tensorflow/compiler/tf2xla/kernels/check_numerics_op.cc @@ -0,0 +1,50 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/tf2xla/xla_op_kernel.h" +#include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/mutex.h" + +namespace tensorflow { +namespace { + +class CheckNumericsOp : public XlaOpKernel { + public: + explicit CheckNumericsOp(OpKernelConstruction* context) + : XlaOpKernel(context) {} + + void Compile(XlaOpKernelContext* ctx) override { + // TODO(b/32223192): add a real implementation of CheckNumerics + { + static mutex mu(tensorflow::LINKER_INITIALIZED); + static int log_counter = 0; + mutex_lock l(mu); + if (log_counter < 20) { + ++log_counter; + LOG(WARNING) << "Ignoring CheckNumerics operator " << name(); + } + } + ctx->SetOutput(0, ctx->Input(0)); + } + + private: + TF_DISALLOW_COPY_AND_ASSIGN(CheckNumericsOp); +}; + +REGISTER_XLA_OP(Name("CheckNumerics"), CheckNumericsOp); + +} // anonymous namespace +} // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/tf2xla_util.cc b/tensorflow/compiler/tf2xla/tf2xla_util.cc index 2fc77cc4bc..7ec85aa3cd 100644 --- a/tensorflow/compiler/tf2xla/tf2xla_util.cc +++ b/tensorflow/compiler/tf2xla/tf2xla_util.cc @@ -288,4 +288,13 @@ Status SetNodeShardingFromNeighbors(Node* n, bool out_edges) { return Status::OK(); } +void AddDtypeToKernalDefConstraint(StringPiece name, DataType dtype, + KernelDef* kdef) { + for (KernelDef::AttrConstraint& constraint : *kdef->mutable_constraint()) { + if (constraint.name() == name) { + constraint.mutable_allowed_values()->mutable_list()->add_type(dtype); + } + } +} + } // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/tf2xla_util.h b/tensorflow/compiler/tf2xla/tf2xla_util.h index e5fba8ede7..745beb39c1 100644 --- a/tensorflow/compiler/tf2xla/tf2xla_util.h +++ b/tensorflow/compiler/tf2xla/tf2xla_util.h @@ -20,6 +20,7 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/tf2xla.pb.h" #include "tensorflow/core/framework/graph.pb.h" +#include "tensorflow/core/framework/kernel_def.pb.h" #include "tensorflow/core/framework/op.h" #include "tensorflow/core/graph/graph.h" #include "tensorflow/core/lib/core/status.h" @@ -51,6 +52,10 @@ string TensorIdToString(const tf2xla::TensorId& id); // edges are considered. Status SetNodeShardingFromNeighbors(Node* n, bool out_edges); +// Add an allowed data type to the AttrConstraint with the given name. +void AddDtypeToKernalDefConstraint(StringPiece name, DataType dtype, + KernelDef* kdef); + } // namespace tensorflow #endif // TENSORFLOW_COMPILER_TF2XLA_TF2XLA_UTIL_H_ diff --git a/tensorflow/compiler/tf2xla/xla_cpu_backend.cc b/tensorflow/compiler/tf2xla/xla_cpu_backend.cc index 8286480e0e..ead229aacc 100644 --- a/tensorflow/compiler/tf2xla/xla_cpu_backend.cc +++ b/tensorflow/compiler/tf2xla/xla_cpu_backend.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include "tensorflow/compiler/tf2xla/tf2xla_util.h" #include "tensorflow/compiler/tf2xla/xla_op_registry.h" #include "tensorflow/core/framework/kernel_def.pb.h" @@ -30,6 +31,12 @@ bool CpuOpFilter(KernelDef* kdef) { DT_FLOAT); return true; } + if (kdef->op() == "Const") { + AddDtypeToKernalDefConstraint("dtype", DT_STRING, kdef); + } + if (kdef->op() == "Assert") { + AddDtypeToKernalDefConstraint("T", DT_STRING, kdef); + } return true; } diff --git a/tensorflow/compiler/tf2xla/xla_gpu_backend.cc b/tensorflow/compiler/tf2xla/xla_gpu_backend.cc index 8ca757e723..62168b6483 100644 --- a/tensorflow/compiler/tf2xla/xla_gpu_backend.cc +++ b/tensorflow/compiler/tf2xla/xla_gpu_backend.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include "tensorflow/compiler/tf2xla/tf2xla_util.h" #include "tensorflow/compiler/tf2xla/xla_op_registry.h" #include "tensorflow/core/framework/kernel_def.pb.h" @@ -25,6 +26,12 @@ bool GpuOpFilter(KernelDef* kdef) { kdef->op() == "RandomUniformInt" || kdef->op() == "TruncatedNormal") { return false; } + if (kdef->op() == "Const") { + AddDtypeToKernalDefConstraint("dtype", DT_STRING, kdef); + } + if (kdef->op() == "Assert") { + AddDtypeToKernalDefConstraint("T", DT_STRING, kdef); + } return true; } -- GitLab From d42e4bde7ace9bb757b0fdf0e2a48c97cabe938b Mon Sep 17 00:00:00 2001 From: Jiri Simsa Date: Thu, 12 Apr 2018 17:32:36 -0700 Subject: [PATCH 294/791] Porting tests for `rpc_op` to OS. PiperOrigin-RevId: 192698931 --- tensorflow/contrib/BUILD | 1 + tensorflow/contrib/__init__.py | 1 + tensorflow/contrib/cmake/tf_python.cmake | 3 +- tensorflow/contrib/rpc/BUILD | 16 + .../contrib/rpc/python/kernel_tests/BUILD | 76 ++++ .../rpc/python/kernel_tests/rpc_op_test.py | 71 ++++ .../python/kernel_tests/rpc_op_test_base.py | 337 ++++++++++++++++++ .../kernel_tests/rpc_op_test_servicer.py | 101 ++++++ .../python/kernel_tests/test_example.proto | 171 +++++++++ .../core/platform/default/build_config.bzl | 86 ++++- tensorflow/tools/pip_package/BUILD | 1 + tensorflow/workspace.bzl | 4 + 12 files changed, 864 insertions(+), 4 deletions(-) create mode 100644 tensorflow/contrib/rpc/python/kernel_tests/BUILD create mode 100644 tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test.py create mode 100644 tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_base.py create mode 100644 tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_servicer.py create mode 100644 tensorflow/contrib/rpc/python/kernel_tests/test_example.proto diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD index ae68f4aec4..7e47516550 100644 --- a/tensorflow/contrib/BUILD +++ b/tensorflow/contrib/BUILD @@ -87,6 +87,7 @@ py_library( "//tensorflow/contrib/remote_fused_graph/pylib:remote_fused_graph_ops_py", "//tensorflow/contrib/resampler:resampler_py", "//tensorflow/contrib/rnn:rnn_py", + "//tensorflow/contrib/rpc", "//tensorflow/contrib/saved_model:saved_model_py", "//tensorflow/contrib/seq2seq:seq2seq_py", "//tensorflow/contrib/signal:signal_py", diff --git a/tensorflow/contrib/__init__.py b/tensorflow/contrib/__init__.py index e27ece8fa5..36cc5144d0 100644 --- a/tensorflow/contrib/__init__.py +++ b/tensorflow/contrib/__init__.py @@ -71,6 +71,7 @@ from tensorflow.contrib import recurrent from tensorflow.contrib import reduce_slice_ops from tensorflow.contrib import resampler from tensorflow.contrib import rnn +from tensorflow.contrib import rpc from tensorflow.contrib import saved_model from tensorflow.contrib import seq2seq from tensorflow.contrib import signal diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake index 21f59d2563..f6aaf41f73 100755 --- a/tensorflow/contrib/cmake/tf_python.cmake +++ b/tensorflow/contrib/cmake/tf_python.cmake @@ -347,7 +347,8 @@ GENERATE_PYTHON_OP_LIB("random_ops") GENERATE_PYTHON_OP_LIB("remote_fused_graph_ops" DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/remote_fused_graph/pylib/python/ops/gen_remote_fused_graph_ops.py) GENERATE_PYTHON_OP_LIB("resource_variable_ops") -GENERATE_PYTHON_OP_LIB("rpc_ops") +GENERATE_PYTHON_OP_LIB("rpc_ops" + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/rpc/python/ops/gen_rpc_op.py) GENERATE_PYTHON_OP_LIB("script_ops") GENERATE_PYTHON_OP_LIB("sdca_ops") GENERATE_PYTHON_OP_LIB("set_ops") diff --git a/tensorflow/contrib/rpc/BUILD b/tensorflow/contrib/rpc/BUILD index 597f18c771..dbd311a276 100644 --- a/tensorflow/contrib/rpc/BUILD +++ b/tensorflow/contrib/rpc/BUILD @@ -4,6 +4,8 @@ licenses(["notice"]) # Apache 2.0 exports_files(["LICENSE"]) +load("//tensorflow/core:platform/default/build_config_root.bzl", "if_static") + py_library( name = "rpc", srcs = [ @@ -11,3 +13,17 @@ py_library( ], deps = ["//tensorflow/contrib/rpc/python/ops:rpc_op_py"], ) + +py_library( + name = "rpc_pip", + data = if_static( + [], + otherwise = ["//tensorflow/contrib/rpc/python/kernel_tests:libtestexample.so"], + ), + deps = [ + ":rpc", + "//tensorflow/contrib/rpc/python/kernel_tests:py_test_deps", + "//tensorflow/contrib/rpc/python/kernel_tests:rpc_op_test_base", + "//tensorflow/contrib/rpc/python/kernel_tests:rpc_op_test_servicer", + ], +) diff --git a/tensorflow/contrib/rpc/python/kernel_tests/BUILD b/tensorflow/contrib/rpc/python/kernel_tests/BUILD new file mode 100644 index 0000000000..08ec1e61a4 --- /dev/null +++ b/tensorflow/contrib/rpc/python/kernel_tests/BUILD @@ -0,0 +1,76 @@ +# TODO(b/76425722): Port everything in here to OS (currently excluded). + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # Apache 2.0 + +exports_files(["LICENSE"]) + +# Placeholder for loading internal BUILD rule. +load("//tensorflow:tensorflow.bzl", "tf_py_test") +load("//tensorflow:tensorflow.bzl", "tf_cc_shared_object") +load("//tensorflow/core:platform/default/build_config_root.bzl", "if_static") +load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library") + +tf_proto_library( + name = "test_example_proto", + srcs = ["test_example.proto"], + has_services = 1, + cc_api_version = 2, + protodeps = ["//tensorflow/core:protos_all"], +) + +py_library( + name = "py_test_deps", + deps = [":test_example_proto_py"], +) + +py_library( + name = "rpc_op_test_base", + srcs = ["rpc_op_test_base.py"], + deps = [ + ":test_example_proto_py", + "//tensorflow/contrib/proto", + "//tensorflow/contrib/rpc", + "//tensorflow/core:protos_all_py", + "//tensorflow/python:dtypes", + "//tensorflow/python:errors", + "//third_party/py/numpy", + ], +) + +py_library( + name = "rpc_op_test_servicer", + srcs = ["rpc_op_test_servicer.py"], + deps = [ + ":py_test_deps", + ":rpc_op_test_base", + "//tensorflow/core:protos_all_py", + "//third_party/py/numpy", + ], +) + +tf_cc_shared_object( + name = "libtestexample.so", + linkstatic = 1, + deps = [ + ":test_example_proto_cc", + ], +) + +tf_py_test( + name = "rpc_op_test", + size = "small", + srcs = ["rpc_op_test.py"], + additional_deps = [ + ":py_test_deps", + ":rpc_op_test_base", + ":rpc_op_test_servicer", + "//tensorflow/core:protos_all_py", + "//tensorflow/python:client_testlib", + ], + data = if_static( + [], + otherwise = [":libtestexample.so"], + ), +) diff --git a/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test.py b/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test.py new file mode 100644 index 0000000000..e2e0dbc7a2 --- /dev/null +++ b/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test.py @@ -0,0 +1,71 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +"""Tests for RpcOp.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import ctypes as ct +import os + +import grpc +from grpc.framework.foundation import logging_pool +import portpicker + +from tensorflow.contrib.rpc.python.kernel_tests import rpc_op_test_base +from tensorflow.contrib.rpc.python.kernel_tests import rpc_op_test_servicer +from tensorflow.contrib.rpc.python.kernel_tests import test_example_pb2_grpc +from tensorflow.python.platform import test + + +class RpcOpTest(test.TestCase, rpc_op_test_base.RpcOpTestBase): + _protocol = 'grpc' + + invalid_method_string = 'Method not found' + + def __init__(self, methodName='runTest'): # pylint: disable=invalid-name + super(RpcOpTest, self).__init__(methodName) + lib = os.path.join(os.path.dirname(__file__), 'libtestexample.so') + if os.path.isfile(lib): + ct.cdll.LoadLibrary(lib) + + def get_method_name(self, suffix): + return '/tensorflow.contrib.rpc.TestCaseService/%s' % suffix + + def setUp(self): + super(RpcOpTest, self).setUp() + + service_port = portpicker.pick_unused_port() + + server = grpc.server(logging_pool.pool(max_workers=25)) + servicer = rpc_op_test_servicer.RpcOpTestServicer() + test_example_pb2_grpc.add_TestCaseServiceServicer_to_server( + servicer, server) + self._address = 'localhost:%d' % service_port + server.add_insecure_port(self._address) + server.start() + self._server = server + + def tearDown(self): + # TODO(ebrevdo): Figure out why this sometimes times out. + # self._service.ExitLoop() + # self._service_thread.join() + # self._server.stop() + super(RpcOpTest, self).tearDown() + + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_base.py b/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_base.py new file mode 100644 index 0000000000..aa03a103ed --- /dev/null +++ b/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_base.py @@ -0,0 +1,337 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +"""Base class for RpcOp tests.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import itertools + +import numpy as np + +from tensorflow.contrib.proto import decode_proto +from tensorflow.contrib.proto import encode_proto +from tensorflow.contrib.rpc import rpc +from tensorflow.contrib.rpc import try_rpc +from tensorflow.contrib.rpc.python.kernel_tests import test_example_pb2 +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors + +__all__ = ['I_WARNED_YOU', 'RpcOpTestBase'] + +I_WARNED_YOU = 'I warned you!' + + +class RpcOpTestBase(object): + # pylint: disable=missing-docstring,invalid-name + """Base class for RpcOp tests.""" + + def get_method_name(self, suffix): + raise NotImplementedError + + def rpc(self, *args, **kwargs): + return rpc(*args, protocol=self._protocol, **kwargs) + + def try_rpc(self, *args, **kwargs): + return try_rpc(*args, protocol=self._protocol, **kwargs) + + def testScalarHostPortRpc(self): + with self.test_session() as sess: + request_tensors = ( + test_example_pb2.TestCase(shape=[1, 2, 3]).SerializeToString()) + response_tensors = self.rpc( + method=self.get_method_name('IncrementTestShapes'), + address=self._address, + request=request_tensors) + self.assertEqual(response_tensors.shape, ()) + response_values = sess.run(response_tensors) + response_message = test_example_pb2.TestCase() + self.assertTrue(response_message.ParseFromString(response_values)) + self.assertAllEqual([2, 3, 4], response_message.shape) + + def testScalarHostPortTryRpc(self): + with self.test_session() as sess: + request_tensors = ( + test_example_pb2.TestCase(shape=[1, 2, 3]).SerializeToString()) + response_tensors, status_code, status_message = self.try_rpc( + method=self.get_method_name('IncrementTestShapes'), + address=self._address, + request=request_tensors) + self.assertEqual(status_code.shape, ()) + self.assertEqual(status_message.shape, ()) + self.assertEqual(response_tensors.shape, ()) + response_values, status_code_values, status_message_values = ( + sess.run((response_tensors, status_code, status_message))) + response_message = test_example_pb2.TestCase() + self.assertTrue(response_message.ParseFromString(response_values)) + self.assertAllEqual([2, 3, 4], response_message.shape) + # For the base Rpc op, don't expect to get error status back. + self.assertEqual(errors.OK, status_code_values) + self.assertEqual(b'', status_message_values) + + def testEmptyHostPortRpc(self): + with self.test_session() as sess: + request_tensors = [] + response_tensors = self.rpc( + method=self.get_method_name('IncrementTestShapes'), + address=self._address, + request=request_tensors) + self.assertAllEqual(response_tensors.shape, [0]) + response_values = sess.run(response_tensors) + self.assertAllEqual(response_values.shape, [0]) + + def testInvalidAddresses(self): + with self.test_session() as sess: + with self.assertRaisesOpError(self.invalid_method_string): + sess.run( + self.rpc( + method='/InvalidService.IncrementTestShapes', + address=self._address, + request='')) + + with self.assertRaisesOpError(self.invalid_method_string): + sess.run( + self.rpc( + method=self.get_method_name('InvalidMethodName'), + address=self._address, + request='')) + + # This also covers the case of address='' + # and address='localhost:293874293874' + with self.assertRaises(errors.UnavailableError): + sess.run( + self.rpc( + method=self.get_method_name('IncrementTestShapes'), + address='unix:/tmp/this_unix_socket_doesnt_exist_97820348!!@', + request='')) + + # Test invalid method with the TryRpc op + _, status_code_value, status_message_value = sess.run( + self.try_rpc( + method=self.get_method_name('InvalidMethodName'), + address=self._address, + request='')) + self.assertEqual(errors.UNIMPLEMENTED, status_code_value) + self.assertTrue( + self.invalid_method_string in status_message_value.decode('ascii')) + + def testAlwaysFailingMethod(self): + with self.test_session() as sess: + response_tensors = self.rpc( + method=self.get_method_name('AlwaysFailWithInvalidArgument'), + address=self._address, + request='') + self.assertEqual(response_tensors.shape, ()) + with self.assertRaisesOpError(I_WARNED_YOU): + sess.run(response_tensors) + + def testSometimesFailingMethodWithManyRequests(self): + with self.test_session() as sess: + # Fail hard by default. + response_tensors = self.rpc( + method=self.get_method_name('SometimesFailWithInvalidArgument'), + address=self._address, + request=[''] * 20) + self.assertEqual(response_tensors.shape, (20,)) + with self.assertRaisesOpError(I_WARNED_YOU): + sess.run(response_tensors) + + # Don't fail hard, use TryRpc - return the failing status instead. + response_tensors, status_code, status_message = self.try_rpc( + method=self.get_method_name('SometimesFailWithInvalidArgument'), + address=self._address, + request=[''] * 20) + self.assertEqual(response_tensors.shape, (20,)) + self.assertEqual(status_code.shape, (20,)) + self.assertEqual(status_message.shape, (20,)) + status_code_values, status_message_values = sess.run((status_code, + status_message)) + self.assertTrue([ + x in (errors.OK, errors.INVALID_ARGUMENT) for x in status_code_values + ]) + expected_message_values = np.where( + status_code_values == errors.INVALID_ARGUMENT, + I_WARNED_YOU.encode('ascii'), b'') + self.assertAllEqual(expected_message_values, status_message_values) + + def testVecHostPortRpc(self): + with self.test_session() as sess: + request_tensors = [ + test_example_pb2.TestCase( + shape=[i, i + 1, i + 2]).SerializeToString() for i in range(20) + ] + response_tensors = self.rpc( + method=self.get_method_name('IncrementTestShapes'), + address=self._address, + request=request_tensors) + self.assertEqual(response_tensors.shape, (20,)) + response_values = sess.run(response_tensors) + self.assertEqual(response_values.shape, (20,)) + for i in range(20): + response_message = test_example_pb2.TestCase() + self.assertTrue(response_message.ParseFromString(response_values[i])) + self.assertAllEqual([i + 1, i + 2, i + 3], response_message.shape) + + def testVecHostPortManyParallelRpcs(self): + with self.test_session() as sess: + request_tensors = [ + test_example_pb2.TestCase( + shape=[i, i + 1, i + 2]).SerializeToString() for i in range(20) + ] + many_response_tensors = [ + self.rpc( + method=self.get_method_name('IncrementTestShapes'), + address=self._address, + request=request_tensors) for _ in range(10) + ] + # Launch parallel 10 calls to the RpcOp, each containing + # 20 rpc requests. + many_response_values = sess.run(many_response_tensors) + self.assertEqual(10, len(many_response_values)) + for response_values in many_response_values: + self.assertEqual(response_values.shape, (20,)) + for i in range(20): + response_message = test_example_pb2.TestCase() + self.assertTrue(response_message.ParseFromString(response_values[i])) + self.assertAllEqual([i + 1, i + 2, i + 3], response_message.shape) + + def testVecHostPortRpcUsingEncodeAndDecodeProto(self): + with self.test_session() as sess: + request_tensors = encode_proto( + message_type='tensorflow.contrib.rpc.TestCase', + field_names=['shape'], + sizes=[[3]] * 20, + values=[ + [[i, i + 1, i + 2] for i in range(20)], + ]) + response_tensor_strings = self.rpc( + method=self.get_method_name('IncrementTestShapes'), + address=self._address, + request=request_tensors) + _, (response_shape,) = decode_proto( + bytes=response_tensor_strings, + message_type='tensorflow.contrib.rpc.TestCase', + field_names=['shape'], + output_types=[dtypes.int32]) + response_shape_values = sess.run(response_shape) + self.assertAllEqual([[i + 1, i + 2, i + 3] + for i in range(20)], response_shape_values) + + def testVecHostPortRpcCancelsUponSessionTimeOutWhenSleepingForever(self): + with self.test_session() as sess: + request_tensors = [''] * 25 # This will launch 25 RPC requests. + response_tensors = self.rpc( + method=self.get_method_name('SleepForever'), + address=self._address, + request=request_tensors) + for timeout_ms in [1, 500, 1000]: + options = config_pb2.RunOptions(timeout_in_ms=timeout_ms) + with self.assertRaises((errors.UnavailableError, + errors.DeadlineExceededError)): + sess.run(response_tensors, options=options) + + def testVecHostPortRpcCancelsUponConfiguredTimeOutWhenSleepingForever(self): + with self.test_session() as sess: + request_tensors = [''] * 25 # This will launch 25 RPC requests. + response_tensors = self.rpc( + method=self.get_method_name('SleepForever'), + address=self._address, + timeout_in_ms=1000, + request=request_tensors) + with self.assertRaises(errors.DeadlineExceededError): + sess.run(response_tensors) + + def testTryRpcPropagatesDeadlineErrorWithSometimesTimingOutRequests(self): + with self.test_session() as sess: + response_tensors, status_code, status_message = self.try_rpc( + method=self.get_method_name('SometimesSleepForever'), + timeout_in_ms=1000, + address=self._address, + request=[''] * 20) + self.assertEqual(response_tensors.shape, (20,)) + self.assertEqual(status_code.shape, (20,)) + self.assertEqual(status_message.shape, (20,)) + status_code_values = sess.run(status_code) + self.assertTrue([ + x in (errors.OK, errors.DEADLINE_EXCEEDED) for x in status_code_values + ]) + + def testTryRpcWithMultipleAddressesSingleRequest(self): + flatten = lambda x: list(itertools.chain.from_iterable(x)) + with self.test_session() as sess: + addresses = flatten([[ + self._address, 'unix:/tmp/this_unix_socket_doesnt_exist_97820348!!@' + ] for _ in range(10)]) + request = test_example_pb2.TestCase(shape=[0, 1, 2]).SerializeToString() + response_tensors, status_code, _ = self.try_rpc( + method=self.get_method_name('IncrementTestShapes'), + address=addresses, + request=request) + response_tensors_values, status_code_values = sess.run((response_tensors, + status_code)) + self.assertAllEqual( + flatten([errors.OK, errors.UNAVAILABLE] for _ in range(10)), + status_code_values) + for i in range(10): + self.assertTrue(response_tensors_values[2 * i]) + self.assertFalse(response_tensors_values[2 * i + 1]) + + def testTryRpcWithMultipleMethodsSingleRequest(self): + flatten = lambda x: list(itertools.chain.from_iterable(x)) + with self.test_session() as sess: + methods = flatten( + [[self.get_method_name('IncrementTestShapes'), 'InvalidMethodName'] + for _ in range(10)]) + request = test_example_pb2.TestCase(shape=[0, 1, 2]).SerializeToString() + response_tensors, status_code, _ = self.try_rpc( + method=methods, address=self._address, request=request) + response_tensors_values, status_code_values = sess.run((response_tensors, + status_code)) + self.assertAllEqual( + flatten([errors.OK, errors.UNIMPLEMENTED] for _ in range(10)), + status_code_values) + for i in range(10): + self.assertTrue(response_tensors_values[2 * i]) + self.assertFalse(response_tensors_values[2 * i + 1]) + + def testTryRpcWithMultipleAddressesAndRequests(self): + flatten = lambda x: list(itertools.chain.from_iterable(x)) + with self.test_session() as sess: + addresses = flatten([[ + self._address, 'unix:/tmp/this_unix_socket_doesnt_exist_97820348!!@' + ] for _ in range(10)]) + requests = [ + test_example_pb2.TestCase( + shape=[i, i + 1, i + 2]).SerializeToString() for i in range(20) + ] + response_tensors, status_code, _ = self.try_rpc( + method=self.get_method_name('IncrementTestShapes'), + address=addresses, + request=requests) + response_tensors_values, status_code_values = sess.run((response_tensors, + status_code)) + self.assertAllEqual( + flatten([errors.OK, errors.UNAVAILABLE] for _ in range(10)), + status_code_values) + for i in range(20): + if i % 2 == 1: + self.assertFalse(response_tensors_values[i]) + else: + response_message = test_example_pb2.TestCase() + self.assertTrue( + response_message.ParseFromString(response_tensors_values[i])) + self.assertAllEqual([i + 1, i + 2, i + 3], response_message.shape) diff --git a/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_servicer.py b/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_servicer.py new file mode 100644 index 0000000000..7cbd636cb1 --- /dev/null +++ b/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_servicer.py @@ -0,0 +1,101 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +"""Test servicer for RpcOp tests.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import random +import time + +import grpc + +from tensorflow.contrib.rpc.python.kernel_tests import rpc_op_test_base +from tensorflow.contrib.rpc.python.kernel_tests import test_example_pb2_grpc + + +class RpcOpTestServicer(test_example_pb2_grpc.TestCaseServiceServicer): + """Test servicer for RpcOp tests.""" + + def IncrementTestShapes(self, request, context): + """Increment the entries in the shape attribute of request. + + Args: + request: input TestCase. + context: the rpc context. + + Returns: + output TestCase. + """ + for i in range(len(request.shape)): + request.shape[i] += 1 + return request + + def AlwaysFailWithInvalidArgument(self, request, context): + """Always fails with an InvalidArgument status. + + Args: + request: input TestCase. + context: the rpc context. + + Returns: + output TestCase. + """ + del request + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + context.set_details(rpc_op_test_base.I_WARNED_YOU) + + def SometimesFailWithInvalidArgument(self, request, context): + """Sometimes fails with an InvalidArgument status. + + Args: + request: input TestCase. + context: the rpc context. + + Returns: + output TestCase. + """ + if random.randint(0, 1) == 1: + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + context.set_details(rpc_op_test_base.I_WARNED_YOU) + return request + + def SleepForever(self, request, context): + """Sleeps forever. + + Args: + request: input TestCase. + context: the rpc context. + + Returns: + output TestCase. + """ + # TODO(ebrevdo): Make this async wait like the stubby version. + time.sleep(5) + + def SometimesSleepForever(self, request, context): + """Sometimes sleeps forever. + + Args: + request: input TestCase. + context: the rpc context. + + Returns: + output TestCase. + """ + if random.randint(0, 1) == 1: + time.sleep(5) + return request diff --git a/tensorflow/contrib/rpc/python/kernel_tests/test_example.proto b/tensorflow/contrib/rpc/python/kernel_tests/test_example.proto new file mode 100644 index 0000000000..96f4550f62 --- /dev/null +++ b/tensorflow/contrib/rpc/python/kernel_tests/test_example.proto @@ -0,0 +1,171 @@ +// Test description and protos to work with it. +// +// Many of the protos in this file are for unit tests that haven't been written yet. + +syntax = "proto2"; + +import "tensorflow/core/framework/types.proto"; + +package tensorflow.contrib.rpc; + +// A TestCase holds a proto and a bunch of assertions +// about how it should decode. +message TestCase { + // A batch of primitives to be serialized and decoded. + repeated RepeatedPrimitiveValue primitive = 1; + // The shape of the batch. + repeated int32 shape = 2; + // Expected sizes for each field. + repeated int32 sizes = 3; + // Expected values for each field. + repeated FieldSpec field = 4; +}; + +service TestCaseService { + // Copy input, and increment each entry in 'shape' by 1. + rpc IncrementTestShapes(TestCase) returns (TestCase) { + } + + // Sleep forever. + rpc SleepForever(TestCase) returns (TestCase) { + } + + // Sleep forever 50% of the time, return immediately the other 50%. + rpc SometimesSleepForever(TestCase) returns (TestCase) { + } + + // Always fails with InvalidArgument. + rpc AlwaysFailWithInvalidArgument(TestCase) returns (TestCase) { + } + + // Fails with InvalidArgument 50% of the time. + rpc SometimesFailWithInvalidArgument(TestCase) returns (TestCase) { + } +}; + +// FieldSpec describes the expected output for a single field. +message FieldSpec { + optional string name = 1; + optional tensorflow.DataType dtype = 2; + optional RepeatedPrimitiveValue expected = 3; +}; + +message TestValue { + optional PrimitiveValue primitive_value = 1; + optional EnumValue enum_value = 2; + optional MessageValue message_value = 3; + optional RepeatedMessageValue repeated_message_value = 4; + optional RepeatedPrimitiveValue repeated_primitive_value = 6; +} + +message PrimitiveValue { + optional double double_value = 1; + optional float float_value = 2; + optional int64 int64_value = 3; + optional uint64 uint64_value = 4; + optional int32 int32_value = 5; + optional fixed64 fixed64_value = 6; + optional fixed32 fixed32_value = 7; + optional bool bool_value = 8; + optional string string_value = 9; + optional bytes bytes_value = 12; + optional uint32 uint32_value = 13; + optional sfixed32 sfixed32_value = 15; + optional sfixed64 sfixed64_value = 16; + optional sint32 sint32_value = 17; + optional sint64 sint64_value = 18; +} + +// NOTE: This definition must be kept in sync with PackedPrimitiveValue. +message RepeatedPrimitiveValue { + repeated double double_value = 1; + repeated float float_value = 2; + repeated int64 int64_value = 3; + repeated uint64 uint64_value = 4; + repeated int32 int32_value = 5; + repeated fixed64 fixed64_value = 6; + repeated fixed32 fixed32_value = 7; + repeated bool bool_value = 8; + repeated string string_value = 9; + repeated bytes bytes_value = 12; + repeated uint32 uint32_value = 13; + repeated sfixed32 sfixed32_value = 15; + repeated sfixed64 sfixed64_value = 16; + repeated sint32 sint32_value = 17; + repeated sint64 sint64_value = 18; + repeated PrimitiveValue message_value = 19; +} + +// A PackedPrimitiveValue looks exactly the same as a RepeatedPrimitiveValue +// in the text format, but the binary serializion is different. +// We test the packed representations by loading the same test cases +// using this definition instead of RepeatedPrimitiveValue. +// NOTE: This definition must be kept in sync with RepeatedPrimitiveValue +// in every way except the packed=true declaration. +message PackedPrimitiveValue { + repeated double double_value = 1 [packed = true]; + repeated float float_value = 2 [packed = true]; + repeated int64 int64_value = 3 [packed = true]; + repeated uint64 uint64_value = 4 [packed = true]; + repeated int32 int32_value = 5 [packed = true]; + repeated fixed64 fixed64_value = 6 [packed = true]; + repeated fixed32 fixed32_value = 7 [packed = true]; + repeated bool bool_value = 8 [packed = true]; + repeated string string_value = 9; + repeated bytes bytes_value = 12; + repeated uint32 uint32_value = 13 [packed = true]; + repeated sfixed32 sfixed32_value = 15 [packed = true]; + repeated sfixed64 sfixed64_value = 16 [packed = true]; + repeated sint32 sint32_value = 17 [packed = true]; + repeated sint64 sint64_value = 18 [packed = true]; + repeated PrimitiveValue message_value = 19; +} + +message EnumValue { + enum Color { + RED = 0; + ORANGE = 1; + YELLOW = 2; + GREEN = 3; + BLUE = 4; + INDIGO = 5; + VIOLET = 6; + }; + optional Color enum_value = 14; + repeated Color repeated_enum_value = 15; +} + + +message InnerMessageValue { + optional float float_value = 2; + repeated bytes bytes_values = 8; +} + +message MiddleMessageValue { + repeated int32 int32_values = 5; + optional InnerMessageValue message_value = 11; + optional uint32 uint32_value = 13; +} + +message MessageValue { + optional double double_value = 1; + optional MiddleMessageValue message_value = 11; +} + +message RepeatedMessageValue { + message NestedMessageValue { + optional float float_value = 2; + repeated bytes bytes_values = 8; + } + + repeated NestedMessageValue message_values = 11; +} + +// Message containing fields with field numbers higher than any field above. An +// instance of this message is prepended to each binary message in the test to +// exercise the code path that handles fields encoded out of order of field +// number. +message ExtraFields { + optional string string_value = 1776; + optional bool bool_value = 1777; +} diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl index 4cfa25bf66..44356e3438 100644 --- a/tensorflow/core/platform/default/build_config.bzl +++ b/tensorflow/core/platform/default/build_config.bzl @@ -1,7 +1,6 @@ # Platform-specific build configurations. load("@protobuf_archive//:protobuf.bzl", "proto_gen") -load("@protobuf_archive//:protobuf.bzl", "py_proto_library") load("//tensorflow:tensorflow.bzl", "if_not_mobile") load("//tensorflow:tensorflow.bzl", "if_windows") load("//tensorflow:tensorflow.bzl", "if_not_windows") @@ -110,6 +109,12 @@ def _proto_cc_srcs(srcs, use_grpc_plugin=False): ret += [s[:-len(".proto")] + ".grpc.pb.cc" for s in srcs] return ret +def _proto_py_outs(srcs, use_grpc_plugin=False): + ret = [s[:-len(".proto")] + "_pb2.py" for s in srcs] + if use_grpc_plugin: + ret += [s[:-len(".proto")] + "_pb2_grpc.py" for s in srcs] + return ret + # Re-defined protocol buffer rule to allow building "header only" protocol # buffers, to avoid duplicate registrations. Also allows non-iterable cc_libs # containing select() statements. @@ -217,6 +222,80 @@ def cc_proto_library( hdrs=gen_hdrs, **kargs) +# Re-defined protocol buffer rule to bring in the change introduced in commit +# https://github.com/google/protobuf/commit/294b5758c373cbab4b72f35f4cb62dc1d8332b68 +# which was not part of a stable protobuf release in 04/2018. +# TODO(jsimsa): Remove this once the protobuf dependency version is updated +# to include the above commit. +def py_proto_library( + name, + srcs=[], + deps=[], + py_libs=[], + py_extra_srcs=[], + include=None, + default_runtime="@protobuf_archive//:protobuf_python", + protoc="@protobuf_archive//:protoc", + use_grpc_plugin=False, + **kargs): + """Bazel rule to create a Python protobuf library from proto source files + + NOTE: the rule is only an internal workaround to generate protos. The + interface may change and the rule may be removed when bazel has introduced + the native rule. + + Args: + name: the name of the py_proto_library. + srcs: the .proto files of the py_proto_library. + deps: a list of dependency labels; must be py_proto_library. + py_libs: a list of other py_library targets depended by the generated + py_library. + py_extra_srcs: extra source files that will be added to the output + py_library. This attribute is used for internal bootstrapping. + include: a string indicating the include path of the .proto files. + default_runtime: the implicitly default runtime which will be depended on by + the generated py_library target. + protoc: the label of the protocol compiler to generate the sources. + use_grpc_plugin: a flag to indicate whether to call the Python C++ plugin + when processing the proto files. + **kargs: other keyword arguments that are passed to cc_library. + """ + outs = _proto_py_outs(srcs, use_grpc_plugin) + + includes = [] + if include != None: + includes = [include] + + grpc_python_plugin = None + if use_grpc_plugin: + grpc_python_plugin = "//external:grpc_python_plugin" + # Note: Generated grpc code depends on Python grpc module. This dependency + # is not explicitly listed in py_libs. Instead, host system is assumed to + # have grpc installed. + + proto_gen( + name=name + "_genproto", + srcs=srcs, + deps=[s + "_genproto" for s in deps], + includes=includes, + protoc=protoc, + gen_py=1, + outs=outs, + visibility=["//visibility:public"], + plugin=grpc_python_plugin, + plugin_language="grpc" + ) + + if default_runtime and not default_runtime in py_libs + deps: + py_libs = py_libs + [default_runtime] + + native.py_library( + name=name, + srcs=outs+py_extra_srcs, + deps=py_libs+deps, + imports=includes, + **kargs) + def tf_proto_library_cc(name, srcs = [], has_services = None, protodeps = [], visibility = [], testonly = 0, @@ -261,8 +340,7 @@ def tf_proto_library_cc(name, srcs = [], has_services = None, ) def tf_proto_library_py(name, srcs=[], protodeps=[], deps=[], visibility=[], - testonly=0, - srcs_version="PY2AND3"): + testonly=0, srcs_version="PY2AND3", use_grpc_plugin=False): py_proto_library( name = name + "_py", srcs = srcs, @@ -272,6 +350,7 @@ def tf_proto_library_py(name, srcs=[], protodeps=[], deps=[], visibility=[], default_runtime = "@protobuf_archive//:protobuf_python", visibility = visibility, testonly = testonly, + use_grpc_plugin = use_grpc_plugin, ) def tf_jspb_proto_library(**kwargs): @@ -310,6 +389,7 @@ def tf_proto_library(name, srcs = [], has_services = None, srcs_version = "PY2AND3", testonly = testonly, visibility = visibility, + use_grpc_plugin = has_services, ) def tf_additional_lib_hdrs(exclude = []): diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD index a0bae23a7c..2ef105755f 100644 --- a/tensorflow/tools/pip_package/BUILD +++ b/tensorflow/tools/pip_package/BUILD @@ -76,6 +76,7 @@ COMMON_PIP_DEPS = [ "//tensorflow/contrib/predictor:predictor_pip", "//tensorflow/contrib/proto:proto_pip", "//tensorflow/contrib/receptive_field:receptive_field_pip", + "//tensorflow/contrib/rpc:rpc_pip", "//tensorflow/contrib/session_bundle:session_bundle_pip", "//tensorflow/contrib/signal:signal_py", "//tensorflow/contrib/signal:test_util", diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 72f446d359..dee2fcd0e1 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -763,6 +763,10 @@ def tf_workspace(path_prefix="", tf_repo_name=""): name = "grpc_cpp_plugin", actual = "@grpc//:grpc_cpp_plugin", ) + native.bind( + name = "grpc_python_plugin", + actual = "@grpc//:grpc_python_plugin", + ) # gRPC has three empty C++ functions which it wants the user to define # at build time. https://github.com/grpc/grpc/issues/13590 -- GitLab From 457e8b3a78d4b31de4113168422786412f8771fc Mon Sep 17 00:00:00 2001 From: James Qin Date: Thu, 12 Apr 2018 17:35:56 -0700 Subject: [PATCH 295/791] Print error msg in CUDATimer.Init() when CreateEvent() is not ok(). PiperOrigin-RevId: 192699277 --- tensorflow/stream_executor/cuda/cuda_timer.cc | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tensorflow/stream_executor/cuda/cuda_timer.cc b/tensorflow/stream_executor/cuda/cuda_timer.cc index 4bd5503348..7d78601fb9 100644 --- a/tensorflow/stream_executor/cuda/cuda_timer.cc +++ b/tensorflow/stream_executor/cuda/cuda_timer.cc @@ -27,16 +27,18 @@ namespace cuda { bool CUDATimer::Init() { CHECK(start_event_ == nullptr && stop_event_ == nullptr); CudaContext* context = parent_->cuda_context(); - if (!CUDADriver::CreateEvent(context, &start_event_, - CUDADriver::EventFlags::kDefault) - .ok()) { + port::Status status = CUDADriver::CreateEvent( + context, &start_event_, CUDADriver::EventFlags::kDefault); + if (!status.ok()) { + LOG(ERROR) << status; return false; } - if (!CUDADriver::CreateEvent(context, &stop_event_, - CUDADriver::EventFlags::kDefault) - .ok()) { - port::Status status = CUDADriver::DestroyEvent(context, &start_event_); + status = CUDADriver::CreateEvent(context, &stop_event_, + CUDADriver::EventFlags::kDefault); + if (!status.ok()) { + LOG(ERROR) << status; + status = CUDADriver::DestroyEvent(context, &start_event_); if (!status.ok()) { LOG(ERROR) << status; } -- GitLab From 5a53c9b54d8781032ebf2cf26f93da3b2a33d1e4 Mon Sep 17 00:00:00 2001 From: Jiri Simsa Date: Thu, 12 Apr 2018 18:02:58 -0700 Subject: [PATCH 296/791] Reintroducing support for constants as outputs of tf.data.map(). This fixes a regression introduced by cl/176147440. PiperOrigin-RevId: 192702279 --- .../data/kernel_tests/map_dataset_op_test.py | 14 +++++++ tensorflow/python/data/ops/dataset_ops.py | 42 +++++++++---------- 2 files changed, 35 insertions(+), 21 deletions(-) diff --git a/tensorflow/python/data/kernel_tests/map_dataset_op_test.py b/tensorflow/python/data/kernel_tests/map_dataset_op_test.py index 0791c614fa..1ad0b9de5e 100644 --- a/tensorflow/python/data/kernel_tests/map_dataset_op_test.py +++ b/tensorflow/python/data/kernel_tests/map_dataset_op_test.py @@ -624,6 +624,20 @@ class MapDatasetTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) + def testConstantOutput(self): + iterator = ( + dataset_ops.Dataset.range(10).map(lambda x: [x, "hello", 10]) + .make_initializable_iterator()) + init_op = iterator.initializer + get_next = iterator.get_next() + + with self.test_session() as sess: + sess.run(init_op) + for i in range(10): + self.assertEqual((i, b"hello", 10), sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + class MapDatasetBenchmark(test.Benchmark): diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py index c28de3d054..406f172e59 100644 --- a/tensorflow/python/data/ops/dataset_ops.py +++ b/tensorflow/python/data/ops/dataset_ops.py @@ -1155,10 +1155,12 @@ class _GeneratorDataset(Dataset): if isinstance(ret, list): ret = tuple(ret) - # Convert any `SparseTensorValue`s to `SparseTensor`s. + # Convert any `SparseTensorValue`s to `SparseTensor`s and all other + # values to tensors. ret = nest.pack_sequence_as(ret, [ sparse_tensor_lib.SparseTensor.from_value(t) - if sparse_tensor_lib.is_sparse(t) else t for t in nest.flatten(ret) + if sparse_tensor_lib.is_sparse(t) else ops.convert_to_tensor(t) + for t in nest.flatten(ret) ]) self._state_classes = sparse.get_classes(ret) @@ -1167,11 +1169,9 @@ class _GeneratorDataset(Dataset): self._state_types = nest.pack_sequence_as( ret, [t.dtype for t in nest.flatten(ret)]) - # Serialize any sparse tensors and convert result to tensors. - ret = nest.pack_sequence_as(ret, [ - ops.convert_to_tensor(t) - for t in nest.flatten(sparse.serialize_sparse_tensors(ret)) - ]) + # Serialize any sparse tensors. + ret = nest.pack_sequence_as( + ret, [t for t in nest.flatten(sparse.serialize_sparse_tensors(ret))]) return nest.flatten(ret) self._init_func = tf_init_func @@ -1214,10 +1214,12 @@ class _GeneratorDataset(Dataset): if isinstance(ret, list): ret = tuple(ret) - # Convert any `SparseTensorValue`s to `SparseTensor`s. + # Convert any `SparseTensorValue`s to `SparseTensor`s and all other + # values to tensors. ret = nest.pack_sequence_as(ret, [ sparse_tensor_lib.SparseTensor.from_value(t) - if sparse_tensor_lib.is_sparse(t) else t for t in nest.flatten(ret) + if sparse_tensor_lib.is_sparse(t) else ops.convert_to_tensor(t) + for t in nest.flatten(ret) ]) self._output_classes = sparse.get_classes(ret) @@ -1226,11 +1228,9 @@ class _GeneratorDataset(Dataset): self._output_types = nest.pack_sequence_as( ret, [t.dtype for t in nest.flatten(ret)]) - # Serialize any sparse tensors and convert result to tensors. - ret = nest.pack_sequence_as(ret, [ - ops.convert_to_tensor(t) - for t in nest.flatten(sparse.serialize_sparse_tensors(ret)) - ]) + # Serialize any sparse tensors. + ret = nest.pack_sequence_as( + ret, [t for t in nest.flatten(sparse.serialize_sparse_tensors(ret))]) return nest.flatten(ret) self._next_func = tf_next_func @@ -1816,10 +1816,12 @@ class MapDataset(Dataset): if isinstance(ret, list): ret = tuple(ret) - # Convert any `SparseTensorValue`s to `SparseTensor`s. + # Convert any `SparseTensorValue`s to `SparseTensor`s and all other + # values to tensors. ret = nest.pack_sequence_as(ret, [ sparse_tensor_lib.SparseTensor.from_value(t) - if sparse_tensor_lib.is_sparse(t) else t for t in nest.flatten(ret) + if sparse_tensor_lib.is_sparse(t) else ops.convert_to_tensor(t) + for t in nest.flatten(ret) ]) self._output_classes = sparse.get_classes(ret) @@ -1828,11 +1830,9 @@ class MapDataset(Dataset): self._output_types = nest.pack_sequence_as( ret, [t.dtype for t in nest.flatten(ret)]) - # Serialize any sparse tensors and convert result to tensors. - ret = nest.pack_sequence_as(ret, [ - ops.convert_to_tensor(t) - for t in nest.flatten(sparse.serialize_sparse_tensors(ret)) - ]) + # Serialize any sparse tensors. + ret = nest.pack_sequence_as( + ret, [t for t in nest.flatten(sparse.serialize_sparse_tensors(ret))]) return nest.flatten(ret) self._map_func = tf_map_func -- GitLab From e489b600f388ae345387881a85368af3cd373ba2 Mon Sep 17 00:00:00 2001 From: Sami Kama Date: Thu, 12 Apr 2018 18:07:50 -0700 Subject: [PATCH 297/791] Replace tuple for version info with a class in DnnSupportr::GetVersion() (#18434) * Replace tuple for version info with a class * Removed clang-format modifications on non-edited code * Update dnn.h Update the comment as per request of reviewer --- .../gpu/cudnn_convolution_algorithm_picker.cc | 4 ++-- tensorflow/stream_executor/cuda/cuda_dnn.cc | 7 ++++--- tensorflow/stream_executor/cuda/cuda_dnn.h | 2 +- tensorflow/stream_executor/dnn.h | 20 +++++++++++++++++-- 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/tensorflow/compiler/xla/service/gpu/cudnn_convolution_algorithm_picker.cc b/tensorflow/compiler/xla/service/gpu/cudnn_convolution_algorithm_picker.cc index d6b457a91b..1eccfe8571 100644 --- a/tensorflow/compiler/xla/service/gpu/cudnn_convolution_algorithm_picker.cc +++ b/tensorflow/compiler/xla/service/gpu/cudnn_convolution_algorithm_picker.cc @@ -99,9 +99,9 @@ bool ShouldIncludeWinogradNonfusedAlgo(const Shape& input_shape, const ConvolutionDimensionNumbers& dnums, se::StreamExecutor* stream_exec) { // Skip this check for cudnn7 and newer. - se::port::StatusOr> version = + auto version = stream_exec->AsDnn()->GetVersion(); - if (version.ok() && std::get<0>(version.ValueOrDie()) >= 7) { + if (version.ok() && version.ValueOrDie().major_version() >= 7) { return true; } diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc index 1dc7f991b3..a11b644ab1 100644 --- a/tensorflow/stream_executor/cuda/cuda_dnn.cc +++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc @@ -477,11 +477,12 @@ port::Status CudnnSupport::Init() { ToString(status))}; } -port::StatusOr> CudnnSupport::GetVersion() { +port::StatusOr +CudnnSupport::GetVersion() { CudnnVersion version; TF_RETURN_IF_ERROR(GetLoadedCudnnVersion(&version)); - return std::make_tuple(version.major_version, version.minor_version, - version.patch_level); + return perftools::gputools::dnn::VersionInfo( + version.major_version, version.minor_version, version.patch_level); } // Turns a BatchDescriptor structure into a cudnn tensor handle within a scope. diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.h b/tensorflow/stream_executor/cuda/cuda_dnn.h index 0e5368aca8..09d248f137 100644 --- a/tensorflow/stream_executor/cuda/cuda_dnn.h +++ b/tensorflow/stream_executor/cuda/cuda_dnn.h @@ -46,7 +46,7 @@ class CudnnSupport : public dnn::DnnSupport { ~CudnnSupport() override; port::Status Init() override; - port::StatusOr> GetVersion() override; + port::StatusOr GetVersion() override; port::StatusOr> createRnnDescriptor( int num_layers, int hidden_size, int input_size, diff --git a/tensorflow/stream_executor/dnn.h b/tensorflow/stream_executor/dnn.h index 3c47d2c2e8..47dcd80218 100644 --- a/tensorflow/stream_executor/dnn.h +++ b/tensorflow/stream_executor/dnn.h @@ -876,6 +876,22 @@ enum class ElementwiseOperation { kAdd, kMultiply }; string ElementwiseOperationString(ElementwiseOperation op); +// A simple class representing the version of the backing library, to +// workaround the "too perfect forwarding" issue in gcc6+ compilers. +// See PR#16309 and issue #18402 for links discussing the issue. +class VersionInfo { + public: + VersionInfo(int major = 0, int minor = 0, int patch = 0) + : major_(major), minor_(minor), patch_(patch) {} + int major_version() { return major_; } + int minor_version() { return minor_; } + int patch() { return patch_; } + private: + int major_; + int minor_; + int patch_; +}; + // Suite of operations typically used for implementing Deep/Convolutional Neural // Nets. Note: A false return value of an operation indicates the // implementation is not available. @@ -886,8 +902,8 @@ class DnnSupport { virtual port::Status Init() = 0; - // Gets the version of the backing library, as a {major, minor, patch} tuple. - virtual port::StatusOr> GetVersion() { + // Gets the version of the backing library, as a VersionInfo object. + virtual port::StatusOr GetVersion() { return port::UnimplementedError( "DnnSupport::GetVersion not implemented on this platform."); } -- GitLab From 7d89bfcd72bef4c5c9328a88ee520d81642b5284 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 18:19:05 -0700 Subject: [PATCH 298/791] Adding autograph built-in function checker. PiperOrigin-RevId: 192703924 --- .../contrib/autograph/converters/call_trees.py | 3 +-- tensorflow/contrib/autograph/impl/api.py | 2 +- tensorflow/contrib/autograph/pyct/inspect_utils.py | 13 +++++++++++++ .../contrib/autograph/pyct/inspect_utils_test.py | 7 +++++++ 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/autograph/converters/call_trees.py b/tensorflow/contrib/autograph/converters/call_trees.py index 61f6bfd7e7..9424966696 100644 --- a/tensorflow/contrib/autograph/converters/call_trees.py +++ b/tensorflow/contrib/autograph/converters/call_trees.py @@ -23,7 +23,6 @@ from __future__ import division from __future__ import print_function from collections import namedtuple -import types import gast @@ -114,7 +113,7 @@ class CallTreeTransformer(transformer.Base): def _function_is_compilable(self, target_entity): """Determines whether an entity can be compiled at all.""" # TODO(mdan): This is just a placeholder. Implement. - return not isinstance(target_entity, types.BuiltinFunctionType) + return not inspect_utils.isbuiltin(target_entity) def _should_compile(self, node, fqn): """Determines whether an entity should be compiled in the context.""" diff --git a/tensorflow/contrib/autograph/impl/api.py b/tensorflow/contrib/autograph/impl/api.py index dce994e50d..a553813e19 100644 --- a/tensorflow/contrib/autograph/impl/api.py +++ b/tensorflow/contrib/autograph/impl/api.py @@ -137,7 +137,7 @@ def converted_call(f, recursive, verbose, arg_types, *args, **kwargs): unknown_arg_value = object() # Sentinel for arguments of unknown value - if tf_inspect.isbuiltin(f): + if inspect_utils.isbuiltin(f): return builtins.dynamic_builtin(f, *args, **kwargs) if tf_inspect.isfunction(f) or tf_inspect.ismethod(f): diff --git a/tensorflow/contrib/autograph/pyct/inspect_utils.py b/tensorflow/contrib/autograph/pyct/inspect_utils.py index 386a6d21ec..63361cc4f2 100644 --- a/tensorflow/contrib/autograph/pyct/inspect_utils.py +++ b/tensorflow/contrib/autograph/pyct/inspect_utils.py @@ -22,12 +22,25 @@ from __future__ import division from __future__ import print_function import itertools +import types import six from tensorflow.python.util import tf_inspect +def isbuiltin(f): + # Note these return false for isinstance(f, types.BuiltinFunctionType) so we + # need to specifically check for them. + if f in (range, int, float): + return True + if isinstance(f, types.BuiltinFunctionType): + return True + if tf_inspect.isbuiltin(f): + return True + return False + + def getnamespace(f): """Returns the complete namespace of a function. diff --git a/tensorflow/contrib/autograph/pyct/inspect_utils_test.py b/tensorflow/contrib/autograph/pyct/inspect_utils_test.py index 58f827b79a..cf841dae81 100644 --- a/tensorflow/contrib/autograph/pyct/inspect_utils_test.py +++ b/tensorflow/contrib/autograph/pyct/inspect_utils_test.py @@ -258,6 +258,13 @@ class InspectUtilsTest(test.TestCase): self.assertTrue( inspect_utils.getdefiningclass(Subclass.baz, Subclass) is Subclass) + def test_isbuiltin(self): + self.assertTrue(inspect_utils.isbuiltin(range)) + self.assertTrue(inspect_utils.isbuiltin(float)) + self.assertTrue(inspect_utils.isbuiltin(int)) + self.assertTrue(inspect_utils.isbuiltin(len)) + self.assertFalse(inspect_utils.isbuiltin(function_decorator)) + if __name__ == '__main__': test.main() -- GitLab From 93afca507ec09ff3b5cdf05cbd5eb265e83fc8cb Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 18:29:05 -0700 Subject: [PATCH 299/791] Convert GrapplerFunctionItem to (Specialized)FunctionDef. PiperOrigin-RevId: 192704808 --- tensorflow/core/grappler/utils/BUILD | 3 + tensorflow/core/grappler/utils/functions.cc | 328 +++++++++++++++--- tensorflow/core/grappler/utils/functions.h | 92 +++-- .../core/grappler/utils/functions_test.cc | 179 ++++++++-- 4 files changed, 504 insertions(+), 98 deletions(-) diff --git a/tensorflow/core/grappler/utils/BUILD b/tensorflow/core/grappler/utils/BUILD index 05d9cbaa2b..b473f32c45 100644 --- a/tensorflow/core/grappler/utils/BUILD +++ b/tensorflow/core/grappler/utils/BUILD @@ -165,6 +165,7 @@ cc_library( "//tensorflow/core:lib_internal", "//tensorflow/core:protos_all_cc", "//tensorflow/core/grappler:grappler_item", + "//tensorflow/core/grappler:op_types", "//tensorflow/core/grappler:utils", ], ) @@ -177,6 +178,8 @@ tf_cc_test( "//tensorflow/cc:cc_ops", "//tensorflow/core:all_kernels", "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", "//tensorflow/core:protos_all_cc", "//tensorflow/core:test", "//tensorflow/core:test_main", diff --git a/tensorflow/core/grappler/utils/functions.cc b/tensorflow/core/grappler/utils/functions.cc index dd0d918e72..e8d423a759 100644 --- a/tensorflow/core/grappler/utils/functions.cc +++ b/tensorflow/core/grappler/utils/functions.cc @@ -23,27 +23,82 @@ limitations under the License. #include "tensorflow/core/framework/node_def.pb.h" #include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/grappler/op_types.h" #include "tensorflow/core/grappler/utils.h" +#include "tensorflow/core/lib/gtl/map_util.h" #include "tensorflow/core/lib/strings/scanner.h" namespace tensorflow { namespace grappler { +namespace { + +Status OutputNameRange(const FunctionLibraryDefinition& flib, + const NodeDef& node, + tensorflow::NameRangeMap* outputs_range_map) { + const OpRegistrationData* registration; + TF_RETURN_IF_ERROR(flib.LookUp(node.op(), ®istration)); + TF_RETURN_IF_ERROR(tensorflow::NameRangesForNode(node, registration->op_def, + nullptr, outputs_range_map)); + return Status::OK(); +} + +Status RegisterFunctionBodyOutputs(const FunctionLibraryDefinition& flib, + const NodeDef& node, + GrapplerFunctionConnectivity* connectivity) { + tensorflow::NameRangeMap outputs_range_map; + TF_RETURN_IF_ERROR(OutputNameRange(flib, node, &outputs_range_map)); + connectivity->RegisterFunctionBodyOutputs(node.name(), outputs_range_map); + return Status::OK(); +} + +// Replace the placeholder attribute values with the values specified in +// instantiation attributes. +Status ResolveFunctionBodyNodeAttrPlaceholders( + const AttrValueMap& func_instantiation_attr, NodeDef* node) { + for (auto& attr : *node->mutable_attr()) { + const string& placeholder = attr.second.placeholder(); + if (placeholder.empty()) continue; + + auto it = func_instantiation_attr.find(placeholder); + if (it != func_instantiation_attr.end()) { + attr.second = it->second; + } else { + return errors::InvalidArgument("Can't resolve placeholder: ", + placeholder); + } + } + return Status::OK(); +} + +} // namespace + void GrapplerFunctionConnectivity::RegisterInputArgExpansion( const InputArgExpansion& input_arg_expansion) { - input_arg_expansions_.insert( - {input_arg_expansion.input_name, input_arg_expansion}); + const auto& input_name = input_arg_expansion.input_name; + const auto& placeholders = input_arg_expansion.placeholders; + input_arg_expansions_.emplace(input_name, input_arg_expansion); + for (int i = 0; i < placeholders.size(); ++i) { + const string& placeholder = input_arg_expansion.placeholders[i]; + input_arg_placeholders_.emplace( + placeholder, InputArgPlaceholder{input_name, /*position=*/i}); + } } void GrapplerFunctionConnectivity::RegisterFunctionBodyOutputs( const string& node_name, const tensorflow::NameRangeMap& outputs) { - function_body_outputs_.insert({node_name, outputs}); + function_body_outputs_[node_name] = outputs; } Status GrapplerFunctionConnectivity::ExpandFunctionDefInput( const string& func_def_input, std::vector* graph_def_inputs) const { using ::tensorflow::strings::Scanner; + if (IsControlInput(func_def_input)) { + graph_def_inputs->push_back(func_def_input); + return Status::OK(); + } + // Parse input format: "node_name[:node_output][:position]" string node_name; string node_output; @@ -150,11 +205,8 @@ Status GrapplerFunctionConnectivity::ExpandNodeInputs( std::vector expanded_inputs; for (const string& function_def_input : function_body_node->input()) { - if (!IsControlInput(function_def_input)) - TF_RETURN_IF_ERROR( - ExpandFunctionDefInput(function_def_input, &expanded_inputs)); - else - expanded_inputs.push_back(function_def_input); + TF_RETURN_IF_ERROR( + ExpandFunctionDefInput(function_def_input, &expanded_inputs)); } function_body_node->clear_input(); @@ -163,10 +215,66 @@ Status GrapplerFunctionConnectivity::ExpandNodeInputs( return Status::OK(); } -Status GrapplerFunctionItemBuilder::GetTypeAttr(const string& type_attr_name, - DataType* data_type) const { - auto it = func_attr_->find(type_attr_name); - if (it == func_attr_->end()) { +Status GrapplerFunctionConnectivity::AsFunctionDefInput( + const string& graph_def_input, string* func_def_input) const { + using gtl::FindOrNull; + + if (IsControlInput(graph_def_input)) { + *func_def_input = graph_def_input; + return Status::OK(); + } + + int position; + string node_name = ParseNodeName(graph_def_input, &position); + CHECK_GE(position, 0); + + // Check if it's an input arg placeholder + if (position == 0) { + const InputArgPlaceholder* placeholder = + FindOrNull(input_arg_placeholders_, node_name); + if (placeholder != nullptr) { + *func_def_input = + strings::StrCat(placeholder->input_name, ":", placeholder->position); + return Status::OK(); + } + } + + // It must be output from one of the function body nodes + const tensorflow::NameRangeMap* outputs_range_map = + FindOrNull(function_body_outputs_, node_name); + if (outputs_range_map != nullptr) { + for (const auto& el : *outputs_range_map) { + const auto& output_name = el.first; + const auto& output_range = el.second; + if (position >= output_range.first && position < output_range.second) { + int pos = position - output_range.first; + *func_def_input = + strings::StrCat(node_name, ":", output_name, ":", pos); + return Status::OK(); + } + } + } + + return errors::InvalidArgument("Unknown graph def input: ", graph_def_input); +} + +Status GrapplerFunctionConnectivity::AsFunctionDefNode( + NodeDef* function_body_node) const { + string func_def_input; + + for (int i = 0; i < function_body_node->input_size(); ++i) { + TF_RETURN_IF_ERROR( + AsFunctionDefInput(function_body_node->input(i), &func_def_input)); + function_body_node->set_input(i, func_def_input); + } + + return Status::OK(); +} + +Status GrapplerFunctionItemInstantiation::GetTypeAttr( + const string& type_attr_name, DataType* data_type) const { + auto it = func_instantiation_attr_->find(type_attr_name); + if (it == func_instantiation_attr_->end()) { return errors::InvalidArgument("Type attribute ", type_attr_name, " is not defined"); } else if (it->second.type() == DT_INVALID) { @@ -178,31 +286,48 @@ Status GrapplerFunctionItemBuilder::GetTypeAttr(const string& type_attr_name, return Status::OK(); } -Status GrapplerFunctionItemBuilder::GetArgType(const OpDef::ArgDef& arg, - DataType* data_type) const { +Status GrapplerFunctionItemInstantiation::GetArgType( + const OpDef::ArgDef& arg, DataType* data_type) const { if (arg.type() != DT_INVALID) { *data_type = arg.type(); } else { + if (!arg.type_list_attr().empty() || !arg.number_attr().empty()) { + return errors::InvalidArgument( + "Arguments with sequence of tensors are not supported. Unsupported " + "argument name: ", + arg.name()); + } TF_RETURN_IF_ERROR(GetTypeAttr(arg.type_attr(), data_type)); } return Status::OK(); } GrapplerFunctionItem::GrapplerFunctionItem( - const string& function_name, + const string& func_name, const AttrValueMap& func_attr, const std::vector& input_arg_expansions, const std::vector& output_arg_expansions, GraphDef&& function_body) - : function_name_(function_name), + : func_attr_(func_attr), input_arg_expansions_(input_arg_expansions), output_arg_expansions_(output_arg_expansions) { + id = func_name; + // Fill the feed nodes with input placeholders + for (const InputArgExpansion& input_arg : input_arg_expansions_) { + for (const string& placeholder : input_arg.placeholders) { + feed.emplace_back(placeholder, Tensor()); + input_arg_placeholders_.insert(placeholder); + } + } + // Fill the fetch nodes with outputs + for (const OutputArgExpansion& output_arg : output_arg_expansions_) { + for (const string& output_tensor : output_arg.output_tensors) { + fetch.push_back(output_tensor); + } + } + // Swap the graph body graph.Swap(&function_body); } -const string& GrapplerFunctionItem::function_name() const { - return function_name_; -} - const std::vector& GrapplerFunctionItem::inputs() const { return input_arg_expansions_; } @@ -215,6 +340,11 @@ const std::size_t GrapplerFunctionItem::input_size() const { return input_arg_expansions_.size(); } +bool GrapplerFunctionItem::IsInputPlaceholder(const string& node_name) const { + return input_arg_placeholders_.find(node_name) != + input_arg_placeholders_.end(); +} + const std::vector& GrapplerFunctionItem::outputs() const { return output_arg_expansions_; } @@ -227,10 +357,19 @@ const std::size_t GrapplerFunctionItem::output_size() const { return output_arg_expansions_.size(); } +const AttrValueMap& GrapplerFunctionItem::func_attr() const { + return func_attr_; +} + const GraphDef& GrapplerFunctionItem::function_body() const { return graph; } GraphDef& GrapplerFunctionItem::mutable_function_body() { return graph; } +GrapplerFunctionItem& GrapplerFunctionItem::SwapFunctionBody(GraphDef&& other) { + graph.Swap(&other); + return *this; +} + std::vector OutputTensors(const GrapplerFunctionItem& item) { std::vector output_tensors; for (const OutputArgExpansion& output : item.outputs()) { @@ -241,18 +380,27 @@ std::vector OutputTensors(const GrapplerFunctionItem& item) { return output_tensors; } -Status MakeGrapplerFunctionItem( - const FunctionDef& func, - const std::unordered_map& func_attr, - const FunctionLibraryDefinition& func_library, GrapplerFunctionItem* item) { +Status MakeGrapplerFunctionItem(const FunctionDef& func, + const AttrValueMap& func_instantiation_attr, + const FunctionLibraryDefinition& flib, + GrapplerFunctionItem* item) { const OpDef& signature = func.signature(); if (signature.name().empty()) { return errors::InvalidArgument("Function name must be specified"); } - // Helper methods to lookup function attributes - GrapplerFunctionItemBuilder builder(&func_attr); + // Function types will be resolved from function instantiation attributes. All + // other attributes will be lost during conversion to FunctionDef. + for (const OpDef::AttrDef& attr : signature.attr()) { + if (attr.type() != "type") { + return errors::InvalidArgument( + "Function signature must have only type attributes"); + } + } + + // Helper methods to lookup function instantiation attributes + GrapplerFunctionItemInstantiation instantiation(&func_instantiation_attr); // Mapping from FunctionDef input format (name[:output][:position]) to // GraphDef input format (name[:position]) @@ -260,7 +408,10 @@ Status MakeGrapplerFunctionItem( std::vector inputs; std::vector outputs; + + // Function body shares the library with the graph that instantiated it. GraphDef function_body; + *function_body.mutable_library() = flib.ToProto(); // TODO(ezhulenev): support functions with tensor sequence inputs/outputs @@ -284,7 +435,7 @@ Status MakeGrapplerFunctionItem( } DataType input_data_type; - TF_RETURN_IF_ERROR(builder.GetArgType(input, &input_data_type)); + TF_RETURN_IF_ERROR(instantiation.GetArgType(input, &input_data_type)); NodeDef* placeholder = function_body.add_node(); placeholder->set_name(input.name()); @@ -292,6 +443,7 @@ Status MakeGrapplerFunctionItem( (*placeholder->mutable_attr())["T"].set_type(input_data_type); InputArgExpansion input_expansion{/*input_name=*/input.name(), + /*data_type=*/input_data_type, /*placeholders=*/{input.name()}}; connectivity.RegisterInputArgExpansion(input_expansion); inputs.push_back(input_expansion); @@ -302,24 +454,12 @@ Status MakeGrapplerFunctionItem( NodeDef* new_node = function_body.add_node(); *new_node = func_def_node; - // Replace the placeholder attribute values with the specified value - for (auto& attr : *new_node->mutable_attr()) { - const string& ph_name = attr.second.placeholder(); - auto it = func_attr.find(ph_name); - if (it != func_attr.end()) { - attr.second = it->second; - } - } - - // Functions use a custom format to encode connectivity. Map these custom - // strings to regular ones. - tensorflow::NameRangeMap outputs_range_map; - const OpRegistrationData* registration; - TF_RETURN_IF_ERROR(func_library.LookUp(func_def_node.op(), ®istration)); - TF_RETURN_IF_ERROR(tensorflow::NameRangesForNode( - func_def_node, registration->op_def, nullptr, &outputs_range_map)); - connectivity.RegisterFunctionBodyOutputs(func_def_node.name(), - outputs_range_map); + // Resolve all placeholder values using function instantiation attributes. + TF_RETURN_IF_ERROR(ResolveFunctionBodyNodeAttrPlaceholders( + func_instantiation_attr, new_node)); + // Register node output range in a function connectivity. + TF_RETURN_IF_ERROR( + RegisterFunctionBodyOutputs(flib, func_def_node, &connectivity)); } // Rewrite inputs to use GraphDef format @@ -331,20 +471,96 @@ Status MakeGrapplerFunctionItem( for (const OpDef::ArgDef& out : signature.output_arg()) { std::vector output_tensors; auto ret = func.ret().find(out.name()); - if (ret != func.ret().end()) { - // Expand outputs using provided output mapping - TF_RETURN_IF_ERROR( - connectivity.ExpandFunctionDefInput(ret->second, &output_tensors)); - } else { - // Otherwise output must be one of the function inputs - TF_RETURN_IF_ERROR( - connectivity.ExpandFunctionDefInput(out.name(), &output_tensors)); + TF_RETURN_IF_ERROR( + ret != func.ret().end() + // Expand outputs using provided output mapping + ? connectivity.ExpandFunctionDefInput(ret->second, &output_tensors) + // Otherwise output must be one of the function inputs + : connectivity.ExpandFunctionDefInput(out.name(), &output_tensors)); + + DataType output_data_type; + TF_RETURN_IF_ERROR(instantiation.GetArgType(out, &output_data_type)); + + OutputArgExpansion output{/*output_name=*/out.name(), + /*data_type=*/output_data_type, + /*output_tensors=*/output_tensors}; + outputs.push_back(output); + } + + *item = GrapplerFunctionItem( + /*func_name=*/signature.name(), + /*func_attr=*/AttrValueMap(func.attr().begin(), func.attr().end()), + inputs, outputs, std::move(function_body)); + return Status::OK(); +} + +// Register GrapplerFunctionItem input arg expansion and function body outputs +// in the GrapplerFunctionConnectivity +Status RegisterGrapplerFunctionConnectivity( + const GrapplerFunctionItem& item, const FunctionLibraryDefinition& flib, + GrapplerFunctionConnectivity* connectivity) { + for (const InputArgExpansion& input : item.inputs()) { + connectivity->RegisterInputArgExpansion(input); + } + for (const NodeDef& func_body_node : item.function_body().node()) { + TF_RETURN_IF_ERROR( + RegisterFunctionBodyOutputs(flib, func_body_node, connectivity)); + } + return Status::OK(); +} + +Status MakeSpecializedFunctionDef(const GrapplerFunctionItem& item, + const FunctionLibraryDefinition& flib, + FunctionDef* func) { + func->mutable_signature()->set_name(item.id); + + // Build a GrapplerFunctionConnectivity from inputs and new function body. + GrapplerFunctionConnectivity connectivity; + TF_RETURN_IF_ERROR( + RegisterGrapplerFunctionConnectivity(item, flib, &connectivity)); + + // Add function input arguments. + for (const InputArgExpansion& input_arg : item.inputs()) { + OpDef::ArgDef arg_def; + arg_def.set_name(input_arg.input_name); + arg_def.set_type(input_arg.data_type); + *func->mutable_signature()->add_input_arg() = arg_def; + } + + // Add function output arguments. + for (const OutputArgExpansion& output_arg : item.outputs()) { + OpDef::ArgDef arg_def; + arg_def.set_name(output_arg.output_name); + arg_def.set_type(output_arg.data_type); + *func->mutable_signature()->add_output_arg() = arg_def; + + CHECK(output_arg.output_tensors.size() == 1) // do some sanity checking + << "Outputs of tensor sequences are not supported"; + + string ret; + for (const string& output_tensor : output_arg.output_tensors) { + TF_RETURN_IF_ERROR(connectivity.AsFunctionDefInput(output_tensor, &ret)); + (*func->mutable_ret())[output_arg.output_name] = ret; } - outputs.push_back({out.name(), output_tensors}); } - *item = GrapplerFunctionItem(signature.name(), inputs, outputs, - std::move(function_body)); + // Copy function definition specific attributes. + for (const auto& attr : item.func_attr()) { + const auto& attr_name = attr.first; + const auto& attr_value = attr.second; + (*func->mutable_attr())[attr_name] = attr_value; + } + + // Copy function body nodes to the FunctionDef and update input format + for (const NodeDef& func_body_node : item.function_body().node()) { + // Do not copy input placeholders + if (item.IsInputPlaceholder(func_body_node.name())) continue; + + NodeDef* func_def_node = func->add_node_def(); + *func_def_node = func_body_node; + TF_RETURN_IF_ERROR(connectivity.AsFunctionDefNode(func_def_node)); + } + return Status::OK(); } diff --git a/tensorflow/core/grappler/utils/functions.h b/tensorflow/core/grappler/utils/functions.h index 60ea8857c0..2ac3917a66 100644 --- a/tensorflow/core/grappler/utils/functions.h +++ b/tensorflow/core/grappler/utils/functions.h @@ -28,14 +28,19 @@ limitations under the License. namespace tensorflow { namespace grappler { +using AttrValueMap = std::unordered_map; + // Depending on the function instantiation attributes, input argument to the // function might be a single tensor, list of tensors of the same type, or a // list of tensors of different types. // // InputArgExpansion keeps track of the placeholders that were added to the -// function body in place of function inputs. +// function body in place of function inputs and a resolved input data type. struct InputArgExpansion { + // TODO(ezhulenev): Add support for functions with tensor sequence inputs of + // different data types string input_name; // name of the function input argument + DataType data_type; // input data type std::vector placeholders; // names of placeholder nodes in the // function body }; @@ -44,11 +49,14 @@ struct InputArgExpansion { // to one or more outputs of one of the function body nodes. // // OutputArgExpansion keeps mapping from a function output arg to the output -// tensors of a function body nodes, that compute function outputs. +// tensors of a function body nodes and a resolved output data type struct OutputArgExpansion { + // TODO(ezhulenev): Add support for functions with tensor sequence outputs of + // different data types string output_name; // name of the function output argument - std::vector output_tensors; // names of output tensors from the - // function body graph nodes + DataType data_type; // output data type + std::vector output_tensors; // names of output tensor from the + // function body nodes }; // FunctionDef uses different connectivity encoding for the function body nodes, @@ -67,26 +75,46 @@ class GrapplerFunctionConnectivity { Status ExpandFunctionDefInput(const string& func_def_input, std::vector* graph_def_inputs) const; - // Update Node inputs from FunctionDef to GraphDef format + // Update Node inputs from FunctionDef to GraphDef format. Status ExpandNodeInputs(NodeDef* function_body_node) const; - // TODO(ezhulenev): fold GraphDef inputs back to FunctionDef format - // Status FoldGraphDefInputs(const std::vector graph_def_inputs, - // std::vector* function_def_inputs) const; + // When expanding inputs in function def format, single input might be + // expanded into multiple tensors. When converting back to the function def + // format from graph def format, it's always a 1-to-1 relationship. + // FunctionDef built from GrapplerFunctionItem is always specialized to it's + // instantiation attributes and length of input args (and node def outputs) is + // known. + + // Map from GraphDef input format to FunctionDef input format using registered + // input arg expansion and function body outputs. + Status AsFunctionDefInput(const string& graph_def_input, + string* func_def_input) const; + + // Update Node inputs from GraphDef to FunctionDef format. + Status AsFunctionDefNode(NodeDef* function_body_node) const; private: + // Mapping from input name to input arg expansion. std::unordered_map input_arg_expansions_; + // Mapping from function body node name to output names range map. std::unordered_map function_body_outputs_; + + struct InputArgPlaceholder { + string input_name; + int position; + }; + + // Mapping from input arg placeholder to the function input tensor. + std::unordered_map input_arg_placeholders_; }; -// Helper methods to build GrapplerFunctionItem from a function def and function -// attributes. -class GrapplerFunctionItemBuilder { +// Get Function type attributes using attributes of a node that instantiated +// a function. +class GrapplerFunctionItemInstantiation { public: - using FunctionAttr = std::unordered_map; - - explicit GrapplerFunctionItemBuilder(const FunctionAttr* func_attr) - : func_attr_(func_attr) {} + explicit GrapplerFunctionItemInstantiation( + const AttrValueMap* func_instantiation_attr) + : func_instantiation_attr_(func_instantiation_attr) {} // Get DataType from attributes by name. Return error if attribute is missing, // or it doesn't define a valid data type. @@ -97,20 +125,20 @@ class GrapplerFunctionItemBuilder { Status GetArgType(const OpDef::ArgDef& arg, DataType* data_type) const; private: - const FunctionAttr* func_attr_; // do not own + const AttrValueMap* func_instantiation_attr_; // do not own }; // A special case of GrapplerItem, constructed from a TensorFlow Function. class GrapplerFunctionItem : public GrapplerItem { public: - GrapplerFunctionItem() {} + GrapplerFunctionItem() = default; GrapplerFunctionItem( - const string& function_name, + const string& func_name, const AttrValueMap& func_attr, const std::vector& input_arg_expansions, const std::vector& output_arg_expansions, GraphDef&& function_body); - const string& function_name() const; + bool IsInputPlaceholder(const string& node_name) const; const std::vector& inputs() const; const InputArgExpansion& input(int i) const; @@ -120,13 +148,20 @@ class GrapplerFunctionItem : public GrapplerItem { const OutputArgExpansion& output(int i) const; const std::size_t output_size() const; + const AttrValueMap& func_attr() const; const GraphDef& function_body() const; GraphDef& mutable_function_body(); + GrapplerFunctionItem& SwapFunctionBody(GraphDef&& other); + private: - string function_name_; + AttrValueMap func_attr_; // Attributes specific to function definition that + // produced this item (FuncDef.attr field). + std::vector input_arg_expansions_; std::vector output_arg_expansions_; + + std::set input_arg_placeholders_; }; // Return all output tensors referenced by item output args. @@ -136,8 +171,21 @@ std::vector OutputTensors(const GrapplerFunctionItem& item); // Return error if the given function def cannot be converted. Status MakeGrapplerFunctionItem( const FunctionDef& func, - const std::unordered_map& func_attr, - const FunctionLibraryDefinition& func_library, GrapplerFunctionItem* item); + const std::unordered_map& func_instantiation_attr, + const FunctionLibraryDefinition& flib, GrapplerFunctionItem* item); + +// Register GrapplerFunctionItem input arg expansion and function body outputs +// in the GrapplerFunctionConnectivity. Use function library definition to +// lookup function body nodes output names and ranges. +Status RegisterGrapplerFunctionConnectivity( + const GrapplerFunctionItem& item, const FunctionLibraryDefinition& flib, + GrapplerFunctionConnectivity* connectivity); + +// Make a specialized FunctionDef from the GrapplerFunctionItem. Use function +// library definition to lookup function body nodes output names and ranges. +Status MakeSpecializedFunctionDef(const GrapplerFunctionItem& item, + const FunctionLibraryDefinition& flib, + FunctionDef* func); } // end namespace grappler } // end namespace tensorflow diff --git a/tensorflow/core/grappler/utils/functions_test.cc b/tensorflow/core/grappler/utils/functions_test.cc index 1eb3298e89..a9a708bf67 100644 --- a/tensorflow/core/grappler/utils/functions_test.cc +++ b/tensorflow/core/grappler/utils/functions_test.cc @@ -20,6 +20,7 @@ limitations under the License. #include "tensorflow/core/framework/node_def_util.h" #include "tensorflow/core/framework/tensor_testutil.h" #include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/lib/gtl/map_util.h" #include "tensorflow/core/platform/test.h" #include "tensorflow/core/protobuf/meta_graph.pb.h" @@ -32,8 +33,9 @@ class FunctionsTest : public ::testing::Test {}; TEST_F(FunctionsTest, GrapplerFunctionConnectivity_ExpandFunctionDefInput) { GrapplerFunctionConnectivity connectivity; - connectivity.RegisterInputArgExpansion({"inputA", {"inputA"}}); - connectivity.RegisterInputArgExpansion({"inputB", {"inputB_0", "inputB_1"}}); + connectivity.RegisterInputArgExpansion({"inputA", DT_FLOAT, {"inputA"}}); + connectivity.RegisterInputArgExpansion( + {"inputB", DT_FLOAT, {"inputB_0", "inputB_1"}}); connectivity.RegisterFunctionBodyOutputs("Add", {{"z", {0, 1}}}); connectivity.RegisterFunctionBodyOutputs("Func", @@ -93,11 +95,50 @@ TEST_F(FunctionsTest, GrapplerFunctionConnectivity_ExpandFunctionDefInput) { EXPECT_EQ("Func:3", inputs[0]); } +TEST_F(FunctionsTest, GrapplerFunctionConnectivity_AsFunctionDefInput) { + GrapplerFunctionConnectivity connectivity; + + connectivity.RegisterInputArgExpansion({"inputA", DT_FLOAT, {"inputA"}}); + connectivity.RegisterInputArgExpansion( + {"inputB", DT_FLOAT, {"inputB_0", "inputB_1"}}); + + connectivity.RegisterFunctionBodyOutputs("Add", {{"z", {0, 1}}}); + connectivity.RegisterFunctionBodyOutputs("Func", + {{"o1", {0, 2}}, {"o2", {2, 4}}}); + + string input; + + TF_EXPECT_OK(connectivity.AsFunctionDefInput("inputA", &input)); + EXPECT_EQ("inputA:0", input); + + TF_EXPECT_OK(connectivity.AsFunctionDefInput("inputB_0", &input)); + EXPECT_EQ("inputB:0", input); + + TF_EXPECT_OK(connectivity.AsFunctionDefInput("inputB_1", &input)); + EXPECT_EQ("inputB:1", input); + + TF_EXPECT_OK(connectivity.AsFunctionDefInput("Add", &input)); + EXPECT_EQ("Add:z:0", input); + + TF_EXPECT_OK(connectivity.AsFunctionDefInput("Func", &input)); + EXPECT_EQ("Func:o1:0", input); + + TF_EXPECT_OK(connectivity.AsFunctionDefInput("Func:1", &input)); + EXPECT_EQ("Func:o1:1", input); + + TF_EXPECT_OK(connectivity.AsFunctionDefInput("Func:2", &input)); + EXPECT_EQ("Func:o2:0", input); + + TF_EXPECT_OK(connectivity.AsFunctionDefInput("Func:3", &input)); + EXPECT_EQ("Func:o2:1", input); +} + TEST_F(FunctionsTest, GrapplerFunctionConnectivity_ExpandNodeInputs) { GrapplerFunctionConnectivity connectivity; - connectivity.RegisterInputArgExpansion({"inputA", {"inputA"}}); - connectivity.RegisterInputArgExpansion({"inputB", {"inputB_0", "inputB_1"}}); + connectivity.RegisterInputArgExpansion({"inputA", DT_FLOAT, {"inputA"}}); + connectivity.RegisterInputArgExpansion( + {"inputB", DT_FLOAT, {"inputB_0", "inputB_1"}}); NodeDef node; node.add_input("inputA:0"); @@ -131,12 +172,12 @@ TEST_F(FunctionsTest, FromSimpleFunctionDef) { std::unordered_map func_attr; func_attr["T"].set_type(DT_FLOAT); - FunctionLibraryDefinition library(OpRegistry::Global(), FunctionDefLibrary()); + FunctionLibraryDefinition flib(OpRegistry::Global(), FunctionDefLibrary()); GrapplerFunctionItem item; - TF_EXPECT_OK(MakeGrapplerFunctionItem(func, func_attr, library, &item)); + TF_EXPECT_OK(MakeGrapplerFunctionItem(func, func_attr, flib, &item)); - EXPECT_EQ("XTimesTwo", item.function_name()); + EXPECT_EQ("XTimesTwo", item.id); EXPECT_EQ(4, item.function_body().node_size()); EXPECT_EQ(1, item.input_size()); @@ -206,12 +247,12 @@ TEST_F(FunctionsTest, FromFunctionDefWithMultiOutputNodes) { std::unordered_map func_attr; func_attr["T"].set_type(DT_FLOAT); - FunctionLibraryDefinition library(OpRegistry::Global(), FunctionDefLibrary()); + FunctionLibraryDefinition flib(OpRegistry::Global(), FunctionDefLibrary()); GrapplerFunctionItem item; - TF_EXPECT_OK(MakeGrapplerFunctionItem(func, func_attr, library, &item)); + TF_EXPECT_OK(MakeGrapplerFunctionItem(func, func_attr, flib, &item)); - EXPECT_EQ("SubGrad", item.function_name()); + EXPECT_EQ("SubGrad", item.id); EXPECT_EQ(12, item.function_body().node_size()); ASSERT_EQ(3, item.input_size()); @@ -251,8 +292,8 @@ TEST_F(FunctionsTest, FromFunctionDefWithMultiOutputNodes) { } TEST_F(FunctionsTest, FromFunctionDefWithNestedFuncs) { - FunctionLibraryDefinition library(OpRegistry::Global(), FunctionDefLibrary()); - TF_ASSERT_OK(library.AddFunctionDef(FunctionDefHelper::Define( + FunctionLibraryDefinition flib(OpRegistry::Global(), FunctionDefLibrary()); + TF_ASSERT_OK(flib.AddFunctionDef(FunctionDefHelper::Define( // Name "Swap", // Args @@ -290,7 +331,7 @@ TEST_F(FunctionsTest, FromFunctionDefWithNestedFuncs) { func_attr["T"].set_type(DT_FLOAT); GrapplerFunctionItem item; - TF_EXPECT_OK(MakeGrapplerFunctionItem(func, func_attr, library, &item)); + TF_EXPECT_OK(MakeGrapplerFunctionItem(func, func_attr, flib, &item)); int count = 0; for (const NodeDef &node : item.function_body().node()) { @@ -348,10 +389,10 @@ TEST_F(FunctionsTest, FromFunctionDefWithOutputMappings) { {{"out", "Exp:y:0"}}); std::unordered_map func_attr; - FunctionLibraryDefinition library(OpRegistry::Global(), FunctionDefLibrary()); + FunctionLibraryDefinition flib(OpRegistry::Global(), FunctionDefLibrary()); GrapplerFunctionItem item; - TF_EXPECT_OK(MakeGrapplerFunctionItem(func, func_attr, library, &item)); + TF_EXPECT_OK(MakeGrapplerFunctionItem(func, func_attr, flib, &item)); EXPECT_EQ(1, item.output_size()); EXPECT_EQ("Exp", item.output(0).output_tensors[0]); @@ -391,12 +432,12 @@ TEST_F(FunctionsTest, FromFunctionDefWithInputForwarding) { {{"out0", "in0"}}); std::unordered_map func_attr; - FunctionLibraryDefinition library(OpRegistry::Global(), FunctionDefLibrary()); + FunctionLibraryDefinition flib(OpRegistry::Global(), FunctionDefLibrary()); GrapplerFunctionItem item; - TF_EXPECT_OK(MakeGrapplerFunctionItem(func, func_attr, library, &item)); + TF_EXPECT_OK(MakeGrapplerFunctionItem(func, func_attr, flib, &item)); - EXPECT_EQ("ForwardInputs", item.function_name()); + EXPECT_EQ("ForwardInputs", item.id); EXPECT_EQ(5, item.function_body().node_size()); EXPECT_EQ(3, item.output_size()); @@ -437,10 +478,10 @@ TEST_F(FunctionsTest, FromFunctionDefWithoutInput) { std::unordered_map func_attr; func_attr["T"].set_type(DT_FLOAT); - FunctionLibraryDefinition library(OpRegistry::Global(), FunctionDefLibrary()); + FunctionLibraryDefinition flib(OpRegistry::Global(), FunctionDefLibrary()); GrapplerFunctionItem item; - TF_EXPECT_OK(MakeGrapplerFunctionItem(func, func_attr, library, &item)); + TF_EXPECT_OK(MakeGrapplerFunctionItem(func, func_attr, flib, &item)); EXPECT_EQ(0, item.input_size()); EXPECT_EQ(1, item.output_size()); @@ -456,6 +497,104 @@ TEST_F(FunctionsTest, FromFunctionDefWithoutInput) { EXPECT_EQ("two", cast.input(0)); } +TEST_F(FunctionsTest, MakeSpecializedFunctionDef) { + const Tensor kTwo = test::AsScalar(2); + FunctionDef func = FunctionDefHelper::Define( + // Name + "XTimesTwo", + // Args + {"x: T"}, + // Return values + {"y: T"}, + // Attr def + {"T: {float, double, int32, int64}"}, + // Nodes + { + {{"two"}, "Const", {}, {{"value", kTwo}, {"dtype", DT_INT64}}}, + {{"scale"}, "Cast", {"two"}, {{"SrcT", DT_INT64}, {"DstT", "$T"}}}, + {{"y"}, "Mul", {"x", "scale"}, {{"T", "$T"}}}, + }); + + std::unordered_map func_attr; + func_attr["T"].set_type(DT_FLOAT); + FunctionLibraryDefinition flib(OpRegistry::Global(), FunctionDefLibrary()); + + GrapplerFunctionItem item; + TF_EXPECT_OK(MakeGrapplerFunctionItem(func, func_attr, flib, &item)); + + FunctionDef specialized; + TF_EXPECT_OK(MakeSpecializedFunctionDef(item, flib, &specialized)); + + // Input and output types are resolved based on instantiation attributes. + EXPECT_EQ("x", specialized.signature().input_arg(0).name()); + EXPECT_EQ(DT_FLOAT, specialized.signature().input_arg(0).type()); + EXPECT_EQ("y", specialized.signature().output_arg(0).name()); + EXPECT_EQ(DT_FLOAT, specialized.signature().output_arg(0).type()); + + // Function body specialized for instantiation types + int count = 0; + for (const NodeDef &node : specialized.node_def()) { + if (node.name() == "scale" && count++) { + EXPECT_EQ(DT_FLOAT, node.attr().at("DstT").type()); + } else if (node.name() == "y" && count++) { + EXPECT_EQ("Mul", node.op()); + EXPECT_EQ("x:0", node.input(0)); + EXPECT_EQ("scale:y:0", node.input(1)); + EXPECT_EQ(DT_FLOAT, node.attr().at("T").type()); + } + } + EXPECT_EQ(2, count); +} + +TEST_F(FunctionsTest, SwapFunctionBodyAndMakeSpecializedFunctionDef) { + using test::function::NDef; + + FunctionDef mul_func = FunctionDefHelper::Create( + "MyMul", {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"}, + {{{"output"}, "Mul", {"x", "y"}, {{"T", "$T"}}}}, + /* Mapping between function returns and function node outputs. */ + {{"z", "output:z:0"}}); + + FunctionDef func = FunctionDefHelper::Create( + "MySquare", {"x:T"}, {"z:T"}, {"T: {float, double}"}, + {{{"output"}, "MyMul", {"x", "x"}, {{"T", "$T"}}}}, + /* Mapping between function returns and function node outputs. */ + {{"z", "output:z:0"}}); + + GraphDef id_func_body = test::function::GDef( + {/* pass input to output through identity */ + NDef("output", "Identity", {"x"}, {{"T", "float"}})}); + + std::unordered_map func_attr; + func_attr["T"].set_type(DT_FLOAT); + + FunctionDefLibrary lib_def; + *lib_def.add_function() = func; + *lib_def.add_function() = mul_func; + FunctionLibraryDefinition flib(OpRegistry::Global(), lib_def); + + GrapplerFunctionItem item; + TF_EXPECT_OK(MakeGrapplerFunctionItem(func, func_attr, flib, &item)); + + // Replace function body with identity function + item.SwapFunctionBody(std::move(id_func_body)); + FunctionDef specialized; + TF_EXPECT_OK(MakeSpecializedFunctionDef(item, flib, &specialized)); + + // Check that graph body was updated. + int count = 0; + for (const NodeDef &node : specialized.node_def()) { + if (node.name() == "output" && count++) { + EXPECT_EQ("Identity", node.op()); + EXPECT_EQ("x:0", node.input(0)); + } + } + EXPECT_EQ(1, count); + + // And return tensor mapping was updated with a new output name (z->output). + EXPECT_EQ("output:output:0", (*specialized.mutable_ret())["z"]); +} + } // namespace } // namespace grappler } // namespace tensorflow -- GitLab From c4526e50b2ac2d6819c8eb67db5423af103a1bb7 Mon Sep 17 00:00:00 2001 From: Francois Chollet Date: Thu, 12 Apr 2018 18:36:13 -0700 Subject: [PATCH 300/791] Avoid calling K.learning_phase() when not necessary in Dropout layer since it instantiates a placeholder_with_default, which is not supported by TPU compilation. PiperOrigin-RevId: 192705478 --- tensorflow/python/keras/_impl/keras/layers/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/keras/_impl/keras/layers/core.py b/tensorflow/python/keras/_impl/keras/layers/core.py index f64174a23f..9c4cb0f4fd 100644 --- a/tensorflow/python/keras/_impl/keras/layers/core.py +++ b/tensorflow/python/keras/_impl/keras/layers/core.py @@ -130,6 +130,7 @@ class Dropout(Layer): return nn_ops._get_noise_shape(inputs, self.noise_shape) # pylint: disable=protected-access def call(self, inputs, training=None): + original_training_value = training if training is None: training = K.learning_phase() @@ -141,7 +142,7 @@ class Dropout(Layer): dropped_inputs, lambda: array_ops.identity(inputs)) # EagerTensor object has no attribute _uses_learning_phase - if not context.executing_eagerly() and training is K.learning_phase(): + if not context.executing_eagerly() and original_training_value is None: output._uses_learning_phase = True # pylint: disable=protected-access return output -- GitLab From 5a6d5a1b3982e59548340422f831ada6f5d5e0be Mon Sep 17 00:00:00 2001 From: Francois Chollet Date: Thu, 12 Apr 2018 19:01:10 -0700 Subject: [PATCH 301/791] Enable efficient feeding of symbolic tensors to placeholders in the Keras backend. PiperOrigin-RevId: 192707345 --- .../python/keras/_impl/keras/backend.py | 110 ++++++++++++++---- .../python/keras/_impl/keras/backend_test.py | 43 ++++++- .../keras/_impl/keras/integration_test.py | 2 +- 3 files changed, 124 insertions(+), 31 deletions(-) diff --git a/tensorflow/python/keras/_impl/keras/backend.py b/tensorflow/python/keras/_impl/keras/backend.py index 096db8db32..6647cc5b79 100644 --- a/tensorflow/python/keras/_impl/keras/backend.py +++ b/tensorflow/python/keras/_impl/keras/backend.py @@ -2760,8 +2760,7 @@ class Function(object): outputs: Output tensors to fetch. updates: Additional update ops to be run at function call. name: A name to help users identify what this function does. - session_kwargs: Arguments to `tf.Session.run()`: `fetches`, `feed_dict`, - `options`, `run_metadata` + session_kwargs: Arguments to `tf.Session.run()`: `fetches`, `feed_dict`. """ def __init__(self, inputs, outputs, updates=None, name=None, @@ -2795,19 +2794,74 @@ class Function(object): self.fetches = session_kwargs.pop('fetches', []) if not isinstance(self.fetches, list): self.fetches = [self.fetches] + # The main use case of `fetches` being passed to a model is the ability + # to run custom updates (since the outputs of fetches are never returned). + # This requires us to wrap fetches in `identity` ops. + self.fetches = [array_ops.identity(x) for x in self.fetches] self.session_kwargs = session_kwargs + if session_kwargs: + raise ValueError('Some keys in session_kwargs are not supported at this ' + 'time: %s', session_kwargs.keys()) + + self._callable_fn = None + self._feed_arrays = None + self._feed_symbols = None + self._symbol_vals = None + self._session = None + + def _make_callable(self, feed_arrays, feed_symbols, symbol_vals, session): + """Generates a callable that runs the graph. + + Arguments: + feed_arrays: List of input tensors to be fed Numpy arrays at runtime. + feed_symbols: List of input tensors to be fed symbolic tensors at runtime. + symbol_vals: List of symbolic tensors to be fed to `feed_symbols`. + session: Session to use to generate the callable. + + Returns: + Function that runs the graph according to the above options. + """ + # Prepare callable options. + callable_opts = config_pb2.CallableOptions() + # Handle external-data feed. + for x in feed_arrays: + callable_opts.feed.append(x.name) + if self.feed_dict: + for key in sorted(self.feed_dict.keys()): + callable_opts.feed.append(key.name) + # Handle symbolic feed. + for x, y in zip(feed_symbols, symbol_vals): + connection = callable_opts.tensor_connection.add() + from_tensor = ops._as_graph_element(y) + if from_tensor is None: + from_tensor = y + connection.from_tensor = from_tensor.name # Data tensor + connection.to_tensor = x.name # Placeholder + # Handle fetches. + for x in self.outputs + self.fetches: + callable_opts.fetch.append(x.name) + # Handle updates. + callable_opts.target.append(self.updates_op.name) + # Create callable. + callable_fn = session._make_callable_from_options(callable_opts) + # Cache parameters corresponding to the generated callable, so that + # we can detect future mismatches and refresh the callable. + self._callable_fn = callable_fn + self._feed_arrays = feed_arrays + self._feed_symbols = feed_symbols + self._symbol_vals = symbol_vals + self._session = session + def __call__(self, inputs): if not isinstance(inputs, (list, tuple)): raise TypeError('`inputs` should be a list or tuple.') - if self.feed_dict: - feed_dict = self.feed_dict.copy() - else: - feed_dict = {} - session = get_session() - data_tensors_to_feed = [] + feed_arrays = [] + array_vals = [] + feed_symbols = [] + symbol_vals = [] for tensor, value in zip(self.inputs, inputs): if value is None: continue @@ -2816,23 +2870,31 @@ class Function(object): indices = np.concatenate((np.expand_dims(sparse_coo.row, 1), np.expand_dims(sparse_coo.col, 1)), 1) value = (indices, sparse_coo.data, sparse_coo.shape) - elif tensor_util.is_tensor(value): - data_tensors_to_feed.append((tensor, value)) + if tensor_util.is_tensor(value): + # Case: feeding symbolic tensor. + feed_symbols.append(tensor) + symbol_vals.append(value) else: - feed_dict[tensor] = value - - if data_tensors_to_feed: - # This is a *temporary* workaround (i.e. hack) to feed a symbolic tensor - # to `feed_dict`. It is very inefficient. It will be removed as soon - # as it becomes possible to pass symbolic tensors to `feed_dict`. - data_tensor_values = session.run([x[1] for x in data_tensors_to_feed]) - for i, v in enumerate(data_tensor_values): - feed_dict[data_tensors_to_feed[i][0]] = v - - fetches = self.outputs + [self.updates_op] + self.fetches - updated = session.run( - fetches=fetches, feed_dict=feed_dict, **self.session_kwargs) - return updated[:len(self.outputs)] + # Case: feeding Numpy array. + feed_arrays.append(tensor) + # We need to do array conversion and type casting at this level, since + # `callable_fn` only supports exact matches. + array_vals.append(np.asarray(value, dtype=tensor.dtype.base_dtype.name)) + if self.feed_dict: + for key in sorted(self.feed_dict.keys()): + array_vals.append( + np.asarray(self.feed_dict[key], dtype=key.dtype.base_dtype.name)) + + # Refresh callable if anything has changed. + if (self._callable_fn is None or + feed_arrays != self._feed_arrays or + symbol_vals != self._symbol_vals or + feed_symbols != self._feed_symbols or + session != self._session): + self._make_callable(feed_arrays, feed_symbols, symbol_vals, session) + + fetched = self._callable_fn(*array_vals) + return fetched[:len(self.outputs)] @tf_export('keras.backend.function') diff --git a/tensorflow/python/keras/_impl/keras/backend_test.py b/tensorflow/python/keras/_impl/keras/backend_test.py index fb4b2a0e1d..0193fc6976 100644 --- a/tensorflow/python/keras/_impl/keras/backend_test.py +++ b/tensorflow/python/keras/_impl/keras/backend_test.py @@ -189,6 +189,34 @@ class BackendUtilsTest(test.TestCase): for y in ys: self.assertEqual(y.op.name[:12], 'StopGradient') + def test_function_tf_feed_symbols(self): + with self.test_session(): + # Test feeding a resource variable to `function`. + x1 = keras.backend.placeholder(shape=()) + x2 = keras.backend.placeholder(shape=()) + lr = keras.backend.learning_phase() # Include a placeholder_with_default. + + y1 = keras.backend.variable(10.) + y2 = 3 + + f = keras.backend.function( + inputs=[x1, x2, lr], + outputs=[x1 + 1, + keras.backend.in_train_phase(x2 + 2, x2 - 1)]) + outs = f([y1, y2, None]) # Use default learning_phase value. + self.assertEqual(outs, [11., 2.]) + outs = f([y1, y2, 1]) # Set learning phase value. + self.assertEqual(outs, [11., 5.]) + + # Test triggering a callable refresh by changing the input. + y3 = keras.backend.constant(20.) # Test with tensor + outs = f([y3, y2, None]) + self.assertEqual(outs, [21., 2.]) + + y4 = 4 # Test with non-symbol + outs = f([y4, y2, None]) + self.assertEqual(outs, [5., 2.]) + def test_function_tf_fetches(self): # Additional operations can be passed to tf.Session().run() via its # `fetches` arguments. In contrast to `updates` argument of @@ -206,8 +234,9 @@ class BackendUtilsTest(test.TestCase): updates=[(x, x_placeholder + 1.)], fetches=[keras.backend.update(y, 5.)]) output = f([10., 20.]) - assert output == [30.] - assert keras.backend.get_session().run(fetches=[x, y]) == [11., 5.] + self.assertEqual(output, [30.]) + self.assertEqual( + keras.backend.get_session().run(fetches=[x, y]), [11., 5.]) def test_function_tf_feed_dict(self): # Additional substitutions can be passed to `tf.Session().run()` via its @@ -229,14 +258,16 @@ class BackendUtilsTest(test.TestCase): feed_dict=feed_dict, fetches=fetches) output = f([10.]) - assert output == [11.] - assert keras.backend.get_session().run(fetches=[x, y]) == [20., 30.] + self.assertEqual(output, [11.]) + self.assertEqual( + keras.backend.get_session().run(fetches=[x, y]), [20., 30.]) # updated value in feed_dict will be modified within the K.function() feed_dict[y_placeholder] = 4. output = f([20.]) - assert output == [21.] - assert keras.backend.get_session().run(fetches=[x, y]) == [30., 40.] + self.assertEqual(output, [21.]) + self.assertEqual( + keras.backend.get_session().run(fetches=[x, y]), [30., 40.]) class BackendVariableTest(test.TestCase): diff --git a/tensorflow/python/keras/_impl/keras/integration_test.py b/tensorflow/python/keras/_impl/keras/integration_test.py index c44808421f..43aff67ef9 100644 --- a/tensorflow/python/keras/_impl/keras/integration_test.py +++ b/tensorflow/python/keras/_impl/keras/integration_test.py @@ -95,7 +95,7 @@ class KerasIntegrationTest(test.TestCase): model.compile(loss='categorical_crossentropy', optimizer=keras.optimizers.Adam(lr=0.1), metrics=['accuracy']) - history = model.fit(x_train, y_train, epochs=10, batch_size=16, + history = model.fit(x_train, y_train, epochs=15, batch_size=16, validation_data=(x_train, y_train), verbose=2) self.assertGreater(history.history['val_acc'][-1], 0.7) -- GitLab From 4f615adc1d7875f9fbe592619dc6b0f31cc7fd9e Mon Sep 17 00:00:00 2001 From: Jiri Simsa Date: Thu, 12 Apr 2018 19:13:18 -0700 Subject: [PATCH 302/791] Automated g4 rollback of changelist 192691078 PiperOrigin-RevId: 192708480 --- tensorflow/contrib/BUILD | 1 - tensorflow/contrib/__init__.py | 1 - tensorflow/contrib/cmake/tf_python.cmake | 6 ++---- .../python/kernel_tests/decode_proto_fail_test.py | 4 ++-- .../python/kernel_tests/decode_proto_op_test.py | 4 ++-- .../python/kernel_tests/encode_proto_op_test.py | 15 +++++++-------- 6 files changed, 13 insertions(+), 18 deletions(-) diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD index 7e47516550..192d053683 100644 --- a/tensorflow/contrib/BUILD +++ b/tensorflow/contrib/BUILD @@ -77,7 +77,6 @@ py_library( "//tensorflow/contrib/optimizer_v2:optimizer_v2_py", "//tensorflow/contrib/periodic_resample:init_py", "//tensorflow/contrib/predictor", - "//tensorflow/contrib/proto", "//tensorflow/contrib/quantization:quantization_py", "//tensorflow/contrib/quantize:quantize_graph", "//tensorflow/contrib/autograph", diff --git a/tensorflow/contrib/__init__.py b/tensorflow/contrib/__init__.py index 36cc5144d0..e02dd5e759 100644 --- a/tensorflow/contrib/__init__.py +++ b/tensorflow/contrib/__init__.py @@ -64,7 +64,6 @@ from tensorflow.contrib import nn from tensorflow.contrib import opt from tensorflow.contrib import periodic_resample from tensorflow.contrib import predictor -from tensorflow.contrib import proto from tensorflow.contrib import quantization from tensorflow.contrib import quantize from tensorflow.contrib import recurrent diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake index f6aaf41f73..9d9db82513 100755 --- a/tensorflow/contrib/cmake/tf_python.cmake +++ b/tensorflow/contrib/cmake/tf_python.cmake @@ -330,10 +330,8 @@ GENERATE_PYTHON_OP_LIB("ctc_ops") GENERATE_PYTHON_OP_LIB("cudnn_rnn_ops") GENERATE_PYTHON_OP_LIB("data_flow_ops") GENERATE_PYTHON_OP_LIB("dataset_ops") -GENERATE_PYTHON_OP_LIB("decode_proto_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/proto/python/ops/gen_decode_proto_op.py) -GENERATE_PYTHON_OP_LIB("encode_proto_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/proto/python/ops/gen_encode_proto_op.py) +GENERATE_PYTHON_OP_LIB("decode_proto_ops") +GENERATE_PYTHON_OP_LIB("encode_proto_ops") GENERATE_PYTHON_OP_LIB("image_ops") GENERATE_PYTHON_OP_LIB("io_ops") GENERATE_PYTHON_OP_LIB("linalg_ops") diff --git a/tensorflow/contrib/proto/python/kernel_tests/decode_proto_fail_test.py b/tensorflow/contrib/proto/python/kernel_tests/decode_proto_fail_test.py index f8969b0bd5..f019833905 100644 --- a/tensorflow/contrib/proto/python/kernel_tests/decode_proto_fail_test.py +++ b/tensorflow/contrib/proto/python/kernel_tests/decode_proto_fail_test.py @@ -21,7 +21,7 @@ from __future__ import print_function import numpy as np -from tensorflow.contrib.proto import decode_proto +from tensorflow.contrib import proto from tensorflow.contrib.proto.python.kernel_tests import test_case from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors @@ -46,7 +46,7 @@ class DecodeProtoFailTest(test_case.ProtoOpTestCase): field_types = [dtypes.int32] with self.test_session() as sess: - ctensor, vtensor = decode_proto( + ctensor, vtensor = proto.decode_proto( batch, message_type=msg_type, field_names=field_names, diff --git a/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test.py b/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test.py index cd5121cdba..30ceac5f5f 100644 --- a/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test.py +++ b/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test.py @@ -27,7 +27,7 @@ import numpy as np from google.protobuf import text_format -from tensorflow.contrib.proto import decode_proto +from tensorflow.contrib import proto from tensorflow.contrib.proto.python.kernel_tests import test_case from tensorflow.contrib.proto.python.kernel_tests import test_example_pb2 from tensorflow.python.framework import dtypes @@ -175,7 +175,7 @@ class DecodeProtoOpTest(test_case.ProtoOpTestCase): output_types = [f.dtype for f in fields] with self.test_session() as sess: - sizes, vtensor = decode_proto( + sizes, vtensor = proto.decode_proto( batch, message_type=message_type, field_names=field_names, diff --git a/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test.py b/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test.py index a289ff290a..2a24c3b8ce 100644 --- a/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test.py +++ b/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test.py @@ -30,8 +30,7 @@ import numpy as np from google.protobuf import text_format -from tensorflow.contrib.proto import decode_proto -from tensorflow.contrib.proto import encode_proto +from tensorflow.contrib import proto from tensorflow.contrib.proto.python.kernel_tests import test_case from tensorflow.contrib.proto.python.kernel_tests import test_example_pb2 from tensorflow.python.framework import dtypes @@ -51,7 +50,7 @@ class EncodeProtoOpTest(test_case.ProtoOpTestCase): # Invalid field name with self.test_session(): with self.assertRaisesOpError('Unknown field: non_existent_field'): - encode_proto( + proto.encode_proto( sizes=[[1]], values=[np.array([[0.0]], dtype=np.int32)], message_type='tensorflow.contrib.proto.RepeatedPrimitiveValue', @@ -61,7 +60,7 @@ class EncodeProtoOpTest(test_case.ProtoOpTestCase): with self.test_session(): with self.assertRaisesOpError( 'Incompatible type for field double_value.'): - encode_proto( + proto.encode_proto( sizes=[[1]], values=[np.array([[0.0]], dtype=np.int32)], message_type='tensorflow.contrib.proto.RepeatedPrimitiveValue', @@ -73,7 +72,7 @@ class EncodeProtoOpTest(test_case.ProtoOpTestCase): r'sizes should be batch_size \+ \[len\(field_names\)\]'): sizes = array_ops.placeholder(dtypes.int32) values = array_ops.placeholder(dtypes.float64) - encode_proto( + proto.encode_proto( sizes=sizes, values=[values], message_type='tensorflow.contrib.proto.RepeatedPrimitiveValue', @@ -89,7 +88,7 @@ class EncodeProtoOpTest(test_case.ProtoOpTestCase): sizes = array_ops.placeholder(dtypes.int32) values1 = array_ops.placeholder(dtypes.float64) values2 = array_ops.placeholder(dtypes.int32) - (encode_proto( + (proto.encode_proto( sizes=[[1, 1]], values=[values1, values2], message_type='tensorflow.contrib.proto.RepeatedPrimitiveValue', @@ -104,13 +103,13 @@ class EncodeProtoOpTest(test_case.ProtoOpTestCase): out_types = [f.dtype for f in fields] with self.test_session() as sess: - sizes, field_tensors = decode_proto( + sizes, field_tensors = proto.decode_proto( in_bufs, message_type=message_type, field_names=field_names, output_types=out_types) - out_tensors = encode_proto( + out_tensors = proto.encode_proto( sizes, field_tensors, message_type=message_type, -- GitLab From 3c9870524b86fe7e3cff5a49daa692cd52e7f0c4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Apr 2018 19:52:18 -0700 Subject: [PATCH 303/791] Add boolean type to tflite in favor of comparison implementations. PiperOrigin-RevId: 192711203 --- tensorflow/contrib/lite/context.h | 2 ++ tensorflow/contrib/lite/interpreter.cc | 8 ++++++-- tensorflow/contrib/lite/interpreter.h | 4 ++++ tensorflow/contrib/lite/kernels/internal/tensor.h | 5 +++++ tensorflow/contrib/lite/model.cc | 3 +++ tensorflow/contrib/lite/optional_debug_tools.cc | 2 ++ .../interpreter_wrapper/interpreter_wrapper.cc | 4 ++++ tensorflow/contrib/lite/schema/schema.fbs | 1 + tensorflow/contrib/lite/schema/schema_generated.h | 9 ++++++--- tensorflow/contrib/lite/testing/split.h | 10 ++++++++++ tensorflow/contrib/lite/testing/split_test.cc | 5 +++++ tensorflow/contrib/lite/testing/tflite_driver.cc | 15 +++++++++++++++ 12 files changed, 63 insertions(+), 5 deletions(-) diff --git a/tensorflow/contrib/lite/context.h b/tensorflow/contrib/lite/context.h index 45184b05ec..0b38f43cd3 100644 --- a/tensorflow/contrib/lite/context.h +++ b/tensorflow/contrib/lite/context.h @@ -137,6 +137,7 @@ typedef enum { kTfLiteUInt8 = 3, kTfLiteInt64 = 4, kTfLiteString = 5, + kTfLiteBool = 6, } TfLiteType; // Parameters for asymmetric quantization. Quantized values can be converted @@ -155,6 +156,7 @@ typedef union { char* raw; const char* raw_const; uint8_t* uint8; + bool* b; } TfLitePtrUnion; // Memory allocation strategies. kTfLiteMmapRo is for read-only memory-mapped diff --git a/tensorflow/contrib/lite/interpreter.cc b/tensorflow/contrib/lite/interpreter.cc index 4575fe884d..f258654608 100644 --- a/tensorflow/contrib/lite/interpreter.cc +++ b/tensorflow/contrib/lite/interpreter.cc @@ -337,9 +337,13 @@ TfLiteStatus Interpreter::BytesRequired(TfLiteType type, const int* dims, case kTfLiteInt64: *bytes = sizeof(int64_t) * count; break; + case kTfLiteBool: + *bytes = sizeof(bool) * count; + break; default: - ReportError(&context_, - "Only float32, int32, int64, uint8 supported currently."); + ReportError( + &context_, + "Only float32, int32, int64, uint8, bool supported currently."); return kTfLiteError; } return kTfLiteOk; diff --git a/tensorflow/contrib/lite/interpreter.h b/tensorflow/contrib/lite/interpreter.h index a6d582a813..df67cce9de 100644 --- a/tensorflow/contrib/lite/interpreter.h +++ b/tensorflow/contrib/lite/interpreter.h @@ -48,6 +48,10 @@ template <> constexpr TfLiteType typeToTfLiteType() { return kTfLiteUInt8; } +template <> +constexpr TfLiteType typeToTfLiteType() { + return kTfLiteBool; +} // Forward declare since NNAPIDelegate uses Interpreter. class NNAPIDelegate; diff --git a/tensorflow/contrib/lite/kernels/internal/tensor.h b/tensorflow/contrib/lite/kernels/internal/tensor.h index 4bce2ffaaf..62cea143e6 100644 --- a/tensorflow/contrib/lite/kernels/internal/tensor.h +++ b/tensorflow/contrib/lite/kernels/internal/tensor.h @@ -44,6 +44,11 @@ inline int64_t* GetTensorData(TfLiteTensor* tensor) { return tensor != nullptr ? tensor->data.i64 : nullptr; } +template <> +inline bool* GetTensorData(TfLiteTensor* tensor) { + return tensor != nullptr ? tensor->data.b : nullptr; +} + inline int RemapDim(int max_dimensions, int d) { return max_dimensions - d - 1; } diff --git a/tensorflow/contrib/lite/model.cc b/tensorflow/contrib/lite/model.cc index 87af953061..0b65884025 100644 --- a/tensorflow/contrib/lite/model.cc +++ b/tensorflow/contrib/lite/model.cc @@ -57,6 +57,9 @@ TfLiteStatus ConvertTensorType(TensorType tensor_type, TfLiteType* type, case TensorType_STRING: *type = kTfLiteString; break; + case TensorType_BOOL: + *type = kTfLiteBool; + break; default: error_reporter->Report("Unimplemented data type %s (%d) in tensor\n", EnumNameTensorType(tensor_type), tensor_type); diff --git a/tensorflow/contrib/lite/optional_debug_tools.cc b/tensorflow/contrib/lite/optional_debug_tools.cc index 1f762e6688..e1366639c7 100644 --- a/tensorflow/contrib/lite/optional_debug_tools.cc +++ b/tensorflow/contrib/lite/optional_debug_tools.cc @@ -48,6 +48,8 @@ const char* TensorTypeName(TfLiteType type) { return "kTfLiteInt64"; case kTfLiteString: return "kTfLiteString"; + case kTfLiteBool: + return "kTfLiteBool"; } return "(invalid)"; } diff --git a/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.cc b/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.cc index 4b34969356..04fc098129 100644 --- a/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.cc +++ b/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.cc @@ -72,6 +72,8 @@ int TfLiteTypeToPyArrayType(TfLiteType tf_lite_type) { return NPY_INT64; case kTfLiteString: return NPY_OBJECT; + case kTfLiteBool: + return NPY_BOOL; case kTfLiteNoType: return -1; } @@ -90,6 +92,8 @@ TfLiteType TfLiteTypeFromPyArray(PyArrayObject* array) { return kTfLiteUInt8; case NPY_INT64: return kTfLiteInt64; + case NPY_BOOL: + return kTfLiteBool; case NPY_OBJECT: case NPY_STRING: case NPY_UNICODE: diff --git a/tensorflow/contrib/lite/schema/schema.fbs b/tensorflow/contrib/lite/schema/schema.fbs index 357493755d..fa825500fd 100644 --- a/tensorflow/contrib/lite/schema/schema.fbs +++ b/tensorflow/contrib/lite/schema/schema.fbs @@ -33,6 +33,7 @@ enum TensorType : byte { UINT8 = 3, INT64 = 4, STRING = 5, + BOOL = 6, } // Parameters for converting a quantized tensor back to float. Given a diff --git a/tensorflow/contrib/lite/schema/schema_generated.h b/tensorflow/contrib/lite/schema/schema_generated.h index c638daf66e..909c4ccb3b 100755 --- a/tensorflow/contrib/lite/schema/schema_generated.h +++ b/tensorflow/contrib/lite/schema/schema_generated.h @@ -173,18 +173,20 @@ enum TensorType { TensorType_UINT8 = 3, TensorType_INT64 = 4, TensorType_STRING = 5, + TensorType_BOOL = 6, TensorType_MIN = TensorType_FLOAT32, - TensorType_MAX = TensorType_STRING + TensorType_MAX = TensorType_BOOL }; -inline TensorType (&EnumValuesTensorType())[6] { +inline TensorType (&EnumValuesTensorType())[7] { static TensorType values[] = { TensorType_FLOAT32, TensorType_FLOAT16, TensorType_INT32, TensorType_UINT8, TensorType_INT64, - TensorType_STRING + TensorType_STRING, + TensorType_BOOL }; return values; } @@ -197,6 +199,7 @@ inline const char **EnumNamesTensorType() { "UINT8", "INT64", "STRING", + "BOOL", nullptr }; return names; diff --git a/tensorflow/contrib/lite/testing/split.h b/tensorflow/contrib/lite/testing/split.h index 428cfda4f2..896f2949ef 100644 --- a/tensorflow/contrib/lite/testing/split.h +++ b/tensorflow/contrib/lite/testing/split.h @@ -80,6 +80,16 @@ inline std::vector Split(const string& s, const string& delimiter) { return fields; } +template <> +inline std::vector Split(const string& s, const string& delimiter) { + std::vector fields; + for (const auto& p : SplitToPos(s, delimiter)) { + fields.push_back( + static_cast(strtol(s.data() + p.first, nullptr, 10))); + } + return fields; +} + } // namespace testing } // namespace tflite diff --git a/tensorflow/contrib/lite/testing/split_test.cc b/tensorflow/contrib/lite/testing/split_test.cc index 3d1e25d9c7..76b918cbcd 100644 --- a/tensorflow/contrib/lite/testing/split_test.cc +++ b/tensorflow/contrib/lite/testing/split_test.cc @@ -52,6 +52,11 @@ TEST(SplitTest, SplitUint8) { EXPECT_THAT(Split("1,-1,258", ","), ElementsAre(1, 255, 2)); } +TEST(SplitTest, SplitBool) { + EXPECT_THAT(Split("1, 0, 0, 1", ","), + ElementsAre(true, false, false, true)); +} + } // namespace } // namespace testing } // namespace tflite diff --git a/tensorflow/contrib/lite/testing/tflite_driver.cc b/tensorflow/contrib/lite/testing/tflite_driver.cc index 3764bab035..58fe5bd6e4 100644 --- a/tensorflow/contrib/lite/testing/tflite_driver.cc +++ b/tensorflow/contrib/lite/testing/tflite_driver.cc @@ -42,6 +42,10 @@ template <> uint8_t Value(const TfLitePtrUnion& data, int index) { return data.uint8[index]; } +template <> +bool Value(const TfLitePtrUnion& data, int index) { + return data.b[index]; +} template void SetTensorData(const std::vector& values, TfLitePtrUnion* data) { @@ -79,6 +83,8 @@ class TfLiteDriver::Expectation { return TypedCheck(verbose, tensor); case kTfLiteUInt8: return TypedCheck(verbose, tensor); + case kTfLiteBool: + return TypedCheck(verbose, tensor); default: fprintf(stderr, "Unsupported type %d in Check\n", tensor.type); return false; @@ -203,6 +209,12 @@ void TfLiteDriver::SetInput(int id, const string& csv_values) { SetTensorData(values, &tensor->data); break; } + case kTfLiteBool: { + const auto& values = testing::Split(csv_values, ","); + if (!CheckSizes(tensor->bytes, values.size())) return; + SetTensorData(values, &tensor->data); + break; + } default: fprintf(stderr, "Unsupported type %d in SetInput\n", tensor->type); Invalidate("Unsupported tensor data type"); @@ -231,6 +243,9 @@ void TfLiteDriver::SetExpectation(int id, const string& csv_values) { case kTfLiteUInt8: expected_output_[id]->SetData(csv_values); break; + case kTfLiteBool: + expected_output_[id]->SetData(csv_values); + break; default: fprintf(stderr, "Unsupported type %d in SetExpectation\n", tensor->type); Invalidate("Unsupported tensor data type"); -- GitLab From 3438c3f4f18e2057aee38d38537d96cc485b8fab Mon Sep 17 00:00:00 2001 From: Jiri Simsa Date: Thu, 12 Apr 2018 19:56:38 -0700 Subject: [PATCH 304/791] Automated g4 rollback of changelist 192504411 PiperOrigin-RevId: 192711501 --- tensorflow/contrib/proto/BUILD | 16 - .../contrib/proto/python/kernel_tests/BUILD | 81 ----- .../proto/python/kernel_tests/build_defs.bzl | 78 ----- .../kernel_tests/decode_proto_fail_test.py | 68 ---- .../kernel_tests/decode_proto_op_test.py | 300 ------------------ .../kernel_tests/encode_proto_op_test.py | 179 ----------- .../python/kernel_tests/minmax.TestCase.pbtxt | 161 ---------- .../python/kernel_tests/nested.TestCase.pbtxt | 16 - .../kernel_tests/optional.TestCase.pbtxt | 20 -- .../promote_unsigned.TestCase.pbtxt | 21 -- .../python/kernel_tests/ragged.TestCase.pbtxt | 32 -- .../kernel_tests/shaped_batch.TestCase.pbtxt | 62 ---- .../python/kernel_tests/simple.TestCase.pbtxt | 21 -- .../proto/python/kernel_tests/test_case.py | 35 -- .../python/kernel_tests/test_example.proto | 149 --------- tensorflow/tools/pip_package/BUILD | 1 - 16 files changed, 1240 deletions(-) delete mode 100644 tensorflow/contrib/proto/python/kernel_tests/BUILD delete mode 100644 tensorflow/contrib/proto/python/kernel_tests/build_defs.bzl delete mode 100644 tensorflow/contrib/proto/python/kernel_tests/decode_proto_fail_test.py delete mode 100644 tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test.py delete mode 100644 tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test.py delete mode 100644 tensorflow/contrib/proto/python/kernel_tests/minmax.TestCase.pbtxt delete mode 100644 tensorflow/contrib/proto/python/kernel_tests/nested.TestCase.pbtxt delete mode 100644 tensorflow/contrib/proto/python/kernel_tests/optional.TestCase.pbtxt delete mode 100644 tensorflow/contrib/proto/python/kernel_tests/promote_unsigned.TestCase.pbtxt delete mode 100644 tensorflow/contrib/proto/python/kernel_tests/ragged.TestCase.pbtxt delete mode 100644 tensorflow/contrib/proto/python/kernel_tests/shaped_batch.TestCase.pbtxt delete mode 100644 tensorflow/contrib/proto/python/kernel_tests/simple.TestCase.pbtxt delete mode 100644 tensorflow/contrib/proto/python/kernel_tests/test_case.py delete mode 100644 tensorflow/contrib/proto/python/kernel_tests/test_example.proto diff --git a/tensorflow/contrib/proto/BUILD b/tensorflow/contrib/proto/BUILD index 3e9b1a0b8d..046652cbc5 100644 --- a/tensorflow/contrib/proto/BUILD +++ b/tensorflow/contrib/proto/BUILD @@ -4,8 +4,6 @@ licenses(["notice"]) # Apache 2.0 exports_files(["LICENSE"]) -load("//tensorflow/core:platform/default/build_config_root.bzl", "if_static") - py_library( name = "proto", srcs = [ @@ -16,17 +14,3 @@ py_library( "//tensorflow/contrib/proto/python/ops:encode_proto_op_py", ], ) - -py_library( - name = "proto_pip", - data = [ - "//tensorflow/contrib/proto/python/kernel_tests:test_messages", - ] + if_static( - [], - otherwise = ["//tensorflow/contrib/proto/python/kernel_tests:libtestexample.so"], - ), - deps = [ - ":proto", - "//tensorflow/contrib/proto/python/kernel_tests:py_test_deps", - ], -) diff --git a/tensorflow/contrib/proto/python/kernel_tests/BUILD b/tensorflow/contrib/proto/python/kernel_tests/BUILD deleted file mode 100644 index 4125ea8a2a..0000000000 --- a/tensorflow/contrib/proto/python/kernel_tests/BUILD +++ /dev/null @@ -1,81 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -licenses(["notice"]) # Apache 2.0 - -exports_files(["LICENSE"]) - -# Much of the work in this BUILD file actually happens in the corresponding -# build_defs.bzl, which creates an individual testcase for each example .pbtxt -# file in this directory. -# -load(":build_defs.bzl", "decode_proto_test_suite") -load(":build_defs.bzl", "encode_proto_test_suite") - -# This expands to a tf_py_test for each test file. -# It defines the test_suite :decode_proto_op_tests. -decode_proto_test_suite( - name = "decode_proto_tests", - examples = glob(["*.pbtxt"]), -) - -# This expands to a tf_py_test for each test file. -# It defines the test_suite :encode_proto_op_tests. -encode_proto_test_suite( - name = "encode_proto_tests", - examples = glob(["*.pbtxt"]), -) - -# Below here are tests that are not tied to an example text proto. -filegroup( - name = "test_messages", - srcs = glob(["*.pbtxt"]), -) - -load("//tensorflow:tensorflow.bzl", "tf_py_test") -load("//tensorflow:tensorflow.bzl", "tf_cc_shared_object") -load("//tensorflow/core:platform/default/build_config_root.bzl", "if_static") -load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library") - -tf_py_test( - name = "decode_proto_fail_test", - size = "small", - srcs = ["decode_proto_fail_test.py"], - additional_deps = [ - ":py_test_deps", - "//third_party/py/numpy", - "//tensorflow/contrib/proto:proto", - ], - data = if_static( - [], - otherwise = [":libtestexample.so"], - ), -) - -py_library( - name = "test_case", - srcs = ["test_case.py"], - deps = ["//tensorflow/python:client_testlib"], -) - -py_library( - name = "py_test_deps", - deps = [ - ":test_case", - ":test_example_proto_py", - ], -) - -tf_proto_library( - name = "test_example_proto", - srcs = ["test_example.proto"], - cc_api_version = 2, - protodeps = ["//tensorflow/core:protos_all"], -) - -tf_cc_shared_object( - name = "libtestexample.so", - linkstatic = 1, - deps = [ - ":test_example_proto_cc", - ], -) diff --git a/tensorflow/contrib/proto/python/kernel_tests/build_defs.bzl b/tensorflow/contrib/proto/python/kernel_tests/build_defs.bzl deleted file mode 100644 index 6fe48ae807..0000000000 --- a/tensorflow/contrib/proto/python/kernel_tests/build_defs.bzl +++ /dev/null @@ -1,78 +0,0 @@ -"""BUILD rules for generating file-driven proto test cases. - -The decode_proto_test_suite() and encode_proto_test_suite() rules take a list -of text protos and generates a tf_py_test() for each one. -""" - -load("//tensorflow:tensorflow.bzl", "tf_py_test") -load("//tensorflow:tensorflow.bzl", "register_extension_info") -load("//tensorflow/core:platform/default/build_config_root.bzl", "if_static") - -def _test_name(test, path): - return "%s_%s_test" % (test, path.split("/")[-1].split(".")[0]) - -def decode_proto_test_suite(name, examples): - """Build the decode_proto py_test for each test filename.""" - for test_filename in examples: - tf_py_test( - name = _test_name("decode_proto", test_filename), - srcs = ["decode_proto_op_test.py"], - size = "small", - data = [test_filename] + if_static( - [], - otherwise = [":libtestexample.so"], - ), - main = "decode_proto_op_test.py", - args = [ - "--message_text_file=\"%s/%s\"" % (native.package_name(), test_filename), - ], - additional_deps = [ - ":py_test_deps", - "//third_party/py/numpy", - "//tensorflow/contrib/proto:proto", - ], - ) - native.test_suite( - name = name, - tests = [":" + _test_name("decode_proto", test_filename) - for test_filename in examples], - ) - -def encode_proto_test_suite(name, examples): - """Build the encode_proto py_test for each test filename.""" - for test_filename in examples: - tf_py_test( - name = _test_name("encode_proto", test_filename), - srcs = ["encode_proto_op_test.py"], - size = "small", - data = [test_filename] + if_static( - [], - otherwise = [":libtestexample.so"], - ), - main = "encode_proto_op_test.py", - args = [ - "--message_text_file=\"%s/%s\"" % (native.package_name(), test_filename), - ], - additional_deps = [ - ":py_test_deps", - "//third_party/py/numpy", - "//tensorflow/contrib/proto:proto", - ], - ) - native.test_suite( - name = name, - tests = [":" + _test_name("encode_proto", test_filename) - for test_filename in examples], - ) - -register_extension_info( - extension_name = "decode_proto_test_suite", - label_regex_map = { - "deps": "deps:decode_example_.*", - }) - -register_extension_info( - extension_name = "encode_proto_test_suite", - label_regex_map = { - "deps": "deps:encode_example_.*", - }) diff --git a/tensorflow/contrib/proto/python/kernel_tests/decode_proto_fail_test.py b/tensorflow/contrib/proto/python/kernel_tests/decode_proto_fail_test.py deleted file mode 100644 index f019833905..0000000000 --- a/tensorflow/contrib/proto/python/kernel_tests/decode_proto_fail_test.py +++ /dev/null @@ -1,68 +0,0 @@ -# ============================================================================= -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -# Python3 preparedness imports. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib import proto -from tensorflow.contrib.proto.python.kernel_tests import test_case -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.platform import test - - -class DecodeProtoFailTest(test_case.ProtoOpTestCase): - """Test failure cases for DecodeToProto.""" - - def _TestCorruptProtobuf(self, sanitize): - """Test failure cases for DecodeToProto.""" - - # The goal here is to check the error reporting. - # Testing against a variety of corrupt protobufs is - # done by fuzzing. - corrupt_proto = 'This is not a binary protobuf' - - # Numpy silently truncates the strings if you don't specify dtype=object. - batch = np.array(corrupt_proto, dtype=object) - msg_type = 'tensorflow.contrib.proto.TestCase' - field_names = ['sizes'] - field_types = [dtypes.int32] - - with self.test_session() as sess: - ctensor, vtensor = proto.decode_proto( - batch, - message_type=msg_type, - field_names=field_names, - output_types=field_types, - sanitize=sanitize) - with self.assertRaisesRegexp(errors.DataLossError, - 'Unable to parse binary protobuf' - '|Failed to consume entire buffer'): - _ = sess.run([ctensor] + vtensor) - - def testCorrupt(self): - self._TestCorruptProtobuf(sanitize=False) - - def testSanitizerCorrupt(self): - self._TestCorruptProtobuf(sanitize=True) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test.py b/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test.py deleted file mode 100644 index 30ceac5f5f..0000000000 --- a/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test.py +++ /dev/null @@ -1,300 +0,0 @@ -# ============================================================================= -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= -"""Table-driven test for decode_proto op. - -This test is run once with each of the *.TestCase.pbtxt files -in the test directory. -""" -# Python3 preparedness imports. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from google.protobuf import text_format - -from tensorflow.contrib import proto -from tensorflow.contrib.proto.python.kernel_tests import test_case -from tensorflow.contrib.proto.python.kernel_tests import test_example_pb2 -from tensorflow.python.framework import dtypes -from tensorflow.python.platform import flags -from tensorflow.python.platform import test - -FLAGS = flags.FLAGS - -flags.DEFINE_string('message_text_file', None, - 'A file containing a text serialized TestCase protobuf.') - - -class DecodeProtoOpTest(test_case.ProtoOpTestCase): - - def _compareValues(self, fd, vs, evs): - """Compare lists/arrays of field values.""" - - if len(vs) != len(evs): - self.fail('Field %s decoded %d outputs, expected %d' % - (fd.name, len(vs), len(evs))) - for i, ev in enumerate(evs): - # Special case fuzzy match for float32. TensorFlow seems to mess with - # MAX_FLT slightly and the test doesn't work otherwise. - # TODO(nix): ask on TF list about why MAX_FLT doesn't pass through. - if fd.cpp_type == fd.CPPTYPE_FLOAT: - # Numpy isclose() is better than assertIsClose() which uses an absolute - # value comparison. - self.assertTrue( - np.isclose(vs[i], ev), 'expected %r, actual %r' % (ev, vs[i])) - elif fd.cpp_type == fd.CPPTYPE_STRING: - # In Python3 string tensor values will be represented as bytes, so we - # reencode the proto values to match that. - self.assertEqual(vs[i], ev.encode('ascii')) - else: - # Doubles and other types pass through unscathed. - self.assertEqual(vs[i], ev) - - def _compareRepeatedPrimitiveValue(self, batch_shape, sizes, fields, - field_dict): - """Compare protos of type RepeatedPrimitiveValue. - - Args: - batch_shape: the shape of the input tensor of serialized messages. - sizes: int matrix of repeat counts returned by decode_proto - fields: list of test_example_pb2.FieldSpec (types and expected values) - field_dict: map from field names to decoded numpy tensors of values - """ - - # Check that expected values match. - for field in fields: - values = field_dict[field.name] - self.assertEqual(dtypes.as_dtype(values.dtype), field.dtype) - - fd = field.expected.DESCRIPTOR.fields_by_name[field.name] - - # Values has the same shape as the input plus an extra - # dimension for repeats. - self.assertEqual(list(values.shape)[:-1], batch_shape) - - # Nested messages are represented as TF strings, requiring - # some special handling. - if field.name == 'message_value': - vs = [] - for buf in values.flat: - msg = test_example_pb2.PrimitiveValue() - msg.ParseFromString(buf) - vs.append(msg) - evs = getattr(field.expected, field.name) - if len(vs) != len(evs): - self.fail('Field %s decoded %d outputs, expected %d' % - (fd.name, len(vs), len(evs))) - for v, ev in zip(vs, evs): - self.assertEqual(v, ev) - continue - - # This can be a little confusing. For testing we are using - # RepeatedPrimitiveValue in two ways: it's the proto that we - # decode for testing, and it's used in the expected value as a - # union type. The two cases are slightly different: this is the - # second case. - # We may be fetching the uint64_value from the test proto, but - # in the expected proto we store it in the int64_value field - # because TensorFlow doesn't support unsigned int64. - tf_type_to_primitive_value_field = { - dtypes.float32: - 'float_value', - dtypes.float64: - 'double_value', - dtypes.int32: - 'int32_value', - dtypes.uint8: - 'uint8_value', - dtypes.int8: - 'int8_value', - dtypes.string: - 'string_value', - dtypes.int64: - 'int64_value', - dtypes.bool: - 'bool_value', - # Unhandled TensorFlow types: - # DT_INT16 DT_COMPLEX64 DT_QINT8 DT_QUINT8 DT_QINT32 - # DT_BFLOAT16 DT_QINT16 DT_QUINT16 DT_UINT16 - } - tf_field_name = tf_type_to_primitive_value_field.get(field.dtype) - if tf_field_name is None: - self.fail('Unhandled tensorflow type %d' % field.dtype) - - self._compareValues(fd, values.flat, - getattr(field.expected, tf_field_name)) - - def _runDecodeProtoTests(self, fields, case_sizes, batch_shape, batch, - message_type, message_format, sanitize, - force_disordered=False): - """Run decode tests on a batch of messages. - - Args: - fields: list of test_example_pb2.FieldSpec (types and expected values) - case_sizes: expected sizes array - batch_shape: the shape of the input tensor of serialized messages - batch: list of serialized messages - message_type: descriptor name for messages - message_format: format of messages, 'text' or 'binary' - sanitize: whether to sanitize binary protobuf inputs - force_disordered: whether to force fields encoded out of order. - """ - - if force_disordered: - # Exercise code path that handles out-of-order fields by prepending extra - # fields with tag numbers higher than any real field. Note that this won't - # work with sanitization because that forces reserialization using a - # trusted decoder and encoder. - assert not sanitize - extra_fields = test_example_pb2.ExtraFields() - extra_fields.string_value = 'IGNORE ME' - extra_fields.bool_value = False - extra_msg = extra_fields.SerializeToString() - batch = [extra_msg + msg for msg in batch] - - # Numpy silently truncates the strings if you don't specify dtype=object. - batch = np.array(batch, dtype=object) - batch = np.reshape(batch, batch_shape) - - field_names = [f.name for f in fields] - output_types = [f.dtype for f in fields] - - with self.test_session() as sess: - sizes, vtensor = proto.decode_proto( - batch, - message_type=message_type, - field_names=field_names, - output_types=output_types, - message_format=message_format, - sanitize=sanitize) - - vlist = sess.run([sizes] + vtensor) - sizes = vlist[0] - # Values is a list of tensors, one for each field. - value_tensors = vlist[1:] - - # Check that the repeat sizes are correct. - self.assertTrue( - np.all(np.array(sizes.shape) == batch_shape + [len(field_names)])) - - # Check that the decoded sizes match the expected sizes. - self.assertEqual(len(sizes.flat), len(case_sizes)) - self.assertTrue( - np.all(sizes.flat == np.array( - case_sizes, dtype=np.int32))) - - field_dict = dict(zip(field_names, value_tensors)) - - self._compareRepeatedPrimitiveValue(batch_shape, sizes, fields, - field_dict) - - def testBinary(self): - with open(FLAGS.message_text_file, 'r') as fp: - case = text_format.Parse(fp.read(), test_example_pb2.TestCase()) - - batch = [primitive.SerializeToString() for primitive in case.primitive] - self._runDecodeProtoTests( - case.field, - case.sizes, - list(case.shape), - batch, - 'tensorflow.contrib.proto.RepeatedPrimitiveValue', - 'binary', - sanitize=False) - - def testBinaryDisordered(self): - with open(FLAGS.message_text_file, 'r') as fp: - case = text_format.Parse(fp.read(), test_example_pb2.TestCase()) - - batch = [primitive.SerializeToString() for primitive in case.primitive] - self._runDecodeProtoTests( - case.field, - case.sizes, - list(case.shape), - batch, - 'tensorflow.contrib.proto.RepeatedPrimitiveValue', - 'binary', - sanitize=False, - force_disordered=True) - - def testPacked(self): - with open(FLAGS.message_text_file, 'r') as fp: - case = text_format.Parse(fp.read(), test_example_pb2.TestCase()) - - # Now try with the packed serialization. - # We test the packed representations by loading the same test cases - # using PackedPrimitiveValue instead of RepeatedPrimitiveValue. - # To do this we rely on the text format being the same for packed and - # unpacked fields, and reparse the test message using the packed version - # of the proto. - packed_batch = [ - # Note: float_format='.17g' is necessary to ensure preservation of - # doubles and floats in text format. - text_format.Parse( - text_format.MessageToString( - primitive, float_format='.17g'), - test_example_pb2.PackedPrimitiveValue()).SerializeToString() - for primitive in case.primitive - ] - - self._runDecodeProtoTests( - case.field, - case.sizes, - list(case.shape), - packed_batch, - 'tensorflow.contrib.proto.PackedPrimitiveValue', - 'binary', - sanitize=False) - - def testText(self): - with open(FLAGS.message_text_file, 'r') as fp: - case = text_format.Parse(fp.read(), test_example_pb2.TestCase()) - - # Note: float_format='.17g' is necessary to ensure preservation of - # doubles and floats in text format. - text_batch = [ - text_format.MessageToString( - primitive, float_format='.17g') for primitive in case.primitive - ] - - self._runDecodeProtoTests( - case.field, - case.sizes, - list(case.shape), - text_batch, - 'tensorflow.contrib.proto.RepeatedPrimitiveValue', - 'text', - sanitize=False) - - def testSanitizerGood(self): - with open(FLAGS.message_text_file, 'r') as fp: - case = text_format.Parse(fp.read(), test_example_pb2.TestCase()) - - batch = [primitive.SerializeToString() for primitive in case.primitive] - self._runDecodeProtoTests( - case.field, - case.sizes, - list(case.shape), - batch, - 'tensorflow.contrib.proto.RepeatedPrimitiveValue', - 'binary', - sanitize=True) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test.py b/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test.py deleted file mode 100644 index 2a24c3b8ce..0000000000 --- a/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test.py +++ /dev/null @@ -1,179 +0,0 @@ -# ============================================================================= -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= -"""Table-driven test for encode_proto op. - -This test is run once with each of the *.TestCase.pbtxt files -in the test directory. - -It tests that encode_proto is a lossless inverse of decode_proto -(for the specified fields). -""" -# Python3 readiness boilerplate -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from google.protobuf import text_format - -from tensorflow.contrib import proto -from tensorflow.contrib.proto.python.kernel_tests import test_case -from tensorflow.contrib.proto.python.kernel_tests import test_example_pb2 -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import flags -from tensorflow.python.platform import test - -FLAGS = flags.FLAGS - -flags.DEFINE_string('message_text_file', None, - 'A file containing a text serialized TestCase protobuf.') - - -class EncodeProtoOpTest(test_case.ProtoOpTestCase): - - def testBadInputs(self): - # Invalid field name - with self.test_session(): - with self.assertRaisesOpError('Unknown field: non_existent_field'): - proto.encode_proto( - sizes=[[1]], - values=[np.array([[0.0]], dtype=np.int32)], - message_type='tensorflow.contrib.proto.RepeatedPrimitiveValue', - field_names=['non_existent_field']).eval() - - # Incorrect types. - with self.test_session(): - with self.assertRaisesOpError( - 'Incompatible type for field double_value.'): - proto.encode_proto( - sizes=[[1]], - values=[np.array([[0.0]], dtype=np.int32)], - message_type='tensorflow.contrib.proto.RepeatedPrimitiveValue', - field_names=['double_value']).eval() - - # Incorrect shapes of sizes. - with self.test_session(): - with self.assertRaisesOpError( - r'sizes should be batch_size \+ \[len\(field_names\)\]'): - sizes = array_ops.placeholder(dtypes.int32) - values = array_ops.placeholder(dtypes.float64) - proto.encode_proto( - sizes=sizes, - values=[values], - message_type='tensorflow.contrib.proto.RepeatedPrimitiveValue', - field_names=['double_value']).eval(feed_dict={ - sizes: [[[0, 0]]], - values: [[0.0]] - }) - - # Inconsistent shapes of values. - with self.test_session(): - with self.assertRaisesOpError( - 'Values must match up to the last dimension'): - sizes = array_ops.placeholder(dtypes.int32) - values1 = array_ops.placeholder(dtypes.float64) - values2 = array_ops.placeholder(dtypes.int32) - (proto.encode_proto( - sizes=[[1, 1]], - values=[values1, values2], - message_type='tensorflow.contrib.proto.RepeatedPrimitiveValue', - field_names=['double_value', 'int32_value']).eval(feed_dict={ - values1: [[0.0]], - values2: [[0], [0]] - })) - - def _testRoundtrip(self, in_bufs, message_type, fields): - - field_names = [f.name for f in fields] - out_types = [f.dtype for f in fields] - - with self.test_session() as sess: - sizes, field_tensors = proto.decode_proto( - in_bufs, - message_type=message_type, - field_names=field_names, - output_types=out_types) - - out_tensors = proto.encode_proto( - sizes, - field_tensors, - message_type=message_type, - field_names=field_names) - - out_bufs, = sess.run([out_tensors]) - - # Check that the re-encoded tensor has the same shape. - self.assertEqual(in_bufs.shape, out_bufs.shape) - - # Compare the input and output. - for in_buf, out_buf in zip(in_bufs.flat, out_bufs.flat): - in_obj = test_example_pb2.RepeatedPrimitiveValue() - in_obj.ParseFromString(in_buf) - - out_obj = test_example_pb2.RepeatedPrimitiveValue() - out_obj.ParseFromString(out_buf) - - # Check that the deserialized objects are identical. - self.assertEqual(in_obj, out_obj) - - # Check that the input and output serialized messages are identical. - # If we fail here, there is a difference in the serialized - # representation but the new serialization still parses. This could - # be harmless (a change in map ordering?) or it could be bad (e.g. - # loss of packing in the encoding). - self.assertEqual(in_buf, out_buf) - - def testRoundtrip(self): - with open(FLAGS.message_text_file, 'r') as fp: - case = text_format.Parse(fp.read(), test_example_pb2.TestCase()) - - in_bufs = [primitive.SerializeToString() for primitive in case.primitive] - - # np.array silently truncates strings if you don't specify dtype=object. - in_bufs = np.reshape(np.array(in_bufs, dtype=object), list(case.shape)) - return self._testRoundtrip( - in_bufs, 'tensorflow.contrib.proto.RepeatedPrimitiveValue', case.field) - - def testRoundtripPacked(self): - with open(FLAGS.message_text_file, 'r') as fp: - case = text_format.Parse(fp.read(), test_example_pb2.TestCase()) - - # Now try with the packed serialization. - # We test the packed representations by loading the same test cases - # using PackedPrimitiveValue instead of RepeatedPrimitiveValue. - # To do this we rely on the text format being the same for packed and - # unpacked fields, and reparse the test message using the packed version - # of the proto. - in_bufs = [ - # Note: float_format='.17g' is necessary to ensure preservation of - # doubles and floats in text format. - text_format.Parse( - text_format.MessageToString( - primitive, float_format='.17g'), - test_example_pb2.PackedPrimitiveValue()).SerializeToString() - for primitive in case.primitive - ] - - # np.array silently truncates strings if you don't specify dtype=object. - in_bufs = np.reshape(np.array(in_bufs, dtype=object), list(case.shape)) - return self._testRoundtrip( - in_bufs, 'tensorflow.contrib.proto.PackedPrimitiveValue', case.field) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/proto/python/kernel_tests/minmax.TestCase.pbtxt b/tensorflow/contrib/proto/python/kernel_tests/minmax.TestCase.pbtxt deleted file mode 100644 index b170f89c0f..0000000000 --- a/tensorflow/contrib/proto/python/kernel_tests/minmax.TestCase.pbtxt +++ /dev/null @@ -1,161 +0,0 @@ -primitive { - double_value: -1.7976931348623158e+308 - double_value: 2.2250738585072014e-308 - double_value: 1.7976931348623158e+308 - float_value: -3.402823466e+38 - float_value: 1.175494351e-38 - float_value: 3.402823466e+38 - int64_value: -9223372036854775808 - int64_value: 9223372036854775807 - uint64_value: 0 - uint64_value: 18446744073709551615 - int32_value: -2147483648 - int32_value: 2147483647 - fixed64_value: 0 - fixed64_value: 18446744073709551615 - fixed32_value: 0 - fixed32_value: 4294967295 - bool_value: false - bool_value: true - string_value: "" - string_value: "I refer to the infinite." - uint32_value: 0 - uint32_value: 4294967295 - sfixed32_value: -2147483648 - sfixed32_value: 2147483647 - sfixed64_value: -9223372036854775808 - sfixed64_value: 9223372036854775807 - sint32_value: -2147483648 - sint32_value: 2147483647 - sint64_value: -9223372036854775808 - sint64_value: 9223372036854775807 -} -shape: 1 -sizes: 3 -sizes: 3 -sizes: 2 -sizes: 2 -sizes: 2 -sizes: 2 -sizes: 2 -sizes: 2 -sizes: 2 -sizes: 2 -sizes: 2 -sizes: 2 -sizes: 2 -sizes: 2 -field { - name: "double_value" - dtype: DT_DOUBLE - expected { - double_value: -1.7976931348623158e+308 - double_value: 2.2250738585072014e-308 - double_value: 1.7976931348623158e+308 - } -} -field { - name: "float_value" - dtype: DT_FLOAT - expected { - float_value: -3.402823466e+38 - float_value: 1.175494351e-38 - float_value: 3.402823466e+38 - } -} -field { - name: "int64_value" - dtype: DT_INT64 - expected { - int64_value: -9223372036854775808 - int64_value: 9223372036854775807 - } -} -field { - name: "uint64_value" - dtype: DT_INT64 - expected { - int64_value: 0 - int64_value: -1 - } -} -field { - name: "int32_value" - dtype: DT_INT32 - expected { - int32_value: -2147483648 - int32_value: 2147483647 - } -} -field { - name: "fixed64_value" - dtype: DT_INT64 - expected { - int64_value: 0 - int64_value: -1 # unsigned is 18446744073709551615 - } -} -field { - name: "fixed32_value" - dtype: DT_INT32 - expected { - int32_value: 0 - int32_value: -1 # unsigned is 4294967295 - } -} -field { - name: "bool_value" - dtype: DT_BOOL - expected { - bool_value: false - bool_value: true - } -} -field { - name: "string_value" - dtype: DT_STRING - expected { - string_value: "" - string_value: "I refer to the infinite." - } -} -field { - name: "uint32_value" - dtype: DT_INT32 - expected { - int32_value: 0 - int32_value: -1 # unsigned is 4294967295 - } -} -field { - name: "sfixed32_value" - dtype: DT_INT32 - expected { - int32_value: -2147483648 - int32_value: 2147483647 - } -} -field { - name: "sfixed64_value" - dtype: DT_INT64 - expected { - int64_value: -9223372036854775808 - int64_value: 9223372036854775807 - } -} -field { - name: "sint32_value" - dtype: DT_INT32 - expected { - int32_value: -2147483648 - int32_value: 2147483647 - } -} -field { - name: "sint64_value" - dtype: DT_INT64 - expected { - int64_value: -9223372036854775808 - int64_value: 9223372036854775807 - } -} diff --git a/tensorflow/contrib/proto/python/kernel_tests/nested.TestCase.pbtxt b/tensorflow/contrib/proto/python/kernel_tests/nested.TestCase.pbtxt deleted file mode 100644 index c664e52851..0000000000 --- a/tensorflow/contrib/proto/python/kernel_tests/nested.TestCase.pbtxt +++ /dev/null @@ -1,16 +0,0 @@ -primitive { - message_value { - double_value: 23.5 - } -} -shape: 1 -sizes: 1 -field { - name: "message_value" - dtype: DT_STRING - expected { - message_value { - double_value: 23.5 - } - } -} diff --git a/tensorflow/contrib/proto/python/kernel_tests/optional.TestCase.pbtxt b/tensorflow/contrib/proto/python/kernel_tests/optional.TestCase.pbtxt deleted file mode 100644 index 125651d7ea..0000000000 --- a/tensorflow/contrib/proto/python/kernel_tests/optional.TestCase.pbtxt +++ /dev/null @@ -1,20 +0,0 @@ -primitive { - bool_value: true -} -shape: 1 -sizes: 1 -sizes: 0 -field { - name: "bool_value" - dtype: DT_BOOL - expected { - bool_value: true - } -} -field { - name: "double_value" - dtype: DT_DOUBLE - expected { - double_value: 0.0 - } -} diff --git a/tensorflow/contrib/proto/python/kernel_tests/promote_unsigned.TestCase.pbtxt b/tensorflow/contrib/proto/python/kernel_tests/promote_unsigned.TestCase.pbtxt deleted file mode 100644 index db7555bf2d..0000000000 --- a/tensorflow/contrib/proto/python/kernel_tests/promote_unsigned.TestCase.pbtxt +++ /dev/null @@ -1,21 +0,0 @@ -primitive { - fixed32_value: 4294967295 - uint32_value: 4294967295 -} -shape: 1 -sizes: 1 -sizes: 1 -field { - name: "fixed32_value" - dtype: DT_INT64 - expected { - int64_value: 4294967295 - } -} -field { - name: "uint32_value" - dtype: DT_INT64 - expected { - int64_value: 4294967295 - } -} diff --git a/tensorflow/contrib/proto/python/kernel_tests/ragged.TestCase.pbtxt b/tensorflow/contrib/proto/python/kernel_tests/ragged.TestCase.pbtxt deleted file mode 100644 index 61c7ac53f7..0000000000 --- a/tensorflow/contrib/proto/python/kernel_tests/ragged.TestCase.pbtxt +++ /dev/null @@ -1,32 +0,0 @@ -primitive { - double_value: 23.5 - double_value: 123.0 - bool_value: true -} -primitive { - double_value: 3.1 - bool_value: false -} -shape: 2 -sizes: 2 -sizes: 1 -sizes: 1 -sizes: 1 -field { - name: "double_value" - dtype: DT_DOUBLE - expected { - double_value: 23.5 - double_value: 123.0 - double_value: 3.1 - double_value: 0.0 - } -} -field { - name: "bool_value" - dtype: DT_BOOL - expected { - bool_value: true - bool_value: false - } -} diff --git a/tensorflow/contrib/proto/python/kernel_tests/shaped_batch.TestCase.pbtxt b/tensorflow/contrib/proto/python/kernel_tests/shaped_batch.TestCase.pbtxt deleted file mode 100644 index f4828076d5..0000000000 --- a/tensorflow/contrib/proto/python/kernel_tests/shaped_batch.TestCase.pbtxt +++ /dev/null @@ -1,62 +0,0 @@ -primitive { - double_value: 23.5 - bool_value: true -} -primitive { - double_value: 44.0 - bool_value: false -} -primitive { - double_value: 3.14159 - bool_value: true -} -primitive { - double_value: 1.414 - bool_value: true -} -primitive { - double_value: -32.2 - bool_value: false -} -primitive { - double_value: 0.0001 - bool_value: true -} -shape: 3 -shape: 2 -sizes: 1 -sizes: 1 -sizes: 1 -sizes: 1 -sizes: 1 -sizes: 1 -sizes: 1 -sizes: 1 -sizes: 1 -sizes: 1 -sizes: 1 -sizes: 1 -field { - name: "double_value" - dtype: DT_DOUBLE - expected { - double_value: 23.5 - double_value: 44.0 - double_value: 3.14159 - double_value: 1.414 - double_value: -32.2 - double_value: 0.0001 - } -} -field { - name: "bool_value" - dtype: DT_BOOL - expected { - bool_value: true - bool_value: false - bool_value: true - bool_value: true - bool_value: false - bool_value: true - } -} diff --git a/tensorflow/contrib/proto/python/kernel_tests/simple.TestCase.pbtxt b/tensorflow/contrib/proto/python/kernel_tests/simple.TestCase.pbtxt deleted file mode 100644 index dc20ac147b..0000000000 --- a/tensorflow/contrib/proto/python/kernel_tests/simple.TestCase.pbtxt +++ /dev/null @@ -1,21 +0,0 @@ -primitive { - double_value: 23.5 - bool_value: true -} -shape: 1 -sizes: 1 -sizes: 1 -field { - name: "double_value" - dtype: DT_DOUBLE - expected { - double_value: 23.5 - } -} -field { - name: "bool_value" - dtype: DT_BOOL - expected { - bool_value: true - } -} diff --git a/tensorflow/contrib/proto/python/kernel_tests/test_case.py b/tensorflow/contrib/proto/python/kernel_tests/test_case.py deleted file mode 100644 index b95202c5df..0000000000 --- a/tensorflow/contrib/proto/python/kernel_tests/test_case.py +++ /dev/null @@ -1,35 +0,0 @@ -# ============================================================================= -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= -"""Test case base for testing proto operations.""" - -# Python3 preparedness imports. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import ctypes as ct -import os - -from tensorflow.python.platform import test - - -class ProtoOpTestCase(test.TestCase): - - def __init__(self, methodName='runTest'): # pylint: disable=invalid-name - super(ProtoOpTestCase, self).__init__(methodName) - lib = os.path.join(os.path.dirname(__file__), 'libtestexample.so') - if os.path.isfile(lib): - ct.cdll.LoadLibrary(lib) diff --git a/tensorflow/contrib/proto/python/kernel_tests/test_example.proto b/tensorflow/contrib/proto/python/kernel_tests/test_example.proto deleted file mode 100644 index dc495034ff..0000000000 --- a/tensorflow/contrib/proto/python/kernel_tests/test_example.proto +++ /dev/null @@ -1,149 +0,0 @@ -// Test description and protos to work with it. -// -// Many of the protos in this file are for unit tests that haven't been written yet. - -syntax = "proto2"; - -import "tensorflow/core/framework/types.proto"; - -package tensorflow.contrib.proto; - -// A TestCase holds a proto and a bunch of assertions -// about how it should decode. -message TestCase { - // A batch of primitives to be serialized and decoded. - repeated RepeatedPrimitiveValue primitive = 1; - // The shape of the batch. - repeated int32 shape = 2; - // Expected sizes for each field. - repeated int32 sizes = 3; - // Expected values for each field. - repeated FieldSpec field = 4; -}; - -// FieldSpec describes the expected output for a single field. -message FieldSpec { - optional string name = 1; - optional tensorflow.DataType dtype = 2; - optional RepeatedPrimitiveValue expected = 3; -}; - -message TestValue { - optional PrimitiveValue primitive_value = 1; - optional EnumValue enum_value = 2; - optional MessageValue message_value = 3; - optional RepeatedMessageValue repeated_message_value = 4; - optional RepeatedPrimitiveValue repeated_primitive_value = 6; -} - -message PrimitiveValue { - optional double double_value = 1; - optional float float_value = 2; - optional int64 int64_value = 3; - optional uint64 uint64_value = 4; - optional int32 int32_value = 5; - optional fixed64 fixed64_value = 6; - optional fixed32 fixed32_value = 7; - optional bool bool_value = 8; - optional string string_value = 9; - optional bytes bytes_value = 12; - optional uint32 uint32_value = 13; - optional sfixed32 sfixed32_value = 15; - optional sfixed64 sfixed64_value = 16; - optional sint32 sint32_value = 17; - optional sint64 sint64_value = 18; -} - -// NOTE: This definition must be kept in sync with PackedPrimitiveValue. -message RepeatedPrimitiveValue { - repeated double double_value = 1; - repeated float float_value = 2; - repeated int64 int64_value = 3; - repeated uint64 uint64_value = 4; - repeated int32 int32_value = 5; - repeated fixed64 fixed64_value = 6; - repeated fixed32 fixed32_value = 7; - repeated bool bool_value = 8; - repeated string string_value = 9; - repeated bytes bytes_value = 12; - repeated uint32 uint32_value = 13; - repeated sfixed32 sfixed32_value = 15; - repeated sfixed64 sfixed64_value = 16; - repeated sint32 sint32_value = 17; - repeated sint64 sint64_value = 18; - repeated PrimitiveValue message_value = 19; -} - -// A PackedPrimitiveValue looks exactly the same as a RepeatedPrimitiveValue -// in the text format, but the binary serializion is different. -// We test the packed representations by loading the same test cases -// using this definition instead of RepeatedPrimitiveValue. -// NOTE: This definition must be kept in sync with RepeatedPrimitiveValue -// in every way except the packed=true declaration. -message PackedPrimitiveValue { - repeated double double_value = 1 [packed = true]; - repeated float float_value = 2 [packed = true]; - repeated int64 int64_value = 3 [packed = true]; - repeated uint64 uint64_value = 4 [packed = true]; - repeated int32 int32_value = 5 [packed = true]; - repeated fixed64 fixed64_value = 6 [packed = true]; - repeated fixed32 fixed32_value = 7 [packed = true]; - repeated bool bool_value = 8 [packed = true]; - repeated string string_value = 9; - repeated bytes bytes_value = 12; - repeated uint32 uint32_value = 13 [packed = true]; - repeated sfixed32 sfixed32_value = 15 [packed = true]; - repeated sfixed64 sfixed64_value = 16 [packed = true]; - repeated sint32 sint32_value = 17 [packed = true]; - repeated sint64 sint64_value = 18 [packed = true]; - repeated PrimitiveValue message_value = 19; -} - -message EnumValue { - enum Color { - RED = 0; - ORANGE = 1; - YELLOW = 2; - GREEN = 3; - BLUE = 4; - INDIGO = 5; - VIOLET = 6; - }; - optional Color enum_value = 14; - repeated Color repeated_enum_value = 15; -} - - -message InnerMessageValue { - optional float float_value = 2; - repeated bytes bytes_values = 8; -} - -message MiddleMessageValue { - repeated int32 int32_values = 5; - optional InnerMessageValue message_value = 11; - optional uint32 uint32_value = 13; -} - -message MessageValue { - optional double double_value = 1; - optional MiddleMessageValue message_value = 11; -} - -message RepeatedMessageValue { - message NestedMessageValue { - optional float float_value = 2; - repeated bytes bytes_values = 8; - } - - repeated NestedMessageValue message_values = 11; -} - -// Message containing fields with field numbers higher than any field above. An -// instance of this message is prepended to each binary message in the test to -// exercise the code path that handles fields encoded out of order of field -// number. -message ExtraFields { - optional string string_value = 1776; - optional bool bool_value = 1777; -} diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD index 2ef105755f..679d2735f9 100644 --- a/tensorflow/tools/pip_package/BUILD +++ b/tensorflow/tools/pip_package/BUILD @@ -74,7 +74,6 @@ COMMON_PIP_DEPS = [ "//tensorflow/contrib/labeled_tensor:labeled_tensor_pip", "//tensorflow/contrib/nn:nn_py", "//tensorflow/contrib/predictor:predictor_pip", - "//tensorflow/contrib/proto:proto_pip", "//tensorflow/contrib/receptive_field:receptive_field_pip", "//tensorflow/contrib/rpc:rpc_pip", "//tensorflow/contrib/session_bundle:session_bundle_pip", -- GitLab From 1c88fac05afbce5aa1131c87f0594f9f0f1b6706 Mon Sep 17 00:00:00 2001 From: Jiri Simsa Date: Thu, 12 Apr 2018 21:39:26 -0700 Subject: [PATCH 305/791] Automated g4 rollback of changelist 192698931 PiperOrigin-RevId: 192718697 --- tensorflow/contrib/BUILD | 1 - tensorflow/contrib/__init__.py | 1 - tensorflow/contrib/cmake/tf_python.cmake | 3 +- tensorflow/contrib/rpc/BUILD | 16 - .../contrib/rpc/python/kernel_tests/BUILD | 76 ---- .../rpc/python/kernel_tests/rpc_op_test.py | 71 ---- .../python/kernel_tests/rpc_op_test_base.py | 337 ------------------ .../kernel_tests/rpc_op_test_servicer.py | 101 ------ .../python/kernel_tests/test_example.proto | 171 --------- .../core/platform/default/build_config.bzl | 86 +---- tensorflow/tools/pip_package/BUILD | 1 - tensorflow/workspace.bzl | 4 - 12 files changed, 4 insertions(+), 864 deletions(-) delete mode 100644 tensorflow/contrib/rpc/python/kernel_tests/BUILD delete mode 100644 tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test.py delete mode 100644 tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_base.py delete mode 100644 tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_servicer.py delete mode 100644 tensorflow/contrib/rpc/python/kernel_tests/test_example.proto diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD index 192d053683..9bef0d8b61 100644 --- a/tensorflow/contrib/BUILD +++ b/tensorflow/contrib/BUILD @@ -86,7 +86,6 @@ py_library( "//tensorflow/contrib/remote_fused_graph/pylib:remote_fused_graph_ops_py", "//tensorflow/contrib/resampler:resampler_py", "//tensorflow/contrib/rnn:rnn_py", - "//tensorflow/contrib/rpc", "//tensorflow/contrib/saved_model:saved_model_py", "//tensorflow/contrib/seq2seq:seq2seq_py", "//tensorflow/contrib/signal:signal_py", diff --git a/tensorflow/contrib/__init__.py b/tensorflow/contrib/__init__.py index e02dd5e759..aaddb06fa0 100644 --- a/tensorflow/contrib/__init__.py +++ b/tensorflow/contrib/__init__.py @@ -70,7 +70,6 @@ from tensorflow.contrib import recurrent from tensorflow.contrib import reduce_slice_ops from tensorflow.contrib import resampler from tensorflow.contrib import rnn -from tensorflow.contrib import rpc from tensorflow.contrib import saved_model from tensorflow.contrib import seq2seq from tensorflow.contrib import signal diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake index 9d9db82513..ded15b4b66 100755 --- a/tensorflow/contrib/cmake/tf_python.cmake +++ b/tensorflow/contrib/cmake/tf_python.cmake @@ -345,8 +345,7 @@ GENERATE_PYTHON_OP_LIB("random_ops") GENERATE_PYTHON_OP_LIB("remote_fused_graph_ops" DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/remote_fused_graph/pylib/python/ops/gen_remote_fused_graph_ops.py) GENERATE_PYTHON_OP_LIB("resource_variable_ops") -GENERATE_PYTHON_OP_LIB("rpc_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/rpc/python/ops/gen_rpc_op.py) +GENERATE_PYTHON_OP_LIB("rpc_ops") GENERATE_PYTHON_OP_LIB("script_ops") GENERATE_PYTHON_OP_LIB("sdca_ops") GENERATE_PYTHON_OP_LIB("set_ops") diff --git a/tensorflow/contrib/rpc/BUILD b/tensorflow/contrib/rpc/BUILD index dbd311a276..597f18c771 100644 --- a/tensorflow/contrib/rpc/BUILD +++ b/tensorflow/contrib/rpc/BUILD @@ -4,8 +4,6 @@ licenses(["notice"]) # Apache 2.0 exports_files(["LICENSE"]) -load("//tensorflow/core:platform/default/build_config_root.bzl", "if_static") - py_library( name = "rpc", srcs = [ @@ -13,17 +11,3 @@ py_library( ], deps = ["//tensorflow/contrib/rpc/python/ops:rpc_op_py"], ) - -py_library( - name = "rpc_pip", - data = if_static( - [], - otherwise = ["//tensorflow/contrib/rpc/python/kernel_tests:libtestexample.so"], - ), - deps = [ - ":rpc", - "//tensorflow/contrib/rpc/python/kernel_tests:py_test_deps", - "//tensorflow/contrib/rpc/python/kernel_tests:rpc_op_test_base", - "//tensorflow/contrib/rpc/python/kernel_tests:rpc_op_test_servicer", - ], -) diff --git a/tensorflow/contrib/rpc/python/kernel_tests/BUILD b/tensorflow/contrib/rpc/python/kernel_tests/BUILD deleted file mode 100644 index 08ec1e61a4..0000000000 --- a/tensorflow/contrib/rpc/python/kernel_tests/BUILD +++ /dev/null @@ -1,76 +0,0 @@ -# TODO(b/76425722): Port everything in here to OS (currently excluded). - -package(default_visibility = ["//visibility:public"]) - -licenses(["notice"]) # Apache 2.0 - -exports_files(["LICENSE"]) - -# Placeholder for loading internal BUILD rule. -load("//tensorflow:tensorflow.bzl", "tf_py_test") -load("//tensorflow:tensorflow.bzl", "tf_cc_shared_object") -load("//tensorflow/core:platform/default/build_config_root.bzl", "if_static") -load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library") - -tf_proto_library( - name = "test_example_proto", - srcs = ["test_example.proto"], - has_services = 1, - cc_api_version = 2, - protodeps = ["//tensorflow/core:protos_all"], -) - -py_library( - name = "py_test_deps", - deps = [":test_example_proto_py"], -) - -py_library( - name = "rpc_op_test_base", - srcs = ["rpc_op_test_base.py"], - deps = [ - ":test_example_proto_py", - "//tensorflow/contrib/proto", - "//tensorflow/contrib/rpc", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:dtypes", - "//tensorflow/python:errors", - "//third_party/py/numpy", - ], -) - -py_library( - name = "rpc_op_test_servicer", - srcs = ["rpc_op_test_servicer.py"], - deps = [ - ":py_test_deps", - ":rpc_op_test_base", - "//tensorflow/core:protos_all_py", - "//third_party/py/numpy", - ], -) - -tf_cc_shared_object( - name = "libtestexample.so", - linkstatic = 1, - deps = [ - ":test_example_proto_cc", - ], -) - -tf_py_test( - name = "rpc_op_test", - size = "small", - srcs = ["rpc_op_test.py"], - additional_deps = [ - ":py_test_deps", - ":rpc_op_test_base", - ":rpc_op_test_servicer", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:client_testlib", - ], - data = if_static( - [], - otherwise = [":libtestexample.so"], - ), -) diff --git a/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test.py b/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test.py deleted file mode 100644 index e2e0dbc7a2..0000000000 --- a/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -"""Tests for RpcOp.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import ctypes as ct -import os - -import grpc -from grpc.framework.foundation import logging_pool -import portpicker - -from tensorflow.contrib.rpc.python.kernel_tests import rpc_op_test_base -from tensorflow.contrib.rpc.python.kernel_tests import rpc_op_test_servicer -from tensorflow.contrib.rpc.python.kernel_tests import test_example_pb2_grpc -from tensorflow.python.platform import test - - -class RpcOpTest(test.TestCase, rpc_op_test_base.RpcOpTestBase): - _protocol = 'grpc' - - invalid_method_string = 'Method not found' - - def __init__(self, methodName='runTest'): # pylint: disable=invalid-name - super(RpcOpTest, self).__init__(methodName) - lib = os.path.join(os.path.dirname(__file__), 'libtestexample.so') - if os.path.isfile(lib): - ct.cdll.LoadLibrary(lib) - - def get_method_name(self, suffix): - return '/tensorflow.contrib.rpc.TestCaseService/%s' % suffix - - def setUp(self): - super(RpcOpTest, self).setUp() - - service_port = portpicker.pick_unused_port() - - server = grpc.server(logging_pool.pool(max_workers=25)) - servicer = rpc_op_test_servicer.RpcOpTestServicer() - test_example_pb2_grpc.add_TestCaseServiceServicer_to_server( - servicer, server) - self._address = 'localhost:%d' % service_port - server.add_insecure_port(self._address) - server.start() - self._server = server - - def tearDown(self): - # TODO(ebrevdo): Figure out why this sometimes times out. - # self._service.ExitLoop() - # self._service_thread.join() - # self._server.stop() - super(RpcOpTest, self).tearDown() - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_base.py b/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_base.py deleted file mode 100644 index aa03a103ed..0000000000 --- a/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_base.py +++ /dev/null @@ -1,337 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -"""Base class for RpcOp tests.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import itertools - -import numpy as np - -from tensorflow.contrib.proto import decode_proto -from tensorflow.contrib.proto import encode_proto -from tensorflow.contrib.rpc import rpc -from tensorflow.contrib.rpc import try_rpc -from tensorflow.contrib.rpc.python.kernel_tests import test_example_pb2 -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors - -__all__ = ['I_WARNED_YOU', 'RpcOpTestBase'] - -I_WARNED_YOU = 'I warned you!' - - -class RpcOpTestBase(object): - # pylint: disable=missing-docstring,invalid-name - """Base class for RpcOp tests.""" - - def get_method_name(self, suffix): - raise NotImplementedError - - def rpc(self, *args, **kwargs): - return rpc(*args, protocol=self._protocol, **kwargs) - - def try_rpc(self, *args, **kwargs): - return try_rpc(*args, protocol=self._protocol, **kwargs) - - def testScalarHostPortRpc(self): - with self.test_session() as sess: - request_tensors = ( - test_example_pb2.TestCase(shape=[1, 2, 3]).SerializeToString()) - response_tensors = self.rpc( - method=self.get_method_name('IncrementTestShapes'), - address=self._address, - request=request_tensors) - self.assertEqual(response_tensors.shape, ()) - response_values = sess.run(response_tensors) - response_message = test_example_pb2.TestCase() - self.assertTrue(response_message.ParseFromString(response_values)) - self.assertAllEqual([2, 3, 4], response_message.shape) - - def testScalarHostPortTryRpc(self): - with self.test_session() as sess: - request_tensors = ( - test_example_pb2.TestCase(shape=[1, 2, 3]).SerializeToString()) - response_tensors, status_code, status_message = self.try_rpc( - method=self.get_method_name('IncrementTestShapes'), - address=self._address, - request=request_tensors) - self.assertEqual(status_code.shape, ()) - self.assertEqual(status_message.shape, ()) - self.assertEqual(response_tensors.shape, ()) - response_values, status_code_values, status_message_values = ( - sess.run((response_tensors, status_code, status_message))) - response_message = test_example_pb2.TestCase() - self.assertTrue(response_message.ParseFromString(response_values)) - self.assertAllEqual([2, 3, 4], response_message.shape) - # For the base Rpc op, don't expect to get error status back. - self.assertEqual(errors.OK, status_code_values) - self.assertEqual(b'', status_message_values) - - def testEmptyHostPortRpc(self): - with self.test_session() as sess: - request_tensors = [] - response_tensors = self.rpc( - method=self.get_method_name('IncrementTestShapes'), - address=self._address, - request=request_tensors) - self.assertAllEqual(response_tensors.shape, [0]) - response_values = sess.run(response_tensors) - self.assertAllEqual(response_values.shape, [0]) - - def testInvalidAddresses(self): - with self.test_session() as sess: - with self.assertRaisesOpError(self.invalid_method_string): - sess.run( - self.rpc( - method='/InvalidService.IncrementTestShapes', - address=self._address, - request='')) - - with self.assertRaisesOpError(self.invalid_method_string): - sess.run( - self.rpc( - method=self.get_method_name('InvalidMethodName'), - address=self._address, - request='')) - - # This also covers the case of address='' - # and address='localhost:293874293874' - with self.assertRaises(errors.UnavailableError): - sess.run( - self.rpc( - method=self.get_method_name('IncrementTestShapes'), - address='unix:/tmp/this_unix_socket_doesnt_exist_97820348!!@', - request='')) - - # Test invalid method with the TryRpc op - _, status_code_value, status_message_value = sess.run( - self.try_rpc( - method=self.get_method_name('InvalidMethodName'), - address=self._address, - request='')) - self.assertEqual(errors.UNIMPLEMENTED, status_code_value) - self.assertTrue( - self.invalid_method_string in status_message_value.decode('ascii')) - - def testAlwaysFailingMethod(self): - with self.test_session() as sess: - response_tensors = self.rpc( - method=self.get_method_name('AlwaysFailWithInvalidArgument'), - address=self._address, - request='') - self.assertEqual(response_tensors.shape, ()) - with self.assertRaisesOpError(I_WARNED_YOU): - sess.run(response_tensors) - - def testSometimesFailingMethodWithManyRequests(self): - with self.test_session() as sess: - # Fail hard by default. - response_tensors = self.rpc( - method=self.get_method_name('SometimesFailWithInvalidArgument'), - address=self._address, - request=[''] * 20) - self.assertEqual(response_tensors.shape, (20,)) - with self.assertRaisesOpError(I_WARNED_YOU): - sess.run(response_tensors) - - # Don't fail hard, use TryRpc - return the failing status instead. - response_tensors, status_code, status_message = self.try_rpc( - method=self.get_method_name('SometimesFailWithInvalidArgument'), - address=self._address, - request=[''] * 20) - self.assertEqual(response_tensors.shape, (20,)) - self.assertEqual(status_code.shape, (20,)) - self.assertEqual(status_message.shape, (20,)) - status_code_values, status_message_values = sess.run((status_code, - status_message)) - self.assertTrue([ - x in (errors.OK, errors.INVALID_ARGUMENT) for x in status_code_values - ]) - expected_message_values = np.where( - status_code_values == errors.INVALID_ARGUMENT, - I_WARNED_YOU.encode('ascii'), b'') - self.assertAllEqual(expected_message_values, status_message_values) - - def testVecHostPortRpc(self): - with self.test_session() as sess: - request_tensors = [ - test_example_pb2.TestCase( - shape=[i, i + 1, i + 2]).SerializeToString() for i in range(20) - ] - response_tensors = self.rpc( - method=self.get_method_name('IncrementTestShapes'), - address=self._address, - request=request_tensors) - self.assertEqual(response_tensors.shape, (20,)) - response_values = sess.run(response_tensors) - self.assertEqual(response_values.shape, (20,)) - for i in range(20): - response_message = test_example_pb2.TestCase() - self.assertTrue(response_message.ParseFromString(response_values[i])) - self.assertAllEqual([i + 1, i + 2, i + 3], response_message.shape) - - def testVecHostPortManyParallelRpcs(self): - with self.test_session() as sess: - request_tensors = [ - test_example_pb2.TestCase( - shape=[i, i + 1, i + 2]).SerializeToString() for i in range(20) - ] - many_response_tensors = [ - self.rpc( - method=self.get_method_name('IncrementTestShapes'), - address=self._address, - request=request_tensors) for _ in range(10) - ] - # Launch parallel 10 calls to the RpcOp, each containing - # 20 rpc requests. - many_response_values = sess.run(many_response_tensors) - self.assertEqual(10, len(many_response_values)) - for response_values in many_response_values: - self.assertEqual(response_values.shape, (20,)) - for i in range(20): - response_message = test_example_pb2.TestCase() - self.assertTrue(response_message.ParseFromString(response_values[i])) - self.assertAllEqual([i + 1, i + 2, i + 3], response_message.shape) - - def testVecHostPortRpcUsingEncodeAndDecodeProto(self): - with self.test_session() as sess: - request_tensors = encode_proto( - message_type='tensorflow.contrib.rpc.TestCase', - field_names=['shape'], - sizes=[[3]] * 20, - values=[ - [[i, i + 1, i + 2] for i in range(20)], - ]) - response_tensor_strings = self.rpc( - method=self.get_method_name('IncrementTestShapes'), - address=self._address, - request=request_tensors) - _, (response_shape,) = decode_proto( - bytes=response_tensor_strings, - message_type='tensorflow.contrib.rpc.TestCase', - field_names=['shape'], - output_types=[dtypes.int32]) - response_shape_values = sess.run(response_shape) - self.assertAllEqual([[i + 1, i + 2, i + 3] - for i in range(20)], response_shape_values) - - def testVecHostPortRpcCancelsUponSessionTimeOutWhenSleepingForever(self): - with self.test_session() as sess: - request_tensors = [''] * 25 # This will launch 25 RPC requests. - response_tensors = self.rpc( - method=self.get_method_name('SleepForever'), - address=self._address, - request=request_tensors) - for timeout_ms in [1, 500, 1000]: - options = config_pb2.RunOptions(timeout_in_ms=timeout_ms) - with self.assertRaises((errors.UnavailableError, - errors.DeadlineExceededError)): - sess.run(response_tensors, options=options) - - def testVecHostPortRpcCancelsUponConfiguredTimeOutWhenSleepingForever(self): - with self.test_session() as sess: - request_tensors = [''] * 25 # This will launch 25 RPC requests. - response_tensors = self.rpc( - method=self.get_method_name('SleepForever'), - address=self._address, - timeout_in_ms=1000, - request=request_tensors) - with self.assertRaises(errors.DeadlineExceededError): - sess.run(response_tensors) - - def testTryRpcPropagatesDeadlineErrorWithSometimesTimingOutRequests(self): - with self.test_session() as sess: - response_tensors, status_code, status_message = self.try_rpc( - method=self.get_method_name('SometimesSleepForever'), - timeout_in_ms=1000, - address=self._address, - request=[''] * 20) - self.assertEqual(response_tensors.shape, (20,)) - self.assertEqual(status_code.shape, (20,)) - self.assertEqual(status_message.shape, (20,)) - status_code_values = sess.run(status_code) - self.assertTrue([ - x in (errors.OK, errors.DEADLINE_EXCEEDED) for x in status_code_values - ]) - - def testTryRpcWithMultipleAddressesSingleRequest(self): - flatten = lambda x: list(itertools.chain.from_iterable(x)) - with self.test_session() as sess: - addresses = flatten([[ - self._address, 'unix:/tmp/this_unix_socket_doesnt_exist_97820348!!@' - ] for _ in range(10)]) - request = test_example_pb2.TestCase(shape=[0, 1, 2]).SerializeToString() - response_tensors, status_code, _ = self.try_rpc( - method=self.get_method_name('IncrementTestShapes'), - address=addresses, - request=request) - response_tensors_values, status_code_values = sess.run((response_tensors, - status_code)) - self.assertAllEqual( - flatten([errors.OK, errors.UNAVAILABLE] for _ in range(10)), - status_code_values) - for i in range(10): - self.assertTrue(response_tensors_values[2 * i]) - self.assertFalse(response_tensors_values[2 * i + 1]) - - def testTryRpcWithMultipleMethodsSingleRequest(self): - flatten = lambda x: list(itertools.chain.from_iterable(x)) - with self.test_session() as sess: - methods = flatten( - [[self.get_method_name('IncrementTestShapes'), 'InvalidMethodName'] - for _ in range(10)]) - request = test_example_pb2.TestCase(shape=[0, 1, 2]).SerializeToString() - response_tensors, status_code, _ = self.try_rpc( - method=methods, address=self._address, request=request) - response_tensors_values, status_code_values = sess.run((response_tensors, - status_code)) - self.assertAllEqual( - flatten([errors.OK, errors.UNIMPLEMENTED] for _ in range(10)), - status_code_values) - for i in range(10): - self.assertTrue(response_tensors_values[2 * i]) - self.assertFalse(response_tensors_values[2 * i + 1]) - - def testTryRpcWithMultipleAddressesAndRequests(self): - flatten = lambda x: list(itertools.chain.from_iterable(x)) - with self.test_session() as sess: - addresses = flatten([[ - self._address, 'unix:/tmp/this_unix_socket_doesnt_exist_97820348!!@' - ] for _ in range(10)]) - requests = [ - test_example_pb2.TestCase( - shape=[i, i + 1, i + 2]).SerializeToString() for i in range(20) - ] - response_tensors, status_code, _ = self.try_rpc( - method=self.get_method_name('IncrementTestShapes'), - address=addresses, - request=requests) - response_tensors_values, status_code_values = sess.run((response_tensors, - status_code)) - self.assertAllEqual( - flatten([errors.OK, errors.UNAVAILABLE] for _ in range(10)), - status_code_values) - for i in range(20): - if i % 2 == 1: - self.assertFalse(response_tensors_values[i]) - else: - response_message = test_example_pb2.TestCase() - self.assertTrue( - response_message.ParseFromString(response_tensors_values[i])) - self.assertAllEqual([i + 1, i + 2, i + 3], response_message.shape) diff --git a/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_servicer.py b/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_servicer.py deleted file mode 100644 index 7cbd636cb1..0000000000 --- a/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_servicer.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -"""Test servicer for RpcOp tests.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import random -import time - -import grpc - -from tensorflow.contrib.rpc.python.kernel_tests import rpc_op_test_base -from tensorflow.contrib.rpc.python.kernel_tests import test_example_pb2_grpc - - -class RpcOpTestServicer(test_example_pb2_grpc.TestCaseServiceServicer): - """Test servicer for RpcOp tests.""" - - def IncrementTestShapes(self, request, context): - """Increment the entries in the shape attribute of request. - - Args: - request: input TestCase. - context: the rpc context. - - Returns: - output TestCase. - """ - for i in range(len(request.shape)): - request.shape[i] += 1 - return request - - def AlwaysFailWithInvalidArgument(self, request, context): - """Always fails with an InvalidArgument status. - - Args: - request: input TestCase. - context: the rpc context. - - Returns: - output TestCase. - """ - del request - context.set_code(grpc.StatusCode.INVALID_ARGUMENT) - context.set_details(rpc_op_test_base.I_WARNED_YOU) - - def SometimesFailWithInvalidArgument(self, request, context): - """Sometimes fails with an InvalidArgument status. - - Args: - request: input TestCase. - context: the rpc context. - - Returns: - output TestCase. - """ - if random.randint(0, 1) == 1: - context.set_code(grpc.StatusCode.INVALID_ARGUMENT) - context.set_details(rpc_op_test_base.I_WARNED_YOU) - return request - - def SleepForever(self, request, context): - """Sleeps forever. - - Args: - request: input TestCase. - context: the rpc context. - - Returns: - output TestCase. - """ - # TODO(ebrevdo): Make this async wait like the stubby version. - time.sleep(5) - - def SometimesSleepForever(self, request, context): - """Sometimes sleeps forever. - - Args: - request: input TestCase. - context: the rpc context. - - Returns: - output TestCase. - """ - if random.randint(0, 1) == 1: - time.sleep(5) - return request diff --git a/tensorflow/contrib/rpc/python/kernel_tests/test_example.proto b/tensorflow/contrib/rpc/python/kernel_tests/test_example.proto deleted file mode 100644 index 96f4550f62..0000000000 --- a/tensorflow/contrib/rpc/python/kernel_tests/test_example.proto +++ /dev/null @@ -1,171 +0,0 @@ -// Test description and protos to work with it. -// -// Many of the protos in this file are for unit tests that haven't been written yet. - -syntax = "proto2"; - -import "tensorflow/core/framework/types.proto"; - -package tensorflow.contrib.rpc; - -// A TestCase holds a proto and a bunch of assertions -// about how it should decode. -message TestCase { - // A batch of primitives to be serialized and decoded. - repeated RepeatedPrimitiveValue primitive = 1; - // The shape of the batch. - repeated int32 shape = 2; - // Expected sizes for each field. - repeated int32 sizes = 3; - // Expected values for each field. - repeated FieldSpec field = 4; -}; - -service TestCaseService { - // Copy input, and increment each entry in 'shape' by 1. - rpc IncrementTestShapes(TestCase) returns (TestCase) { - } - - // Sleep forever. - rpc SleepForever(TestCase) returns (TestCase) { - } - - // Sleep forever 50% of the time, return immediately the other 50%. - rpc SometimesSleepForever(TestCase) returns (TestCase) { - } - - // Always fails with InvalidArgument. - rpc AlwaysFailWithInvalidArgument(TestCase) returns (TestCase) { - } - - // Fails with InvalidArgument 50% of the time. - rpc SometimesFailWithInvalidArgument(TestCase) returns (TestCase) { - } -}; - -// FieldSpec describes the expected output for a single field. -message FieldSpec { - optional string name = 1; - optional tensorflow.DataType dtype = 2; - optional RepeatedPrimitiveValue expected = 3; -}; - -message TestValue { - optional PrimitiveValue primitive_value = 1; - optional EnumValue enum_value = 2; - optional MessageValue message_value = 3; - optional RepeatedMessageValue repeated_message_value = 4; - optional RepeatedPrimitiveValue repeated_primitive_value = 6; -} - -message PrimitiveValue { - optional double double_value = 1; - optional float float_value = 2; - optional int64 int64_value = 3; - optional uint64 uint64_value = 4; - optional int32 int32_value = 5; - optional fixed64 fixed64_value = 6; - optional fixed32 fixed32_value = 7; - optional bool bool_value = 8; - optional string string_value = 9; - optional bytes bytes_value = 12; - optional uint32 uint32_value = 13; - optional sfixed32 sfixed32_value = 15; - optional sfixed64 sfixed64_value = 16; - optional sint32 sint32_value = 17; - optional sint64 sint64_value = 18; -} - -// NOTE: This definition must be kept in sync with PackedPrimitiveValue. -message RepeatedPrimitiveValue { - repeated double double_value = 1; - repeated float float_value = 2; - repeated int64 int64_value = 3; - repeated uint64 uint64_value = 4; - repeated int32 int32_value = 5; - repeated fixed64 fixed64_value = 6; - repeated fixed32 fixed32_value = 7; - repeated bool bool_value = 8; - repeated string string_value = 9; - repeated bytes bytes_value = 12; - repeated uint32 uint32_value = 13; - repeated sfixed32 sfixed32_value = 15; - repeated sfixed64 sfixed64_value = 16; - repeated sint32 sint32_value = 17; - repeated sint64 sint64_value = 18; - repeated PrimitiveValue message_value = 19; -} - -// A PackedPrimitiveValue looks exactly the same as a RepeatedPrimitiveValue -// in the text format, but the binary serializion is different. -// We test the packed representations by loading the same test cases -// using this definition instead of RepeatedPrimitiveValue. -// NOTE: This definition must be kept in sync with RepeatedPrimitiveValue -// in every way except the packed=true declaration. -message PackedPrimitiveValue { - repeated double double_value = 1 [packed = true]; - repeated float float_value = 2 [packed = true]; - repeated int64 int64_value = 3 [packed = true]; - repeated uint64 uint64_value = 4 [packed = true]; - repeated int32 int32_value = 5 [packed = true]; - repeated fixed64 fixed64_value = 6 [packed = true]; - repeated fixed32 fixed32_value = 7 [packed = true]; - repeated bool bool_value = 8 [packed = true]; - repeated string string_value = 9; - repeated bytes bytes_value = 12; - repeated uint32 uint32_value = 13 [packed = true]; - repeated sfixed32 sfixed32_value = 15 [packed = true]; - repeated sfixed64 sfixed64_value = 16 [packed = true]; - repeated sint32 sint32_value = 17 [packed = true]; - repeated sint64 sint64_value = 18 [packed = true]; - repeated PrimitiveValue message_value = 19; -} - -message EnumValue { - enum Color { - RED = 0; - ORANGE = 1; - YELLOW = 2; - GREEN = 3; - BLUE = 4; - INDIGO = 5; - VIOLET = 6; - }; - optional Color enum_value = 14; - repeated Color repeated_enum_value = 15; -} - - -message InnerMessageValue { - optional float float_value = 2; - repeated bytes bytes_values = 8; -} - -message MiddleMessageValue { - repeated int32 int32_values = 5; - optional InnerMessageValue message_value = 11; - optional uint32 uint32_value = 13; -} - -message MessageValue { - optional double double_value = 1; - optional MiddleMessageValue message_value = 11; -} - -message RepeatedMessageValue { - message NestedMessageValue { - optional float float_value = 2; - repeated bytes bytes_values = 8; - } - - repeated NestedMessageValue message_values = 11; -} - -// Message containing fields with field numbers higher than any field above. An -// instance of this message is prepended to each binary message in the test to -// exercise the code path that handles fields encoded out of order of field -// number. -message ExtraFields { - optional string string_value = 1776; - optional bool bool_value = 1777; -} diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl index 44356e3438..4cfa25bf66 100644 --- a/tensorflow/core/platform/default/build_config.bzl +++ b/tensorflow/core/platform/default/build_config.bzl @@ -1,6 +1,7 @@ # Platform-specific build configurations. load("@protobuf_archive//:protobuf.bzl", "proto_gen") +load("@protobuf_archive//:protobuf.bzl", "py_proto_library") load("//tensorflow:tensorflow.bzl", "if_not_mobile") load("//tensorflow:tensorflow.bzl", "if_windows") load("//tensorflow:tensorflow.bzl", "if_not_windows") @@ -109,12 +110,6 @@ def _proto_cc_srcs(srcs, use_grpc_plugin=False): ret += [s[:-len(".proto")] + ".grpc.pb.cc" for s in srcs] return ret -def _proto_py_outs(srcs, use_grpc_plugin=False): - ret = [s[:-len(".proto")] + "_pb2.py" for s in srcs] - if use_grpc_plugin: - ret += [s[:-len(".proto")] + "_pb2_grpc.py" for s in srcs] - return ret - # Re-defined protocol buffer rule to allow building "header only" protocol # buffers, to avoid duplicate registrations. Also allows non-iterable cc_libs # containing select() statements. @@ -222,80 +217,6 @@ def cc_proto_library( hdrs=gen_hdrs, **kargs) -# Re-defined protocol buffer rule to bring in the change introduced in commit -# https://github.com/google/protobuf/commit/294b5758c373cbab4b72f35f4cb62dc1d8332b68 -# which was not part of a stable protobuf release in 04/2018. -# TODO(jsimsa): Remove this once the protobuf dependency version is updated -# to include the above commit. -def py_proto_library( - name, - srcs=[], - deps=[], - py_libs=[], - py_extra_srcs=[], - include=None, - default_runtime="@protobuf_archive//:protobuf_python", - protoc="@protobuf_archive//:protoc", - use_grpc_plugin=False, - **kargs): - """Bazel rule to create a Python protobuf library from proto source files - - NOTE: the rule is only an internal workaround to generate protos. The - interface may change and the rule may be removed when bazel has introduced - the native rule. - - Args: - name: the name of the py_proto_library. - srcs: the .proto files of the py_proto_library. - deps: a list of dependency labels; must be py_proto_library. - py_libs: a list of other py_library targets depended by the generated - py_library. - py_extra_srcs: extra source files that will be added to the output - py_library. This attribute is used for internal bootstrapping. - include: a string indicating the include path of the .proto files. - default_runtime: the implicitly default runtime which will be depended on by - the generated py_library target. - protoc: the label of the protocol compiler to generate the sources. - use_grpc_plugin: a flag to indicate whether to call the Python C++ plugin - when processing the proto files. - **kargs: other keyword arguments that are passed to cc_library. - """ - outs = _proto_py_outs(srcs, use_grpc_plugin) - - includes = [] - if include != None: - includes = [include] - - grpc_python_plugin = None - if use_grpc_plugin: - grpc_python_plugin = "//external:grpc_python_plugin" - # Note: Generated grpc code depends on Python grpc module. This dependency - # is not explicitly listed in py_libs. Instead, host system is assumed to - # have grpc installed. - - proto_gen( - name=name + "_genproto", - srcs=srcs, - deps=[s + "_genproto" for s in deps], - includes=includes, - protoc=protoc, - gen_py=1, - outs=outs, - visibility=["//visibility:public"], - plugin=grpc_python_plugin, - plugin_language="grpc" - ) - - if default_runtime and not default_runtime in py_libs + deps: - py_libs = py_libs + [default_runtime] - - native.py_library( - name=name, - srcs=outs+py_extra_srcs, - deps=py_libs+deps, - imports=includes, - **kargs) - def tf_proto_library_cc(name, srcs = [], has_services = None, protodeps = [], visibility = [], testonly = 0, @@ -340,7 +261,8 @@ def tf_proto_library_cc(name, srcs = [], has_services = None, ) def tf_proto_library_py(name, srcs=[], protodeps=[], deps=[], visibility=[], - testonly=0, srcs_version="PY2AND3", use_grpc_plugin=False): + testonly=0, + srcs_version="PY2AND3"): py_proto_library( name = name + "_py", srcs = srcs, @@ -350,7 +272,6 @@ def tf_proto_library_py(name, srcs=[], protodeps=[], deps=[], visibility=[], default_runtime = "@protobuf_archive//:protobuf_python", visibility = visibility, testonly = testonly, - use_grpc_plugin = use_grpc_plugin, ) def tf_jspb_proto_library(**kwargs): @@ -389,7 +310,6 @@ def tf_proto_library(name, srcs = [], has_services = None, srcs_version = "PY2AND3", testonly = testonly, visibility = visibility, - use_grpc_plugin = has_services, ) def tf_additional_lib_hdrs(exclude = []): diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD index 679d2735f9..376644718f 100644 --- a/tensorflow/tools/pip_package/BUILD +++ b/tensorflow/tools/pip_package/BUILD @@ -75,7 +75,6 @@ COMMON_PIP_DEPS = [ "//tensorflow/contrib/nn:nn_py", "//tensorflow/contrib/predictor:predictor_pip", "//tensorflow/contrib/receptive_field:receptive_field_pip", - "//tensorflow/contrib/rpc:rpc_pip", "//tensorflow/contrib/session_bundle:session_bundle_pip", "//tensorflow/contrib/signal:signal_py", "//tensorflow/contrib/signal:test_util", diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index dee2fcd0e1..72f446d359 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -763,10 +763,6 @@ def tf_workspace(path_prefix="", tf_repo_name=""): name = "grpc_cpp_plugin", actual = "@grpc//:grpc_cpp_plugin", ) - native.bind( - name = "grpc_python_plugin", - actual = "@grpc//:grpc_python_plugin", - ) # gRPC has three empty C++ functions which it wants the user to define # at build time. https://github.com/grpc/grpc/issues/13590 -- GitLab From 68f0f1aadb07ed1e7449b969d8807b5f662be33a Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Thu, 12 Apr 2018 23:05:35 -0700 Subject: [PATCH 306/791] [XLA] Rename Interpreter{Executor,Platform} -> XlaInterpreter{Executor,Platform}. These types live inside StreamExecutor's namespace, but they are specific to XLA. Therefore they either shouldn't live in SE's namespace or should have "XLA" in the name. Moving them out of SE's namespace is ugly, because almost every type used inside of these headers then needs to be qualified. So name-change it is. This patch was generated by a mechanical find/replace. PiperOrigin-RevId: 192724238 --- .../xla/service/interpreter/compiler.cc | 8 ++-- .../xla/service/interpreter/executor.cc | 47 ++++++++++--------- .../xla/service/interpreter/executor.h | 10 ++-- .../interpreter_transfer_manager.cc | 4 +- .../xla/service/interpreter/platform.cc | 33 +++++++------ .../xla/service/interpreter/platform.h | 8 ++-- .../xla/service/interpreter/platform_id.cc | 2 +- .../xla/service/interpreter/platform_id.h | 2 +- 8 files changed, 59 insertions(+), 55 deletions(-) diff --git a/tensorflow/compiler/xla/service/interpreter/compiler.cc b/tensorflow/compiler/xla/service/interpreter/compiler.cc index 9171e859c6..5b9bf5faf3 100644 --- a/tensorflow/compiler/xla/service/interpreter/compiler.cc +++ b/tensorflow/compiler/xla/service/interpreter/compiler.cc @@ -96,7 +96,7 @@ InterpreterCompiler::CompileAheadOfTime( } se::Platform::Id InterpreterCompiler::PlatformId() const { - return sep::kInterpreterPlatformId; + return sep::kXlaInterpreterPlatformId; } HloCostAnalysis::ShapeSizeFunction InterpreterCompiler::ShapeSizeBytesFunction() @@ -109,11 +109,11 @@ static std::unique_ptr CreateComputationPlacer() { } static bool InitModule() { - xla::Compiler::RegisterCompilerFactory(sep::kInterpreterPlatformId, []() { + xla::Compiler::RegisterCompilerFactory(sep::kXlaInterpreterPlatformId, []() { return xla::MakeUnique(); }); - xla::ComputationPlacer::RegisterComputationPlacer(sep::kInterpreterPlatformId, - &CreateComputationPlacer); + xla::ComputationPlacer::RegisterComputationPlacer( + sep::kXlaInterpreterPlatformId, &CreateComputationPlacer); return true; } diff --git a/tensorflow/compiler/xla/service/interpreter/executor.cc b/tensorflow/compiler/xla/service/interpreter/executor.cc index 68371910d7..3caf9e7b82 100644 --- a/tensorflow/compiler/xla/service/interpreter/executor.cc +++ b/tensorflow/compiler/xla/service/interpreter/executor.cc @@ -28,84 +28,85 @@ host::HostStream *AsExecutorStream(Stream *stream) { return dynamic_cast(stream->implementation()); } -InterpreterExecutor::InterpreterExecutor(const PluginConfig &plugin_config) +XlaInterpreterExecutor::XlaInterpreterExecutor( + const PluginConfig &plugin_config) : plugin_config_(plugin_config) {} -InterpreterExecutor::~InterpreterExecutor() {} +XlaInterpreterExecutor::~XlaInterpreterExecutor() {} -void *InterpreterExecutor::Allocate(uint64 size) { return new char[size]; } +void *XlaInterpreterExecutor::Allocate(uint64 size) { return new char[size]; } -void *InterpreterExecutor::AllocateSubBuffer(DeviceMemoryBase *parent, - uint64 offset_bytes, - uint64 /*size_bytes*/) { +void *XlaInterpreterExecutor::AllocateSubBuffer(DeviceMemoryBase *parent, + uint64 offset_bytes, + uint64 /*size_bytes*/) { return parent + offset_bytes; } -void InterpreterExecutor::Deallocate(DeviceMemoryBase *mem) { +void XlaInterpreterExecutor::Deallocate(DeviceMemoryBase *mem) { if (!mem->is_sub_buffer()) { delete[] static_cast(mem->opaque()); } } -bool InterpreterExecutor::Memcpy(Stream *stream, void *host_dst, - const DeviceMemoryBase &dev_src, uint64 size) { +bool XlaInterpreterExecutor::Memcpy(Stream *stream, void *host_dst, + const DeviceMemoryBase &dev_src, + uint64 size) { AsExecutorStream(stream)->EnqueueTask([this, host_dst, dev_src, size]() { port::Status ok = SynchronousMemcpy(host_dst, dev_src, size); }); return true; } -bool InterpreterExecutor::Memcpy(Stream *stream, DeviceMemoryBase *dev_dst, - const void *host_src, uint64 size) { +bool XlaInterpreterExecutor::Memcpy(Stream *stream, DeviceMemoryBase *dev_dst, + const void *host_src, uint64 size) { AsExecutorStream(stream)->EnqueueTask([this, dev_dst, host_src, size]() { port::Status ok = SynchronousMemcpy(dev_dst, host_src, size); }); return true; } -port::Status InterpreterExecutor::SynchronousMemcpy(DeviceMemoryBase *dev_dst, - const void *host_src, - uint64 size) { +port::Status XlaInterpreterExecutor::SynchronousMemcpy( + DeviceMemoryBase *dev_dst, const void *host_src, uint64 size) { memcpy(dev_dst->opaque(), host_src, size); return port::Status::OK(); } -port::Status InterpreterExecutor::SynchronousMemcpy( +port::Status XlaInterpreterExecutor::SynchronousMemcpy( void *host_dst, const DeviceMemoryBase &dev_src, uint64 size) { memcpy(host_dst, dev_src.opaque(), size); return port::Status::OK(); } -bool InterpreterExecutor::HostCallback(Stream *stream, - std::function callback) { +bool XlaInterpreterExecutor::HostCallback(Stream *stream, + std::function callback) { AsExecutorStream(stream)->EnqueueTask(callback); return true; } -bool InterpreterExecutor::CreateStreamDependency(Stream *dependent, - Stream *other) { +bool XlaInterpreterExecutor::CreateStreamDependency(Stream *dependent, + Stream *other) { AsExecutorStream(dependent)->EnqueueTask( [other]() { SE_CHECK_OK(other->BlockHostUntilDone()); }); AsExecutorStream(dependent)->BlockUntilDone(); return true; } -bool InterpreterExecutor::StartTimer(Stream *stream, Timer *timer) { +bool XlaInterpreterExecutor::StartTimer(Stream *stream, Timer *timer) { dynamic_cast(timer->implementation())->Start(stream); return true; } -bool InterpreterExecutor::StopTimer(Stream *stream, Timer *timer) { +bool XlaInterpreterExecutor::StopTimer(Stream *stream, Timer *timer) { dynamic_cast(timer->implementation())->Stop(stream); return true; } -port::Status InterpreterExecutor::BlockHostUntilDone(Stream *stream) { +port::Status XlaInterpreterExecutor::BlockHostUntilDone(Stream *stream) { AsExecutorStream(stream)->BlockUntilDone(); return port::Status::OK(); } -DeviceDescription *InterpreterExecutor::PopulateDeviceDescription() const { +DeviceDescription *XlaInterpreterExecutor::PopulateDeviceDescription() const { internal::DeviceDescriptionBuilder builder; builder.set_device_address_bits(64); diff --git a/tensorflow/compiler/xla/service/interpreter/executor.h b/tensorflow/compiler/xla/service/interpreter/executor.h index c5d07e906d..77426b0820 100644 --- a/tensorflow/compiler/xla/service/interpreter/executor.h +++ b/tensorflow/compiler/xla/service/interpreter/executor.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -// Declares the InterpreterExecutor class, which is a CPU-only implementation of -// the StreamExecutor interface. For now, this is used for testing and to +// Declares the XlaInterpreterExecutor class, which is a CPU-only implementation +// of the StreamExecutor interface. For now, this is used for testing and to // examine the performance of host-based StreamExecutor code. #ifndef TENSORFLOW_COMPILER_XLA_SERVICE_INTERPRETER_EXECUTOR_H_ #define TENSORFLOW_COMPILER_XLA_SERVICE_INTERPRETER_EXECUTOR_H_ @@ -50,10 +50,10 @@ namespace interpreter { using Args = tensorflow::gtl::ArraySlice; -class InterpreterExecutor : public internal::StreamExecutorInterface { +class XlaInterpreterExecutor : public internal::StreamExecutorInterface { public: - explicit InterpreterExecutor(const PluginConfig &plugin_config); - ~InterpreterExecutor() override; + explicit XlaInterpreterExecutor(const PluginConfig &plugin_config); + ~XlaInterpreterExecutor() override; port::Status Init(int device_ordinal, DeviceOptions device_options) override { return port::Status::OK(); diff --git a/tensorflow/compiler/xla/service/interpreter/interpreter_transfer_manager.cc b/tensorflow/compiler/xla/service/interpreter/interpreter_transfer_manager.cc index cf98ecd774..3cf8506d1c 100644 --- a/tensorflow/compiler/xla/service/interpreter/interpreter_transfer_manager.cc +++ b/tensorflow/compiler/xla/service/interpreter/interpreter_transfer_manager.cc @@ -26,7 +26,7 @@ namespace sei = ::perftools::gputools::interpreter; namespace xla { InterpreterTransferManager::InterpreterTransferManager() - : GenericTransferManager(sei::kInterpreterPlatformId, + : GenericTransferManager(sei::kXlaInterpreterPlatformId, /*pointer_size=*/sizeof(void*)) {} } // namespace xla @@ -38,7 +38,7 @@ CreateInterpreterTransferManager() { static bool InitModule() { xla::TransferManager::RegisterTransferManager( - sei::kInterpreterPlatformId, &CreateInterpreterTransferManager); + sei::kXlaInterpreterPlatformId, &CreateInterpreterTransferManager); return true; } diff --git a/tensorflow/compiler/xla/service/interpreter/platform.cc b/tensorflow/compiler/xla/service/interpreter/platform.cc index a60e7fc59f..015e00e1e8 100644 --- a/tensorflow/compiler/xla/service/interpreter/platform.cc +++ b/tensorflow/compiler/xla/service/interpreter/platform.cc @@ -35,17 +35,19 @@ namespace perftools { namespace gputools { namespace interpreter { -InterpreterPlatform::InterpreterPlatform() : name_("Interpreter") {} +XlaInterpreterPlatform::XlaInterpreterPlatform() : name_("Interpreter") {} -InterpreterPlatform::~InterpreterPlatform() {} +XlaInterpreterPlatform::~XlaInterpreterPlatform() {} -Platform::Id InterpreterPlatform::id() const { return kInterpreterPlatformId; } +Platform::Id XlaInterpreterPlatform::id() const { + return kXlaInterpreterPlatformId; +} -int InterpreterPlatform::VisibleDeviceCount() const { return 1; } +int XlaInterpreterPlatform::VisibleDeviceCount() const { return 1; } -const string& InterpreterPlatform::Name() const { return name_; } +const string& XlaInterpreterPlatform::Name() const { return name_; } -port::StatusOr InterpreterPlatform::ExecutorForDevice( +port::StatusOr XlaInterpreterPlatform::ExecutorForDevice( int ordinal) { StreamExecutorConfig config; config.ordinal = ordinal; @@ -55,7 +57,7 @@ port::StatusOr InterpreterPlatform::ExecutorForDevice( } port::StatusOr -InterpreterPlatform::ExecutorForDeviceWithPluginConfig( +XlaInterpreterPlatform::ExecutorForDeviceWithPluginConfig( int device_ordinal, const PluginConfig& plugin_config) { StreamExecutorConfig config; config.ordinal = device_ordinal; @@ -64,16 +66,17 @@ InterpreterPlatform::ExecutorForDeviceWithPluginConfig( return GetExecutor(config); } -port::StatusOr InterpreterPlatform::GetExecutor( +port::StatusOr XlaInterpreterPlatform::GetExecutor( const StreamExecutorConfig& config) { return executor_cache_.GetOrCreate( config, [&]() { return GetUncachedExecutor(config); }); } port::StatusOr> -InterpreterPlatform::GetUncachedExecutor(const StreamExecutorConfig& config) { +XlaInterpreterPlatform::GetUncachedExecutor( + const StreamExecutorConfig& config) { auto executor = port::MakeUnique( - this, port::MakeUnique(config.plugin_config)); + this, port::MakeUnique(config.plugin_config)); auto init_status = executor->Init(config.ordinal, config.device_options); if (!init_status.ok()) { return port::Status{ @@ -86,17 +89,17 @@ InterpreterPlatform::GetUncachedExecutor(const StreamExecutorConfig& config) { return std::move(executor); } -void InterpreterPlatform::RegisterTraceListener( +void XlaInterpreterPlatform::RegisterTraceListener( std::unique_ptr listener) { LOG(FATAL) << "not yet implemented: register executor trace listener"; } -void InterpreterPlatform::UnregisterTraceListener(TraceListener* listener) { +void XlaInterpreterPlatform::UnregisterTraceListener(TraceListener* listener) { LOG(FATAL) << "not yet implemented: unregister executor trace listener"; } -static void InitializeInterpreterPlatform() { - std::unique_ptr platform(new sep::InterpreterPlatform); +static void InitializeXlaInterpreterPlatform() { + std::unique_ptr platform(new sep::XlaInterpreterPlatform); SE_CHECK_OK(se::MultiPlatformManager::RegisterPlatform(std::move(platform))); } @@ -105,7 +108,7 @@ static void InitializeInterpreterPlatform() { } // namespace perftools REGISTER_MODULE_INITIALIZER(interpreter_platform, - sep::InitializeInterpreterPlatform()); + sep::InitializeXlaInterpreterPlatform()); DECLARE_MODULE_INITIALIZER(multi_platform_manager); diff --git a/tensorflow/compiler/xla/service/interpreter/platform.h b/tensorflow/compiler/xla/service/interpreter/platform.h index c66ddb907d..2f71b29be4 100644 --- a/tensorflow/compiler/xla/service/interpreter/platform.h +++ b/tensorflow/compiler/xla/service/interpreter/platform.h @@ -27,10 +27,10 @@ namespace perftools { namespace gputools { namespace interpreter { -class InterpreterPlatform : public Platform { +class XlaInterpreterPlatform : public Platform { public: - InterpreterPlatform(); - ~InterpreterPlatform() override; + XlaInterpreterPlatform(); + ~XlaInterpreterPlatform() override; Platform::Id id() const override; @@ -60,7 +60,7 @@ class InterpreterPlatform : public Platform { // Cache of created StreamExecutors. ExecutorCache executor_cache_; - SE_DISALLOW_COPY_AND_ASSIGN(InterpreterPlatform); + SE_DISALLOW_COPY_AND_ASSIGN(XlaInterpreterPlatform); }; } // namespace interpreter diff --git a/tensorflow/compiler/xla/service/interpreter/platform_id.cc b/tensorflow/compiler/xla/service/interpreter/platform_id.cc index 1a0373cf86..b7fb365b70 100644 --- a/tensorflow/compiler/xla/service/interpreter/platform_id.cc +++ b/tensorflow/compiler/xla/service/interpreter/platform_id.cc @@ -18,7 +18,7 @@ namespace perftools { namespace gputools { namespace interpreter { -PLATFORM_DEFINE_ID(kInterpreterPlatformId); +PLATFORM_DEFINE_ID(kXlaInterpreterPlatformId); } // namespace interpreter } // namespace gputools diff --git a/tensorflow/compiler/xla/service/interpreter/platform_id.h b/tensorflow/compiler/xla/service/interpreter/platform_id.h index 905efef169..292f958449 100644 --- a/tensorflow/compiler/xla/service/interpreter/platform_id.h +++ b/tensorflow/compiler/xla/service/interpreter/platform_id.h @@ -22,7 +22,7 @@ namespace perftools { namespace gputools { namespace interpreter { -extern const Platform::Id kInterpreterPlatformId; +extern const Platform::Id kXlaInterpreterPlatformId; } // namespace interpreter } // namespace gputools -- GitLab From 73cc1d5b6f95ff56207e4c42b62d383c2427fb75 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 00:03:48 -0700 Subject: [PATCH 307/791] -- Add a new histogram/cdf computation method compatible with the TPU. -- Refactor utility functions into pruning_utils.py and add tests PiperOrigin-RevId: 192727737 --- tensorflow/contrib/model_pruning/BUILD | 24 +- tensorflow/contrib/model_pruning/README.md | 2 +- .../contrib/model_pruning/python/pruning.py | 237 +++------------ .../model_pruning/python/pruning_test.py | 15 +- .../model_pruning/python/pruning_utils.py | 269 ++++++++++++++++++ .../python/pruning_utils_test.py | 86 ++++++ 6 files changed, 430 insertions(+), 203 deletions(-) create mode 100644 tensorflow/contrib/model_pruning/python/pruning_utils.py create mode 100644 tensorflow/contrib/model_pruning/python/pruning_utils_test.py diff --git a/tensorflow/contrib/model_pruning/BUILD b/tensorflow/contrib/model_pruning/BUILD index f50575b2cf..54bd39afac 100644 --- a/tensorflow/contrib/model_pruning/BUILD +++ b/tensorflow/contrib/model_pruning/BUILD @@ -71,6 +71,17 @@ py_library( ], ) +py_library( + name = "pruning_utils", + srcs = ["python/pruning_utils.py"], + srcs_version = "PY2AND3", + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/python:platform", + "//third_party/py/numpy", + ], +) + py_library( name = "pruning", srcs = ["python/pruning.py"], @@ -78,9 +89,20 @@ py_library( visibility = ["//visibility:public"], deps = [ ":core_layers", + ":pruning_utils", "//tensorflow/contrib/training:training_py", "//tensorflow/python:platform", - "//third_party/py/numpy", + ], +) + +py_test( + name = "pruning_utils_test", + size = "small", + srcs = ["python/pruning_utils_test.py"], + srcs_version = "PY2AND3", + deps = [ + ":pruning_utils", + "//tensorflow/python:client_testlib", ], ) diff --git a/tensorflow/contrib/model_pruning/README.md b/tensorflow/contrib/model_pruning/README.md index 52b659c69f..86f4fd6adf 100644 --- a/tensorflow/contrib/model_pruning/README.md +++ b/tensorflow/contrib/model_pruning/README.md @@ -45,7 +45,7 @@ The pruning library allows for specification of the following hyper parameters: | do_not_prune | list of strings | [""] | list of layers names that are not pruned | | threshold_decay | float | 0.9 | The decay factor to use for exponential decay of the thresholds | | pruning_frequency | integer | 10 | How often should the masks be updated? (in # of global_steps) | -| nbins | integer | 255 | Number of bins to use for histogram computation | +| nbins | integer | 256 | Number of bins to use for histogram computation | | block_height|integer | 1 | Number of rows in a block for block sparse matrices| | block_width |integer | 1 | Number of cols in a block for block sparse matrices| | block_pooling_function| string | AVG | The function to use to pool weight values in a block: average (AVG) or max (MAX)| diff --git a/tensorflow/contrib/model_pruning/python/pruning.py b/tensorflow/contrib/model_pruning/python/pruning.py index 5146a4a2de..ea6032e588 100644 --- a/tensorflow/contrib/model_pruning/python/pruning.py +++ b/tensorflow/contrib/model_pruning/python/pruning.py @@ -33,12 +33,14 @@ # Returns a list of all the weight tensors that have been masked get_weights() - The Pruning class uses a proto (defined in pruning.proto) to set up the - parameters for a pruning specification. Here's a typical usage: + The Pruning class uses a tf.hparams object to set up the + parameters for a model pruning. Here's a typical usage: - # Initialize a pruning spec from a proto - pruning_spec = '/tmp/pruning.pb' - p = Pruning(pruning_spec) + # Parse pruning hyperparameters + pruning_hparams = pruning.get_pruning_hparams().parse(FLAGS.pruning_hparams) + + # Create a pruning object using the pruning_hparams + p = pruning.Pruning(pruning_hparams) # Add mask update ops to the graph mask_update_op = p.conditional_mask_update_op() @@ -51,24 +53,20 @@ # An object of the pruning also accepts externally defined sparsity: sparsity = tf.Variable(0.5, name = "ConstantSparsity") - pruning_spec = '/tmp/pruning.pb' - p = Pruning(pruning_spec, sparsity=sparsity) - + p = pruning.Pruning(pruning_hparams, sparsity=sparsity) """ # pylint: disable=missing-docstring from __future__ import absolute_import from __future__ import division from __future__ import print_function -import numpy as np - +from tensorflow.contrib.model_pruning.python import pruning_utils from tensorflow.contrib.model_pruning.python.layers import core_layers as core from tensorflow.contrib.training.python.training import hparam +from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops -from tensorflow.python.ops import clip_ops from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import gen_array_ops from tensorflow.python.ops import init_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn_impl @@ -87,172 +85,18 @@ _WEIGHT_COLLECTION = core.WEIGHT_COLLECTION _MASKED_WEIGHT_NAME = core.MASKED_WEIGHT_NAME -def _weight_mask_variable(var, scope): - """Create a mask for the weights. - - This function adds a variable 'mask' to the graph. - - Args: - var: the weight variable that needs to be masked - scope: The variable scope of the variable var - - Returns: - the mask variable of the same size and shape as var, initialized to all 1s. - """ - with variable_scope.variable_scope(scope): - mask = variable_scope.get_variable( - 'mask', - var.get_shape(), - initializer=init_ops.ones_initializer(), - trainable=False, - dtype=var.dtype) - return mask - - -def _weight_threshold_variable(var, scope): - """Create a scalar threshold for the weights. - - This function adds a variable - 'threshold' to the graph. - - Args: - var: The weight variable that needs to be masked - scope: The variable scope of the variable var - - Returns: - a scalar threshold variable initialized to 0. - """ - with variable_scope.variable_scope(scope): - threshold = variable_scope.get_variable( - 'threshold', [], - initializer=init_ops.zeros_initializer(), - trainable=False, - dtype=var.dtype) - return threshold - - -def _kronecker_product(mat1, mat2): - """Computes the Kronecker product of two matrices mat1 and mat2. - - Args: - mat1: A matrix of size m x n - mat2: A matrix of size p x q - Returns: - Kronecker product of matrices mat1 and mat2 of size mp x nq - """ - - m1, n1 = mat1.get_shape().as_list() - mat1_rsh = array_ops.reshape(mat1, [m1, 1, n1, 1]) - m2, n2 = mat2.get_shape().as_list() - mat2_rsh = array_ops.reshape(mat2, [1, m2, 1, n2]) - return array_ops.reshape(mat1_rsh * mat2_rsh, [m1 * m2, n1 * n2]) - - -def _histogram(values, value_range, nbins=100, dtype=np.int32, name=None): - """Return histogram of values. - - Given the tensor `values`, this operation returns a rank 1 histogram counting - the number of entries in `values` that fell into every bin. The bins are - equal width and determined by the arguments `value_range` and `nbins`. - - Args: - values: Numeric `Tensor`. - value_range: Shape [2] `Tensor` of same `dtype` as `values`. - values <= value_range[0] will be mapped to hist[0], - values >= value_range[1] will be mapped to hist[-1]. - nbins: Scalar `int32 Tensor`. Number of histogram bins. - dtype: dtype for returned histogram. - name: A name for this operation (defaults to 'histogram'). - - Returns: - A 1-D `Tensor` holding histogram of values. - - """ - with ops.name_scope(name, 'histogram', [values, value_range, nbins]) as scope: - values = ops.convert_to_tensor(values, name='values') - values = gen_array_ops.reshape(values, [-1]) - value_range = ops.convert_to_tensor(value_range, name='value_range') - nbins = ops.convert_to_tensor(nbins, dtype=np.int32, name='nbins') - nbins_float = math_ops.cast(nbins, values.dtype) - - # Map tensor values that fall within value_range to [0, 1]. - scaled_values = math_ops.truediv( - values - value_range[0], - value_range[1] - value_range[0], - name='scaled_values') - - # map tensor values within the open interval value_range to {0,.., nbins-1}, - # values outside the open interval will be zero or less, or nbins or more. - indices = math_ops.floor(nbins_float * scaled_values, name='indices') - - # Clip edge cases (e.g. value = value_range[1]) or "outliers." - indices = math_ops.cast( - clip_ops.clip_by_value(indices, 0, nbins_float - 1), np.int32) - - return math_ops.unsorted_segment_sum( - array_ops.ones_like(indices, dtype=dtype), indices, nbins, name=scope) - - -def _determine_partitioned_axis(partitioned_variable): - partitioned_axis = 0 - concatenated_variable_shape = partitioned_variable.get_shape() - for partition in partitioned_variable: - partition_shape = partition.get_shape() - maybe_partitioned_axis = np.less(partition_shape, - concatenated_variable_shape) - # Sanity check: make sure number of partitioned axis == 1 - if np.count_nonzero(maybe_partitioned_axis) != 1: - raise ValueError('Number of partitioned axes %s not equal to 1' % - np.count_nonzero(maybe_partitioned_axis)) - partitioned_axis = np.where(maybe_partitioned_axis)[0][0] - return partitioned_axis - - -def _variable_assign(var, new_value): - return state_ops.assign(var, new_value, name=var.op.name + '_assign') - - -def _partitioned_variable_assign(partitioned_var, new_value): - """Assign op for partitioned variables. - - Args: - partitioned_var: A partitioned tensorflow variable - new_value: Value to be assigned to the variable var - - Returns: - A tensorflow op that groups the assign ops for each of the variable slices - """ - # Determine which axis was used to partition the variable. Currently - # tensorflow allows partitioning variable only along 1 axis. - axis = 0 if len(partitioned_var) == 1 else _determine_partitioned_axis( - partitioned_var) - - partition_sizes = np.array( - [partition.get_shape()[axis] for partition in partitioned_var]) - new_partitioned_values = array_ops.split( - new_value, - ops.convert_to_tensor(partition_sizes, dtype=np.int32), - axis=axis) - op_list = [] - for partition in partitioned_var: - op_list.append( - _variable_assign(partition, new_partitioned_values[len(op_list)])) - return control_flow_ops.group( - *op_list, name=partitioned_var.name + '_group_assign') - - def apply_mask(x, scope=''): """Apply mask to a given weight tensor. Args: x: Input weight tensor - scope: The current variable scope. Defaults to "" + scope: The current variable scope. Defaults to "". Returns: Tensor representing masked_weights """ - mask = _weight_mask_variable(x, scope) - threshold = _weight_threshold_variable(x, scope) + mask = pruning_utils.weight_mask_variable(x, scope) + threshold = pruning_utils.weight_threshold_variable(x, scope) # Add masked_weights in the weights namescope so as to make it easier # for the quantization library to add quant ops. masked_weights = math_ops.multiply(mask, x, _MASKED_WEIGHT_NAME) @@ -335,6 +179,8 @@ def get_pruning_hparams(): sparsity_function_exponent: float exponent = 1 is linearly varying sparsity between initial and final. exponent > 1 varies more slowly towards the end than the beginning + use_tpu: False + Indicates whether to use TPU We use the following sparsity function: @@ -357,7 +203,7 @@ def get_pruning_hparams(): do_not_prune=[''], threshold_decay=0.9, pruning_frequency=10, - nbins=255, + nbins=256, block_height=1, block_width=1, block_pooling_function='AVG', @@ -365,7 +211,8 @@ def get_pruning_hparams(): target_sparsity=0.5, sparsity_function_begin_step=0, sparsity_function_end_step=100, - sparsity_function_exponent=3) + sparsity_function_exponent=3, + use_tpu=False) class Pruning(object): @@ -414,7 +261,7 @@ class Pruning(object): if graph_global_step is None: graph_global_step = training_util.get_global_step() - return math_ops.cast(graph_global_step, np.int32) + return math_ops.cast(graph_global_step, dtypes.int32) def _setup_sparsity(self): begin_step = self._spec.sparsity_function_begin_step @@ -429,13 +276,13 @@ class Pruning(object): (begin_step, end_step)) with ops.name_scope(self._spec.name): - p = math_ops.minimum(1.0, - math_ops.maximum( - 0.0, - math_ops.div( - math_ops.cast(self._global_step - begin_step, - np.float32), - end_step - begin_step))) + p = math_ops.minimum( + 1.0, + math_ops.maximum( + 0.0, + math_ops.div( + math_ops.cast(self._global_step - begin_step, dtypes.float32), + end_step - begin_step))) sparsity = math_ops.add( math_ops.multiply(initial_sparsity - target_sparsity, math_ops.pow(1 - p, exponent)), @@ -445,17 +292,18 @@ class Pruning(object): return sparsity def _setup_last_update_step(self): - with variable_scope.variable_scope(self._spec.name) as scope: + with variable_scope.variable_scope( + self._spec.name, use_resource=self._spec.use_tpu) as scope: try: last_update_step = variable_scope.get_variable( 'last_mask_update_step', [], initializer=init_ops.zeros_initializer(), trainable=False, - dtype=np.int32) + dtype=dtypes.int32) except ValueError: scope.reuse_variables() last_update_step = variable_scope.get_variable( - 'last_mask_update_step', dtype=np.int32) + 'last_mask_update_step', dtype=dtypes.int32) return last_update_step def _exists_in_do_not_prune_list(self, tensor_name): @@ -497,18 +345,16 @@ class Pruning(object): with ops.name_scope(weights.op.name + '_pruning_ops'): abs_weights = math_ops.abs(weights) max_value = math_ops.reduce_max(abs_weights) - histogram = _histogram( - abs_weights, [0.0, max_value], - nbins=self._spec.nbins, - dtype=np.float32) + cdf_fn = pruning_utils.compute_cdf_from_histogram + if self._spec.use_tpu: + cdf_fn = pruning_utils.compute_cdf - cdf = math_ops.cumsum(histogram) - norm_cdf = math_ops.div(cdf, math_ops.reduce_sum(histogram)) + norm_cdf = cdf_fn(abs_weights, [0.0, max_value], nbins=self._spec.nbins) current_threshold = math_ops.multiply( math_ops.div( math_ops.reduce_sum( math_ops.cast( - math_ops.less(norm_cdf, self._sparsity), np.float32)), + math_ops.less(norm_cdf, self._sparsity), dtypes.float32)), float(self._spec.nbins)), max_value) smoothed_threshold = math_ops.add_n([ @@ -516,7 +362,7 @@ class Pruning(object): math_ops.multiply(threshold, self._spec.threshold_decay) ]) new_mask = math_ops.cast( - math_ops.greater(abs_weights, smoothed_threshold), np.float32) + math_ops.greater(abs_weights, smoothed_threshold), dtypes.float32) return smoothed_threshold, new_mask def _maybe_update_block_mask(self, weights, threshold): @@ -572,8 +418,8 @@ class Pruning(object): new_mask, [pooled_weights.get_shape()[1], pooled_weights.get_shape()[2]]) - updated_mask = _kronecker_product(reshaped_mask, - array_ops.ones(self._block_dim)) + updated_mask = pruning_utils.kronecker_product( + reshaped_mask, array_ops.ones(self._block_dim)) sliced_mask = array_ops.slice( updated_mask, [0, 0], [squeezed_weights.get_shape()[0], @@ -608,11 +454,12 @@ class Pruning(object): continue new_threshold, new_mask = self._maybe_update_block_mask(weight, threshold) - self._assign_ops.append(_variable_assign(threshold, new_threshold)) + self._assign_ops.append( + pruning_utils.variable_assign(threshold, new_threshold)) self._assign_ops.append( - _partitioned_variable_assign(mask, new_mask) - if is_partitioned else _variable_assign(mask, new_mask)) + pruning_utils.partitioned_variable_assign(mask, new_mask) + if is_partitioned else pruning_utils.variable_assign(mask, new_mask)) def mask_update_op(self): with ops.name_scope(self._spec.name): diff --git a/tensorflow/contrib/model_pruning/python/pruning_test.py b/tensorflow/contrib/model_pruning/python/pruning_test.py index 89e6571319..f80b7c52c0 100644 --- a/tensorflow/contrib/model_pruning/python/pruning_test.py +++ b/tensorflow/contrib/model_pruning/python/pruning_test.py @@ -110,12 +110,12 @@ class PruningTest(test.TestCase): self.assertAllEqual(np.count_nonzero(masked_weights_val), 100) session.run(mask_update_op) masked_weights_val = masked_weights.eval() - self.assertAllEqual(np.count_nonzero(masked_weights_val), 51) + self.assertAllEqual(np.count_nonzero(masked_weights_val), 50) def _blockMasking(self, hparams, weights, expected_mask): threshold = variables.Variable(0.0, name="threshold") - sparsity = variables.Variable(0.51, name="sparsity") + sparsity = variables.Variable(0.5, name="sparsity") test_spec = ",".join(hparams) pruning_hparams = pruning.get_pruning_hparams().parse(test_spec) @@ -138,7 +138,8 @@ class PruningTest(test.TestCase): weights_max = constant_op.constant( [[0.1, 0.0, 0.2, 0.0], [0.0, -0.1, 0.0, -0.2], [0.3, 0.0, 0.4, 0.0], [0.0, -0.3, 0.0, -0.4]]) - expected_mask = [[0, 0, 0, 0], [0, 0, 0, 0], [1, 1, 1, 1], [1, 1, 1, 1]] + expected_mask = [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], + [1., 1., 1., 1.], [1., 1., 1., 1.]] self._blockMasking(param_list + ["block_pooling_function=MAX"], weights_max, expected_mask) @@ -155,7 +156,8 @@ class PruningTest(test.TestCase): weights_max = constant_op.constant( [[[0.1, 0.0, 0.2, 0.0], [0.0, -0.1, 0.0, -0.2], [0.3, 0.0, 0.4, 0.0], [0.0, -0.3, 0.0, -0.4]]]) - expected_mask = [[[0, 0, 0, 0], [0, 0, 0, 0], [1, 1, 1, 1], [1, 1, 1, 1]]] + expected_mask = [[[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], + [1., 1., 1., 1.], [1., 1., 1., 1.]]] self._blockMasking(param_list + ["block_pooling_function=MAX"], weights_max, expected_mask) @@ -178,11 +180,12 @@ class PruningTest(test.TestCase): masked_weights_val = masked_weights.eval() session.run(mask_update_op) masked_weights_val = masked_weights.eval() - self.assertAllEqual(np.count_nonzero(masked_weights_val), 51) + self.assertAllEqual(np.count_nonzero(masked_weights_val), 50) def testConditionalMaskUpdate(self): param_list = [ - "pruning_frequency=2", "begin_pruning_step=1", "end_pruning_step=6" + "pruning_frequency=2", "begin_pruning_step=1", "end_pruning_step=6", + "nbins=100" ] test_spec = ",".join(param_list) pruning_hparams = pruning.get_pruning_hparams().parse(test_spec) diff --git a/tensorflow/contrib/model_pruning/python/pruning_utils.py b/tensorflow/contrib/model_pruning/python/pruning_utils.py new file mode 100644 index 0000000000..56d3dcef20 --- /dev/null +++ b/tensorflow/contrib/model_pruning/python/pruning_utils.py @@ -0,0 +1,269 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utility functions for adding pruning related ops to the graph. +""" +# pylint: disable=missing-docstring +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import clip_ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import state_ops +from tensorflow.python.ops import variable_scope + +_NBINS = 256 + + +def weight_mask_variable(var, scope): + """Create a mask for the weights. + + This function adds a variable 'mask' to the graph. + + Args: + var: the weight variable that needs to be masked + scope: The variable scope of the variable var + + Returns: + the mask variable of the same size and shape as var, initialized to all 1s. + """ + with variable_scope.variable_scope(scope): + mask = variable_scope.get_variable( + 'mask', + var.get_shape(), + initializer=init_ops.ones_initializer(), + trainable=False, + dtype=var.dtype) + return mask + + +def weight_threshold_variable(var, scope): + """Create a scalar threshold for the weights. + + This function adds a variable + 'threshold' to the graph. + + Args: + var: The weight variable that needs to be masked + scope: The variable scope of the variable var + + Returns: + a scalar threshold variable initialized to 0. + """ + with variable_scope.variable_scope(scope): + threshold = variable_scope.get_variable( + 'threshold', [], + initializer=init_ops.zeros_initializer(), + trainable=False, + dtype=var.dtype) + return threshold + + +def kronecker_product(mat1, mat2): + """Computes the Kronecker product of two matrices mat1 and mat2. + + Args: + mat1: A matrix of size m x n + mat2: A matrix of size p x q + Returns: + Kronecker product of matrices mat1 and mat2 of size mp x nq + """ + + m1, n1 = mat1.get_shape().as_list() + mat1_rsh = array_ops.reshape(mat1, [m1, 1, n1, 1]) + m2, n2 = mat2.get_shape().as_list() + mat2_rsh = array_ops.reshape(mat2, [1, m2, 1, n2]) + return array_ops.reshape(mat1_rsh * mat2_rsh, [m1 * m2, n1 * n2]) + + +def _histogram(values, value_range, nbins=100, dtype=dtypes.int32, name=None): + """Return histogram of values. + + Given the tensor `values`, this operation returns a rank 1 histogram counting + the number of entries in `values` that fell into every bin. The bins are + equal width and determined by the arguments `value_range` and `nbins`. + + Args: + values: Numeric `Tensor`. + value_range: Shape [2] `Tensor` of same `dtype` as `values`. + values <= value_range[0] will be mapped to hist[0], + values >= value_range[1] will be mapped to hist[-1]. + nbins: Scalar `int32 Tensor`. Number of histogram bins. + dtype: dtype for returned histogram. + name: A name for this operation (defaults to 'histogram'). + + Returns: + A 1-D `Tensor` holding histogram of values. + + """ + with ops.name_scope(name, 'histogram', [values, value_range, nbins]) as scope: + values = ops.convert_to_tensor(values, name='values') + values = array_ops.reshape(values, [-1]) + value_range = ops.convert_to_tensor(value_range, name='value_range') + nbins_float = np.float32(nbins) + + # Map tensor values that fall within value_range to [0, 1]. + scaled_values = math_ops.truediv( + values - value_range[0], + value_range[1] - value_range[0], + name='scaled_values') + + # map tensor values within the open interval value_range to {0,.., nbins-1}, + # values outside the open interval will be zero or less, or nbins or more. + indices = math_ops.floor(nbins_float * scaled_values, name='indices') + + # Clip edge cases (e.g. value = value_range[1]) or "outliers." + indices = math_ops.cast( + clip_ops.clip_by_value(indices, 0, nbins_float - 1), dtypes.int32) + + return math_ops.unsorted_segment_sum( + array_ops.ones_like(indices, dtype=dtype), indices, nbins, name=scope) + + +def compute_cdf_from_histogram(values, value_range, **kwargs): + """Returns the normalized cumulative distribution of the given values tensor. + + Computes the histogram and uses tf.cumsum to arrive at cdf + + Args: + values: Numeric `Tensor`. + value_range: Shape [2] `Tensor` of same `dtype` as `values`. + **kwargs: keyword arguments: nbins, name + + Returns: + A 1-D `Tensor` holding normalized cdf of values. + + """ + nbins = kwargs.get('nbins', _NBINS) + name = kwargs.get('name', None) + with ops.name_scope(name, 'cdf', [values, value_range, nbins]): + histogram = _histogram( + values, value_range, dtype=dtypes.float32, nbins=nbins) + cdf = math_ops.cumsum(histogram) + return math_ops.div(cdf, math_ops.reduce_max(cdf)) + + +def compute_cdf(values, value_range, **kwargs): + """Returns the normalized cumulative distribution of the given values tensor. + + Uses tf.while_loop to directly compute the cdf of the values. Number of bins + for histogram is fixed at _NBINS=255 + + Args: + values: Numeric `Tensor`. + value_range: Shape [2] `Tensor` of same `dtype` as `values` + **kwargs: keyword arguments: name + + Returns: + A 1-D `Tensor` holding normalized cdf of values. + + """ + nbins = _NBINS + name = kwargs.get('name', None) + with ops.name_scope(name, 'cdf', [values, value_range, nbins]): + values = ops.convert_to_tensor(values, name='values') + value_range = ops.convert_to_tensor(value_range, name='value_range') + nbins_float = np.float32(nbins) + + # Map tensor values that fall within value_range to [0, 1]. + scaled_values = math_ops.truediv( + values - value_range[0], + value_range[1] - value_range[0], + name='scaled_values') + + # map tensor values within the open interval value_range to {0,.., nbins-1}, + # values outside the open interval will be zero or less, or nbins or more. + indices = math_ops.floor(nbins_float * scaled_values, name='indices') + + # Clip edge cases (e.g. value = value_range[1]) or "outliers." + indices = math_ops.cast( + clip_ops.clip_by_value(indices, 0, nbins_float - 1), dtypes.int32) + + cdf = array_ops.zeros(nbins) + i = constant_op.constant(0) + + def loop_cond(loop_count, _): + return math_ops.less(loop_count, nbins) + + def loop_body(loop_count, cdf): + temp = math_ops.reduce_sum( + math_ops.cast( + math_ops.less_equal(indices, loop_count), dtypes.float32)) + cdf = math_ops.add( + cdf, + array_ops.one_hot( + loop_count, depth=_NBINS, on_value=temp, off_value=0.0)) + return [loop_count + 1, cdf] + + _, cdf = control_flow_ops.while_loop( + loop_cond, loop_body, [i, cdf], maximum_iterations=nbins) + + return math_ops.div(cdf, math_ops.reduce_max(cdf)) + + +def determine_partitioned_axis(partitioned_variable): + partitioned_axis = 0 + concatenated_variable_shape = partitioned_variable.get_shape() + for partition in partitioned_variable: + partition_shape = partition.get_shape() + maybe_partitioned_axis = np.less(partition_shape, + concatenated_variable_shape) + # Sanity check: make sure number of partitioned axis == 1 + if np.count_nonzero(maybe_partitioned_axis) != 1: + raise ValueError('Number of partitioned axes %s not equal to 1' % + np.count_nonzero(maybe_partitioned_axis)) + partitioned_axis = np.where(maybe_partitioned_axis)[0][0] + return partitioned_axis + + +def variable_assign(var, new_value): + return state_ops.assign(var, new_value, name=var.op.name + '_assign') + + +def partitioned_variable_assign(partitioned_var, new_value): + """Assign op for partitioned variables. + + Args: + partitioned_var: A partitioned tensorflow variable + new_value: Value to be assigned to the variable var + + Returns: + A tensorflow op that groups the assign ops for each of the variable slices + """ + # Determine which axis was used to partition the variable. Currently + # tensorflow allows partitioning variable only along 1 axis. + axis = 0 if len(partitioned_var) == 1 else determine_partitioned_axis( + partitioned_var) + + partition_sizes = np.array( + [partition.get_shape()[axis] for partition in partitioned_var]) + new_partitioned_values = array_ops.split( + new_value, + ops.convert_to_tensor(partition_sizes, dtype=dtypes.int32), + axis=axis) + op_list = [] + for partition in partitioned_var: + op_list.append( + variable_assign(partition, new_partitioned_values[len(op_list)])) + return control_flow_ops.group( + *op_list, name=partitioned_var.name + '_group_assign') diff --git a/tensorflow/contrib/model_pruning/python/pruning_utils_test.py b/tensorflow/contrib/model_pruning/python/pruning_utils_test.py new file mode 100644 index 0000000000..10e1dd0a8e --- /dev/null +++ b/tensorflow/contrib/model_pruning/python/pruning_utils_test.py @@ -0,0 +1,86 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for utility functions in pruning_utils.py.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.contrib.model_pruning.python import pruning_utils +from tensorflow.python.framework import constant_op +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import variable_scope +from tensorflow.python.ops import variables +from tensorflow.python.platform import test + + +class PruningUtilsTest(test.TestCase): + + def testHistogram(self): + width = 10 + height = 10 + nbins = 100 + expected_histogram = np.full(nbins, 1.0) + init = init_ops.constant_initializer(np.linspace(0.0, 1.0, width * height)) + weights = variable_scope.get_variable( + "weights", [width, height], initializer=init) + histogram = pruning_utils._histogram( + weights, [0, 1.0], nbins, dtype=np.float32) + with self.test_session(): + variables.global_variables_initializer().run() + computed_histogram = histogram.eval() + self.assertAllEqual(expected_histogram, computed_histogram) + + def testCDF(self): + nbins = 5 + weights = constant_op.constant([-1, 0, 1, 1.5, 2, 3, 4, 5, 10, 100]) + abs_weights = math_ops.abs(weights) + norm_cdf = pruning_utils.compute_cdf_from_histogram( + abs_weights, [0.0, 5.0], nbins=nbins) + expected_cdf = np.array([0.1, 0.4, 0.5, 0.6, 1.0], dtype=np.float32) + with self.test_session() as sess: + variables.global_variables_initializer().run() + norm_cdf_val = sess.run(norm_cdf) + self.assertAllEqual(len(norm_cdf_val), nbins) + self.assertAllEqual(expected_cdf, norm_cdf_val) + + def _compare_cdf(self, values): + abs_values = math_ops.abs(values) + max_value = math_ops.reduce_max(abs_values) + with self.test_session(): + variables.global_variables_initializer().run() + cdf_from_histogram = pruning_utils.compute_cdf_from_histogram( + abs_values, [0.0, max_value], nbins=pruning_utils._NBINS) + cdf = pruning_utils.compute_cdf(abs_values, [0.0, max_value]) + return cdf.eval(), cdf_from_histogram.eval() + + def testCDFEquivalence2D(self): + width = 100 + height = 100 + weights = variable_scope.get_variable("weights", shape=[width, height]) + cdf_val, cdf_from_histogram_val = self._compare_cdf(weights) + self.assertAllEqual(cdf_val, cdf_from_histogram_val) + + def testCDFEquivalence4D(self): + weights = variable_scope.get_variable("weights", shape=[5, 5, 128, 128]) + cdf_val, cdf_from_histogram_val = self._compare_cdf(weights) + self.assertAllEqual(cdf_val, cdf_from_histogram_val) + + +if __name__ == "__main__": + test.main() -- GitLab From 1b0c277405171a34c7f41e17cd76459dc36f7f82 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 00:12:41 -0700 Subject: [PATCH 308/791] Implementation of Less PiperOrigin-RevId: 192728635 --- tensorflow/contrib/lite/builtin_ops.h | 1 + .../lite/g3doc/tf_ops_compatibility.md | 13 ++ tensorflow/contrib/lite/kernels/BUILD | 19 +++ .../contrib/lite/kernels/comparisons.cc | 119 +++++++++++++++++ .../contrib/lite/kernels/comparisons_test.cc | 98 ++++++++++++++ .../internal/reference/reference_ops.h | 45 +++++++ tensorflow/contrib/lite/kernels/register.cc | 2 + tensorflow/contrib/lite/model.cc | 3 + tensorflow/contrib/lite/nnapi_delegate.cc | 1 + tensorflow/contrib/lite/schema/schema.fbs | 5 + .../contrib/lite/schema/schema_generated.h | 124 +++++++++++++++++- tensorflow/contrib/lite/testing/BUILD | 1 + .../contrib/lite/testing/generate_examples.py | 33 +++++ .../testing/generated_examples_zip_test.cc | 1 + .../contrib/lite/toco/tflite/operator.cc | 2 + .../contrib/lite/toco/tflite/operator_test.cc | 2 + 16 files changed, 463 insertions(+), 6 deletions(-) create mode 100644 tensorflow/contrib/lite/kernels/comparisons.cc create mode 100644 tensorflow/contrib/lite/kernels/comparisons_test.cc diff --git a/tensorflow/contrib/lite/builtin_ops.h b/tensorflow/contrib/lite/builtin_ops.h index 1ceefafc56..859bc7ab70 100644 --- a/tensorflow/contrib/lite/builtin_ops.h +++ b/tensorflow/contrib/lite/builtin_ops.h @@ -82,6 +82,7 @@ typedef enum { kTfLiteBuiltinMaximum = 55, kTfLiteBuiltinArgMax = 56, kTfLiteBuiltinMinimum = 57, + kTfLiteBuiltinLess = 58, } TfLiteBuiltinOperator; #ifdef __cplusplus diff --git a/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md b/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md index 61ea5231e3..203924f03d 100644 --- a/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md +++ b/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md @@ -302,6 +302,19 @@ Options { } ``` +**LESS** + +``` +Inputs { + 0: a tensor + 1: a tensor +} +Outputs { + 0: a tensor of type bool, true whenever an element of the first tensor is less + than the corresponding element of the second tensor. +} +``` + **LOCAL_RESPONSE_NORMALIZATION** ``` diff --git a/tensorflow/contrib/lite/kernels/BUILD b/tensorflow/contrib/lite/kernels/BUILD index 914893cd90..800e2a9558 100644 --- a/tensorflow/contrib/lite/kernels/BUILD +++ b/tensorflow/contrib/lite/kernels/BUILD @@ -136,6 +136,7 @@ cc_library( "bidirectional_sequence_lstm.cc", "bidirectional_sequence_rnn.cc", "cast.cc", + "comparisons.cc", "concatenation.cc", "conv.cc", "depthwise_conv.cc", @@ -818,6 +819,24 @@ tf_cc_test( ], ) +tf_cc_test( + name = "comparisons_test", + size = "small", + srcs = [ + "comparisons_test.cc", + ], + tags = [ + "tflite_not_portable_ios_arm64", + "tflite_not_portable_ios_x86_64", + ], + deps = [ + ":builtin_ops", + "//tensorflow/contrib/lite:framework", + "//tensorflow/contrib/lite/kernels:test_util", + "@com_google_googletest//:gtest", + ], +) + filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow/contrib/lite/kernels/comparisons.cc b/tensorflow/contrib/lite/kernels/comparisons.cc new file mode 100644 index 0000000000..87c413cb98 --- /dev/null +++ b/tensorflow/contrib/lite/kernels/comparisons.cc @@ -0,0 +1,119 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/context.h" +#include "tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h" +#include "tensorflow/contrib/lite/kernels/internal/tensor.h" +#include "tensorflow/contrib/lite/kernels/kernel_util.h" +#include "tensorflow/contrib/lite/kernels/op_macros.h" +#include "tensorflow/contrib/lite/string_util.h" + +namespace tflite { +namespace ops { +namespace builtin { +namespace comparisons { + +constexpr int kInputTensor1 = 0; +constexpr int kInputTensor2 = 1; +constexpr int kOutputTensor = 0; + +TfLiteStatus LessPrepare(TfLiteContext* context, TfLiteNode* node) { + TF_LITE_ENSURE_EQ(context, NumInputs(node), 2); + TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); + + TfLiteTensor* input1 = GetInput(context, node, kInputTensor1); + TfLiteTensor* input2 = GetInput(context, node, kInputTensor2); + TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + + // Don't support string and bool. + TF_LITE_ENSURE(context, + input1->type != kTfLiteString || input1->type != kTfLiteBool); + // Currently only support tensors have the same type. + TF_LITE_ENSURE_EQ(context, input1->type, input2->type); + output->type = kTfLiteBool; + + bool requires_broadcast = !HaveSameShapes(input1, input2); + + TfLiteIntArray* output_size = nullptr; + if (requires_broadcast) { + TF_LITE_ENSURE_OK(context, CalculateShapeForBroadcast( + context, input1, input2, &output_size)); + } else { + output_size = TfLiteIntArrayCopy(input1->dims); + } + + return context->ResizeTensor(context, output, output_size); +} + +TfLiteStatus LessEval(TfLiteContext* context, TfLiteNode* node) { + TfLiteTensor* input1 = GetInput(context, node, kInputTensor1); + TfLiteTensor* input2 = GetInput(context, node, kInputTensor2); + TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + + bool requires_broadcast = !HaveSameShapes(input1, input2); + +#define TF_LITE_LESS(type, opname) \ + reference_ops::opname(GetTensorData(input1), GetTensorDims(input1), \ + GetTensorData(input2), GetTensorDims(input2), \ + GetTensorData(output), GetTensorDims(output)); + + // TODO(renjieliu): Support quantized data. + if (requires_broadcast) { + switch (input1->type) { + case kTfLiteFloat32: + TF_LITE_LESS(float, BroadcastLess); + break; + case kTfLiteInt32: + TF_LITE_LESS(int32_t, BroadcastLess); + break; + case kTfLiteInt64: + TF_LITE_LESS(int64_t, BroadcastLess); + break; + default: + context->ReportError(context, + "Does not support type other than float|int"); + return kTfLiteError; + } + } else { + switch (input1->type) { + case kTfLiteFloat32: + TF_LITE_LESS(float, Less); + break; + case kTfLiteInt32: + TF_LITE_LESS(int32_t, Less); + break; + case kTfLiteInt64: + TF_LITE_LESS(int64_t, Less); + break; + default: + context->ReportError(context, + "Does not support type other than float|int"); + return kTfLiteError; + } + } +#undef TF_LITE_LESS + return kTfLiteOk; +} + +} // namespace comparisons + +TfLiteRegistration* Register_LESS() { + static TfLiteRegistration r = {nullptr, nullptr, comparisons::LessPrepare, + comparisons::LessEval}; + return &r; +} + +} // namespace builtin +} // namespace ops +} // namespace tflite diff --git a/tensorflow/contrib/lite/kernels/comparisons_test.cc b/tensorflow/contrib/lite/kernels/comparisons_test.cc new file mode 100644 index 0000000000..da2d7f8589 --- /dev/null +++ b/tensorflow/contrib/lite/kernels/comparisons_test.cc @@ -0,0 +1,98 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include "tensorflow/contrib/lite/interpreter.h" +#include "tensorflow/contrib/lite/kernels/register.h" +#include "tensorflow/contrib/lite/kernels/test_util.h" +#include "tensorflow/contrib/lite/model.h" + +namespace tflite { +namespace { + +using ::testing::ElementsAreArray; + +class LessOpModel : public SingleOpModel { + public: + LessOpModel(std::initializer_list input1_shape, + std::initializer_list input2_shape, TensorType input_type) { + input1_ = AddInput(input_type); + input2_ = AddInput(input_type); + output_ = AddOutput(TensorType_BOOL); + SetBuiltinOp(BuiltinOperator_LESS, BuiltinOptions_LessOptions, + CreateLessOptions(builder_).Union()); + BuildInterpreter({input1_shape, input2_shape}); + } + + int input1() { return input1_; } + int input2() { return input2_; } + + std::vector GetOutput() { return ExtractVector(output_); } + std::vector GetOutputShape() { return GetTensorShape(output_); } + + private: + int input1_; + int input2_; + int output_; +}; + +TEST(ArgMaxOpTest, LessFloat) { + LessOpModel model({1, 1, 1, 4}, {1, 1, 1, 4}, TensorType_FLOAT32); + model.PopulateTensor(model.input1(), {0.1, 0.9, 0.7, 0.3}); + model.PopulateTensor(model.input2(), {0.1, 0.2, 0.6, 0.5}); + model.Invoke(); + + EXPECT_THAT(model.GetOutput(), ElementsAreArray({false, false, false, true})); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 1, 4})); +} + +TEST(ArgMaxOpTest, LessInt) { + LessOpModel model({1, 1, 1, 4}, {1, 1, 1, 4}, TensorType_INT32); + model.PopulateTensor(model.input1(), {-1, 9, 7, 3}); + model.PopulateTensor(model.input2(), {1, 2, 6, 5}); + model.Invoke(); + + EXPECT_THAT(model.GetOutput(), ElementsAreArray({true, false, false, true})); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 1, 4})); +} + +TEST(ArgMaxOpTest, LessBroadcast) { + LessOpModel model({1, 1, 1, 4}, {1, 1, 1, 1}, TensorType_INT32); + model.PopulateTensor(model.input1(), {-1, 9, 7, 3}); + model.PopulateTensor(model.input2(), {7}); + model.Invoke(); + + EXPECT_THAT(model.GetOutput(), ElementsAreArray({true, false, false, true})); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 1, 4})); +} + +TEST(ArgMaxOpTest, LessBroadcastTwoD) { + LessOpModel model({1, 1, 2, 4}, {1, 1, 1, 4}, TensorType_INT32); + model.PopulateTensor(model.input1(), {-1, 9, 7, 3, 2, 4, 6, 8}); + model.PopulateTensor(model.input2(), {7, 1, 2, 4}); + model.Invoke(); + + EXPECT_THAT(model.GetOutput(), ElementsAreArray({true, false, false, true, + true, false, false, false})); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 1, 2, 4})); +} + +} // namespace +} // namespace tflite + +int main(int argc, char** argv) { + ::tflite::LogToStderr(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h index c6019390f2..6a89dbc803 100644 --- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h +++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h @@ -3378,6 +3378,51 @@ inline void TransposeConv(const float* input_data, const Dims<4>& input_dims, } } +template +inline void Less(int64_t num_elements, const T* input1, const T* input2, + bool* output) { + for (int64_t i = 0; i < num_elements; ++i) { + output[i] = input1[i] < input2[i]; + } +} + +template +inline void Less(const T* input1_data, const Dims<4>& input1_dims, + const T* input2_data, const Dims<4>& input2_dims, + bool* output_data, const Dims<4>& output_dims) { + const int64_t batches = + MatchingArraySize(input1_dims, 3, input2_dims, 3, output_dims, 3); + const int64_t height = + MatchingArraySize(input1_dims, 2, input2_dims, 2, output_dims, 2); + const int64_t width = + MatchingArraySize(input1_dims, 1, input2_dims, 1, output_dims, 1); + const int64_t depth = + MatchingArraySize(input1_dims, 0, input2_dims, 0, output_dims, 0); + Less(batches * height * width * depth, input1_data, input2_data, output_data); +} + +template +inline void BroadcastLess(T1* input1_data, const Dims<4>& input1_dims, + T2* input2_data, const Dims<4>& input2_dims, + bool* output_data, const Dims<4>& output_dims) { + gemmlowp::ScopedProfilingLabel label("BroadcastLess"); + NdArrayDesc<4> desc1; + NdArrayDesc<4> desc2; + NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2); + + for (int b = 0; b < ArraySize(output_dims, 3); ++b) { + for (int y = 0; y < ArraySize(output_dims, 2); ++y) { + for (int x = 0; x < ArraySize(output_dims, 1); ++x) { + for (int c = 0; c < ArraySize(output_dims, 0); ++c) { + output_data[Offset(output_dims, c, x, y, b)] = + input1_data[SubscriptToIndex(desc1, c, x, y, b)] < + input2_data[SubscriptToIndex(desc2, c, x, y, b)]; + } + } + } + } +} + } // namespace reference_ops } // namespace tflite diff --git a/tensorflow/contrib/lite/kernels/register.cc b/tensorflow/contrib/lite/kernels/register.cc index 67ba8d0f39..b07e7b6ff3 100644 --- a/tensorflow/contrib/lite/kernels/register.cc +++ b/tensorflow/contrib/lite/kernels/register.cc @@ -79,6 +79,7 @@ TfLiteRegistration* Register_PRELU(); TfLiteRegistration* Register_MAXIMUM(); TfLiteRegistration* Register_MINIMUM(); TfLiteRegistration* Register_ARG_MAX(); +TfLiteRegistration* Register_LESS(); BuiltinOpResolver::BuiltinOpResolver() { AddBuiltin(BuiltinOperator_RELU, Register_RELU()); @@ -139,6 +140,7 @@ BuiltinOpResolver::BuiltinOpResolver() { AddBuiltin(BuiltinOperator_MAXIMUM, Register_MAXIMUM()); AddBuiltin(BuiltinOperator_MINIMUM, Register_MINIMUM()); AddBuiltin(BuiltinOperator_ARG_MAX, Register_ARG_MAX()); + AddBuiltin(BuiltinOperator_LESS, Register_LESS()); // TODO(andrewharp, ahentz): Move these somewhere more appropriate so that // custom ops aren't always included by default. diff --git a/tensorflow/contrib/lite/model.cc b/tensorflow/contrib/lite/model.cc index 0b65884025..54b1460173 100644 --- a/tensorflow/contrib/lite/model.cc +++ b/tensorflow/contrib/lite/model.cc @@ -665,6 +665,9 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, *builtin_data = reinterpret_cast(params); break; } + case BuiltinOperator_LESS: { + break; + } case BuiltinOperator_DELEGATE: { // TODO(ycling): Revisit when supporting saving delegated models. error_reporter->Report("DELEGATE op shouldn't exist in model."); diff --git a/tensorflow/contrib/lite/nnapi_delegate.cc b/tensorflow/contrib/lite/nnapi_delegate.cc index 08fb820767..eab82ea8ef 100644 --- a/tensorflow/contrib/lite/nnapi_delegate.cc +++ b/tensorflow/contrib/lite/nnapi_delegate.cc @@ -353,6 +353,7 @@ void AddOpsAndParams(tflite::Interpreter* interpreter, case tflite::BuiltinOperator_MAXIMUM: case tflite::BuiltinOperator_MINIMUM: case tflite::BuiltinOperator_ARG_MAX: + case tflite::BuiltinOperator_LESS: FATAL("Op code %d is currently not delegated to NNAPI", builtin); nn_op_type = -1; // set to invalid break; diff --git a/tensorflow/contrib/lite/schema/schema.fbs b/tensorflow/contrib/lite/schema/schema.fbs index fa825500fd..93980b15f0 100644 --- a/tensorflow/contrib/lite/schema/schema.fbs +++ b/tensorflow/contrib/lite/schema/schema.fbs @@ -135,6 +135,7 @@ enum BuiltinOperator : byte { MAXIMUM = 55, ARG_MAX = 56, MINIMUM = 57, + LESS = 58, } // Options for the builtin operators. @@ -179,6 +180,7 @@ union BuiltinOptions { DequantizeOptions, MaximumMinimumOptions, ArgMaxOptions, + LessOptions, } enum Padding : byte { SAME, VALID } @@ -399,6 +401,9 @@ table ArgMaxOptions { output_type : TensorType; } +table LessOptions { +} + // An OperatorCode can be an enum value (BuiltinOperator) if the operator is a // builtin, or a string if the operator is custom. table OperatorCode { diff --git a/tensorflow/contrib/lite/schema/schema_generated.h b/tensorflow/contrib/lite/schema/schema_generated.h index 909c4ccb3b..b2a799d0ef 100755 --- a/tensorflow/contrib/lite/schema/schema_generated.h +++ b/tensorflow/contrib/lite/schema/schema_generated.h @@ -151,6 +151,9 @@ struct MaximumMinimumOptionsT; struct ArgMaxOptions; struct ArgMaxOptionsT; +struct LessOptions; +struct LessOptionsT; + struct OperatorCode; struct OperatorCodeT; @@ -267,11 +270,12 @@ enum BuiltinOperator { BuiltinOperator_MAXIMUM = 55, BuiltinOperator_ARG_MAX = 56, BuiltinOperator_MINIMUM = 57, + BuiltinOperator_LESS = 58, BuiltinOperator_MIN = BuiltinOperator_ADD, - BuiltinOperator_MAX = BuiltinOperator_MINIMUM + BuiltinOperator_MAX = BuiltinOperator_LESS }; -inline BuiltinOperator (&EnumValuesBuiltinOperator())[56] { +inline BuiltinOperator (&EnumValuesBuiltinOperator())[57] { static BuiltinOperator values[] = { BuiltinOperator_ADD, BuiltinOperator_AVERAGE_POOL_2D, @@ -328,7 +332,8 @@ inline BuiltinOperator (&EnumValuesBuiltinOperator())[56] { BuiltinOperator_PRELU, BuiltinOperator_MAXIMUM, BuiltinOperator_ARG_MAX, - BuiltinOperator_MINIMUM + BuiltinOperator_MINIMUM, + BuiltinOperator_LESS }; return values; } @@ -393,6 +398,7 @@ inline const char **EnumNamesBuiltinOperator() { "MAXIMUM", "ARG_MAX", "MINIMUM", + "LESS", nullptr }; return names; @@ -445,11 +451,12 @@ enum BuiltinOptions { BuiltinOptions_DequantizeOptions = 38, BuiltinOptions_MaximumMinimumOptions = 39, BuiltinOptions_ArgMaxOptions = 40, + BuiltinOptions_LessOptions = 41, BuiltinOptions_MIN = BuiltinOptions_NONE, - BuiltinOptions_MAX = BuiltinOptions_ArgMaxOptions + BuiltinOptions_MAX = BuiltinOptions_LessOptions }; -inline BuiltinOptions (&EnumValuesBuiltinOptions())[41] { +inline BuiltinOptions (&EnumValuesBuiltinOptions())[42] { static BuiltinOptions values[] = { BuiltinOptions_NONE, BuiltinOptions_Conv2DOptions, @@ -491,7 +498,8 @@ inline BuiltinOptions (&EnumValuesBuiltinOptions())[41] { BuiltinOptions_CastOptions, BuiltinOptions_DequantizeOptions, BuiltinOptions_MaximumMinimumOptions, - BuiltinOptions_ArgMaxOptions + BuiltinOptions_ArgMaxOptions, + BuiltinOptions_LessOptions }; return values; } @@ -539,6 +547,7 @@ inline const char **EnumNamesBuiltinOptions() { "DequantizeOptions", "MaximumMinimumOptions", "ArgMaxOptions", + "LessOptions", nullptr }; return names; @@ -713,6 +722,10 @@ template<> struct BuiltinOptionsTraits { static const BuiltinOptions enum_value = BuiltinOptions_ArgMaxOptions; }; +template<> struct BuiltinOptionsTraits { + static const BuiltinOptions enum_value = BuiltinOptions_LessOptions; +}; + struct BuiltinOptionsUnion { BuiltinOptions type; void *value; @@ -1064,6 +1077,14 @@ struct BuiltinOptionsUnion { return type == BuiltinOptions_ArgMaxOptions ? reinterpret_cast(value) : nullptr; } + LessOptionsT *AsLessOptions() { + return type == BuiltinOptions_LessOptions ? + reinterpret_cast(value) : nullptr; + } + const LessOptionsT *AsLessOptions() const { + return type == BuiltinOptions_LessOptions ? + reinterpret_cast(value) : nullptr; + } }; bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *obj, BuiltinOptions type); @@ -3927,6 +3948,46 @@ inline flatbuffers::Offset CreateArgMaxOptions( flatbuffers::Offset CreateArgMaxOptions(flatbuffers::FlatBufferBuilder &_fbb, const ArgMaxOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +struct LessOptionsT : public flatbuffers::NativeTable { + typedef LessOptions TableType; + LessOptionsT() { + } +}; + +struct LessOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + typedef LessOptionsT NativeTableType; + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + verifier.EndTable(); + } + LessOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(LessOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const; + static flatbuffers::Offset Pack(flatbuffers::FlatBufferBuilder &_fbb, const LessOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +}; + +struct LessOptionsBuilder { + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + explicit LessOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + LessOptionsBuilder &operator=(const LessOptionsBuilder &); + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + return o; + } +}; + +inline flatbuffers::Offset CreateLessOptions( + flatbuffers::FlatBufferBuilder &_fbb) { + LessOptionsBuilder builder_(_fbb); + return builder_.Finish(); +} + +flatbuffers::Offset CreateLessOptions(flatbuffers::FlatBufferBuilder &_fbb, const LessOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); + struct OperatorCodeT : public flatbuffers::NativeTable { typedef OperatorCode TableType; BuiltinOperator builtin_code; @@ -4164,6 +4225,9 @@ struct Operator FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { const ArgMaxOptions *builtin_options_as_ArgMaxOptions() const { return builtin_options_type() == BuiltinOptions_ArgMaxOptions ? static_cast(builtin_options()) : nullptr; } + const LessOptions *builtin_options_as_LessOptions() const { + return builtin_options_type() == BuiltinOptions_LessOptions ? static_cast(builtin_options()) : nullptr; + } const flatbuffers::Vector *custom_options() const { return GetPointer *>(VT_CUSTOM_OPTIONS); } @@ -4350,6 +4414,10 @@ template<> inline const ArgMaxOptions *Operator::builtin_options_as inline const LessOptions *Operator::builtin_options_as() const { + return builtin_options_as_LessOptions(); +} + struct OperatorBuilder { flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; @@ -5933,6 +6001,29 @@ inline flatbuffers::Offset CreateArgMaxOptions(flatbuffers::FlatB _output_type); } +inline LessOptionsT *LessOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { + auto _o = new LessOptionsT(); + UnPackTo(_o, _resolver); + return _o; +} + +inline void LessOptions::UnPackTo(LessOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { + (void)_o; + (void)_resolver; +} + +inline flatbuffers::Offset LessOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const LessOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) { + return CreateLessOptions(_fbb, _o, _rehasher); +} + +inline flatbuffers::Offset CreateLessOptions(flatbuffers::FlatBufferBuilder &_fbb, const LessOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) { + (void)_rehasher; + (void)_o; + struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const LessOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + return tflite::CreateLessOptions( + _fbb); +} + inline OperatorCodeT *OperatorCode::UnPack(const flatbuffers::resolver_function_t *_resolver) const { auto _o = new OperatorCodeT(); UnPackTo(_o, _resolver); @@ -6273,6 +6364,10 @@ inline bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *ob auto ptr = reinterpret_cast(obj); return verifier.VerifyTable(ptr); } + case BuiltinOptions_LessOptions: { + auto ptr = reinterpret_cast(obj); + return verifier.VerifyTable(ptr); + } default: return false; } } @@ -6451,6 +6546,10 @@ inline void *BuiltinOptionsUnion::UnPack(const void *obj, BuiltinOptions type, c auto ptr = reinterpret_cast(obj); return ptr->UnPack(resolver); } + case BuiltinOptions_LessOptions: { + auto ptr = reinterpret_cast(obj); + return ptr->UnPack(resolver); + } default: return nullptr; } } @@ -6617,6 +6716,10 @@ inline flatbuffers::Offset BuiltinOptionsUnion::Pack(flatbuffers::FlatBuff auto ptr = reinterpret_cast(value); return CreateArgMaxOptions(_fbb, ptr, _rehasher).Union(); } + case BuiltinOptions_LessOptions: { + auto ptr = reinterpret_cast(value); + return CreateLessOptions(_fbb, ptr, _rehasher).Union(); + } default: return 0; } } @@ -6783,6 +6886,10 @@ inline BuiltinOptionsUnion::BuiltinOptionsUnion(const BuiltinOptionsUnion &u) FL value = new ArgMaxOptionsT(*reinterpret_cast(u.value)); break; } + case BuiltinOptions_LessOptions: { + value = new LessOptionsT(*reinterpret_cast(u.value)); + break; + } default: break; } @@ -6990,6 +7097,11 @@ inline void BuiltinOptionsUnion::Reset() { delete ptr; break; } + case BuiltinOptions_LessOptions: { + auto ptr = reinterpret_cast(value); + delete ptr; + break; + } default: break; } value = nullptr; diff --git a/tensorflow/contrib/lite/testing/BUILD b/tensorflow/contrib/lite/testing/BUILD index 2c226e76d4..bd888a415b 100644 --- a/tensorflow/contrib/lite/testing/BUILD +++ b/tensorflow/contrib/lite/testing/BUILD @@ -34,6 +34,7 @@ gen_zipped_test_files( "global_batch_norm.zip", "l2_pool.zip", "l2norm.zip", + "less.zip", "local_response_norm.zip", "log_softmax.zip", "max_pool.zip", diff --git a/tensorflow/contrib/lite/testing/generate_examples.py b/tensorflow/contrib/lite/testing/generate_examples.py index 4b4ccc0c37..53b41d2358 100644 --- a/tensorflow/contrib/lite/testing/generate_examples.py +++ b/tensorflow/contrib/lite/testing/generate_examples.py @@ -1997,6 +1997,39 @@ def make_arg_max_tests(zip_path): make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs) +def make_less_tests(zip_path): + """Make a set of tests to do less.""" + + test_parameters = [{ + "input_dtype": [tf.float32, tf.int32, tf.int64], + "input_shape_pair": [([1, 1, 1, 3], [1, 1, 1, 3]), + ([2, 3, 4, 5], [2, 3, 4, 5]), ([2, 3, 3], [2, 3]), + ([5, 5], [1]), ([10], [2, 4, 10])], + }] + + def build_graph(parameters): + """Build the less op testing graph.""" + input_value1 = tf.placeholder( + dtype=parameters["input_dtype"], + name="input1", + shape=parameters["input_shape_pair"][0]) + input_value2 = tf.placeholder( + dtype=parameters["input_dtype"], + name="input2", + shape=parameters["input_shape_pair"][1]) + out = tf.less(input_value1, input_value2) + return [input_value1, input_value2], [out] + + def build_inputs(parameters, sess, inputs, outputs): + input_value1 = create_tensor_data(parameters["input_dtype"], + parameters["input_shape_pair"][0]) + input_value2 = create_tensor_data(parameters["input_dtype"], + parameters["input_shape_pair"][1]) + return [input_value1, input_value2], sess.run( + outputs, feed_dict=dict(zip(inputs, [input_value1, input_value2]))) + + make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs) + # Toco binary path provided by the generate rule. bin_path = None diff --git a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc index 84ae1d58fe..9da8bd7a28 100644 --- a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc +++ b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc @@ -280,6 +280,7 @@ INSTANTIATE_TESTS(squeeze) INSTANTIATE_TESTS(strided_slice) INSTANTIATE_TESTS(sub) INSTANTIATE_TESTS(transpose) +INSTANTIATE_TESTS(less) } // namespace testing } // namespace tflite diff --git a/tensorflow/contrib/lite/toco/tflite/operator.cc b/tensorflow/contrib/lite/toco/tflite/operator.cc index 0e057fd252..f41a312b47 100644 --- a/tensorflow/contrib/lite/toco/tflite/operator.cc +++ b/tensorflow/contrib/lite/toco/tflite/operator.cc @@ -895,6 +895,8 @@ std::vector> BuildOperatorList() { "MAXIMUM", OperatorType::kTensorFlowMaximum)); ops.emplace_back(new SimpleOperator( "MINIMUM", OperatorType::kTensorFlowMinimum)); + ops.emplace_back(new SimpleOperator( + "LESS", OperatorType::kTensorFlowLess)); return ops; } diff --git a/tensorflow/contrib/lite/toco/tflite/operator_test.cc b/tensorflow/contrib/lite/toco/tflite/operator_test.cc index a947630e28..36ed741541 100644 --- a/tensorflow/contrib/lite/toco/tflite/operator_test.cc +++ b/tensorflow/contrib/lite/toco/tflite/operator_test.cc @@ -113,6 +113,8 @@ TEST_F(OperatorTest, SimpleOperators) { "MAXIMUM", OperatorType::kTensorFlowMaximum); CheckSimpleOperator( "MINIMUM", OperatorType::kTensorFlowMinimum); + CheckSimpleOperator("LESS", + OperatorType::kTensorFlowLess); } TEST_F(OperatorTest, BuiltinAdd) { -- GitLab From 786668c8300f8f88c21493ecfa500a097a80ccd8 Mon Sep 17 00:00:00 2001 From: jjsjann123 Date: Fri, 13 Apr 2018 04:21:15 -0700 Subject: [PATCH 309/791] updated installation instructions for Tensowflow-TensorRT integration (#18135) * updated installation instructions for Tensowflow-TensorRT integration * Minor format changes to clean it up. * Adding the python symlink command for devel packages too. * Forcing the symlink creation. * Updating the sed command for docker parameterized build. --- tensorflow/contrib/tensorrt/README.md | 60 +++++--------------- tensorflow/docs_src/install/install_linux.md | 36 +++++++++--- 2 files changed, 44 insertions(+), 52 deletions(-) diff --git a/tensorflow/contrib/tensorrt/README.md b/tensorflow/contrib/tensorrt/README.md index 6eafc1754c..687dee07e1 100644 --- a/tensorflow/contrib/tensorrt/README.md +++ b/tensorflow/contrib/tensorrt/README.md @@ -1,59 +1,29 @@ # Using TensorRT in TensorFlow - -This module provides necessary bindings and introduces TRT_engine_op -operator that wraps a subgraph in TensorRT. This is still a work in progress -but should be useable with most common graphs. +This module provides necessary bindings and introduces TRT_engine_op operator +that wraps a subgraph in TensorRT. This is still a work in progress but should +be useable with most common graphs. ## Compilation - -In order to compile the module, you need to have a local TensorRT -installation ( libnvinfer.so and respective include files ). During the -configuration step, TensorRT should be enabled and installation path -should be set. If installed through package managers (deb,rpm), -configure script should find the necessary components from the system -automatically. If installed from tar packages, user has to set path to -location where the library is installed during configuration. +In order to compile the module, you need to have a local TensorRT installation +(libnvinfer.so and respective include files). During the configuration step, +TensorRT should be enabled and installation path should be set. If installed +through package managers (deb,rpm), configure script should find the necessary +components from the system automatically. If installed from tar packages, user +has to set path to location where the library is installed during configuration. ```shell bazel build --config=cuda --config=opt //tensorflow/tools/pip_package:build_pip_package bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/ ``` -After the installation of tensorflow package, TensorRT transformation -will be available. An example use can be found in test/test_tftrt.py script +After the installation of tensorflow package, TensorRT transformation will be +available. An example use can be found in test/test_tftrt.py script ## Installing TensorRT 3.0.4 -In order to make use of TensorRT integration, you will need a local installation of TensorRT 3.0.4 from the [NVIDIA Developer website](https://developer.nvidia.com/tensorrt). Due to compiler compatibility, you will need to download and install the TensorRT 3.0.4 tarball for _Ubuntu 14.04_, i.e., **_TensorRT-3.0.4.Ubuntu-14.04.5.x86_64.cuda-9.0.cudnn7.0-tar.gz_**, even if you are using Ubuntu 16.04 or later. - -### Preparing TensorRT installation - -Once you have downloaded TensorRT-3.0.4.Ubuntu-14.04.5.x86_64.cuda-9.0.cudnn7.0-tar.gz, you will need to unpack it to an installation directory, which will be referred to as . Please replace with the full path of actual installation directory you choose in commands below. - -```shell -cd && tar -zxf /path/to/TensorRT-3.0.4.Ubuntu-14.04.5.x86_64.cuda-9.0.cudnn7.0-tar.gz -``` - -After unpacking the binaries, you have several options to use them: - -#### To run TensorFlow as a user without superuser privileges - -For a regular user without any sudo rights, you should add TensorRT to your `$LD_LIBRARY_PATH`: - - ```shell - export LD_LIBRARY_PATH=/TensorRT-3.0.4/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} - ``` - -Then you are ready to use TensorFlow-TensorRT integration. `$LD_LIBRARY_PATH` must contain the path to TensorRT installation for TensorFlow-TensorRT integration to work. If you are using a VirtualEnv-like setup, you can add the command above to your `bin/activate` script or to your `.bashrc` script. - -#### To run TensorFlow as a superuser - - When running as a superuser, such as in a container or via sudo, the `$LD_LIBRARY_PATH` approach above may not work. The following is preferred when the user has superuser privileges: - - ```shell - echo "/TensorRT-3.0.4/lib" | sudo tee /etc/ld.so.conf.d/tensorrt304.conf && sudo ldconfig - ``` - - Please ensure that any existing deb package installation of TensorRT is removed before following these instructions to avoid package conflicts. \ No newline at end of file +In order to make use of TensorRT integration, you will need a local installation +of TensorRT 3.0.4 from the [NVIDIA Developer website](https://developer.nvidia.com/tensorrt). +Installation instructions for compatibility with TensorFlow are provided on the +[TensorFlow Installation page](https://www.tensorflow.org/install/install_linux#nvidia_requirements_to_run_tensorflow_with_gpu_support). diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md index 04e4242b0f..58f6c60287 100644 --- a/tensorflow/docs_src/install/install_linux.md +++ b/tensorflow/docs_src/install/install_linux.md @@ -65,16 +65,38 @@ must be installed on your system: 
     $ sudo apt-get install libcupti-dev
+ * **[OPTIONAL]** For optimized inferencing performance, you can also install - NVIDIA TensorRT 3.0. For details, see - [NVIDIA's TensorRT documentation](http://docs.nvidia.com/deeplearning/sdk/tensorrt-install-guide/index.html#installing-tar). - Only steps 1-4 in the TensorRT Tar File installation instructions are - required for compatibility with TensorFlow; the Python package installation - in steps 5 and 6 can be omitted. Detailed installation instructions can be found at [package documentataion](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/tensorrt#installing-tensorrt-304) + **NVIDIA TensorRT 3.0**. The minimal set of TensorRT runtime components needed + for use with the pre-built `tensorflow-gpu` package can be installed as follows: + + 
+    $ wget https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1404/x86_64/nvinfer-runtime-trt-repo-ubuntu1404-3.0.4-ga-cuda9.0_1.0-1_amd64.deb
+    $ sudo dpkg -i nvinfer-runtime-trt-repo-ubuntu1404-3.0.4-ga-cuda9.0_1.0-1_amd64.deb
+    $ sudo apt-get update
+    $ sudo apt-get install -y --allow-downgrades libnvinfer-dev libcudnn7-dev=7.0.5.15-1+cuda9.0 libcudnn7=7.0.5.15-1+cuda9.0
+
**IMPORTANT:** For compatibility with the pre-built `tensorflow-gpu` - package, please use the Ubuntu **14.04** tar file package of TensorRT - even when installing onto an Ubuntu 16.04 system. + package, please use the Ubuntu **14.04** package of TensorRT as shown above, + even when installing onto an Ubuntu 16.04 system.
+
+ To build the TensorFlow-TensorRT integration module from source rather than + using pre-built binaries, see the [module documentation](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/tensorrt#using-tensorrt-in-tensorflow). + For detailed TensorRT installation instructions, see [NVIDIA's TensorRT documentation](http://docs.nvidia.com/deeplearning/sdk/tensorrt-install-guide/index.html).
+
+ To avoid cuDNN version conflicts during later system upgrades, you can hold + the cuDNN version at 7.0.5: + + 
+    $  sudo apt-mark hold libcudnn7 libcudnn7-dev
+
+ + To later allow upgrades, you can remove the hold: + + 
+    $  sudo apt-mark unhold libcudnn7 libcudnn7-dev
+
If you have an earlier version of the preceding packages, please upgrade to the specified versions. If upgrading is not possible, then you may still run -- GitLab From bb804104e27400b5e0497cf6c60f4a46a7402d23 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 04:44:10 -0700 Subject: [PATCH 310/791] Fix bug in converted_call, and add tests for it. PiperOrigin-RevId: 192751211 --- tensorflow/contrib/autograph/impl/api.py | 2 +- tensorflow/contrib/autograph/impl/api_test.py | 86 +++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/autograph/impl/api.py b/tensorflow/contrib/autograph/impl/api.py index a553813e19..a00d9c68dc 100644 --- a/tensorflow/contrib/autograph/impl/api.py +++ b/tensorflow/contrib/autograph/impl/api.py @@ -156,7 +156,7 @@ def converted_call(f, recursive, verbose, arg_types, *args, **kwargs): # Constructors target_entity = f arg_map_target = f.__init__ - effective_args = (unknown_arg_value,) + args + effective_args = args partial_types = () elif hasattr(f, '__call__') and hasattr(f, '__class__'): diff --git a/tensorflow/contrib/autograph/impl/api_test.py b/tensorflow/contrib/autograph/impl/api_test.py index f9db07778a..2e09d19621 100644 --- a/tensorflow/contrib/autograph/impl/api_test.py +++ b/tensorflow/contrib/autograph/impl/api_test.py @@ -179,6 +179,92 @@ class ApiTest(test.TestCase): constant_op.constant(-2)) self.assertListEqual([0, 1], sess.run(x).tolist()) + def test_converted_call_builtin(self): + x = api.converted_call(range, False, False, {}, 3) + self.assertEqual((0, 1, 2), tuple(x)) + + def test_converted_call_function(self): + + def test_fn(x): + if x < 0: + return -x + return x + + with self.test_session() as sess: + x = api.converted_call( + test_fn, False, False, {}, constant_op.constant(-1)) + self.assertEqual(1, sess.run(x)) + + def test_converted_call_method(self): + + class TestClass(object): + + def __init__(self, x): + self.x = x + + def test_method(self): + if self.x < 0: + return -self.x + return self.x + + with self.test_session() as sess: + tc = TestClass(constant_op.constant(-1)) + x = api.converted_call(tc.test_method, False, False, {}, tc) + self.assertEqual(1, sess.run(x)) + + def test_converted_call_method_by_class(self): + + class TestClass(object): + + def __init__(self, x): + self.x = x + + def test_method(self): + if self.x < 0: + return -self.x + return self.x + + with self.test_session() as sess: + tc = TestClass(constant_op.constant(-1)) + x = api.converted_call(TestClass.test_method, False, False, {}, tc) + self.assertEqual(1, sess.run(x)) + + def test_converted_call_callable_object(self): + + class TestClass(object): + + def __init__(self, x): + self.x = x + + def __call__(self): + if self.x < 0: + return -self.x + return self.x + + with self.test_session() as sess: + tc = TestClass(constant_op.constant(-1)) + x = api.converted_call(tc, False, False, {}) + self.assertEqual(1, sess.run(x)) + + def test_converted_call_constructor(self): + + class TestClass(object): + + def __init__(self, x): + self.x = x + + def test_method(self): + if self.x < 0: + return -self.x + return self.x + + with self.test_session() as sess: + tc = api.converted_call( + TestClass, False, False, {}, constant_op.constant(-1)) + # tc is now a converted object. + x = tc.test_method() + self.assertEqual(1, sess.run(x)) + def test_to_graph_basic(self): def test_fn(x, s): -- GitLab From b520022c95b246749fa3f63ca818058c22944720 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 05:05:12 -0700 Subject: [PATCH 311/791] Update for upstream LLVM *.def -> *.inc rename PiperOrigin-RevId: 192752798 --- .../xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.cc b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.cc index defd281d74..df9d9be889 100644 --- a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.cc +++ b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.cc @@ -34,7 +34,7 @@ limitations under the License. #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Bitcode/BitcodeReader.h" #include "llvm/Bitcode/BitcodeWriter.h" -#include "llvm/CodeGen/CommandFlags.def" +#include "llvm/CodeGen/CommandFlags.inc" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" -- GitLab From 345414cb4fa43af5906adc64a380986eaade4f53 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 05:47:21 -0700 Subject: [PATCH 312/791] - Fixed small bug in example script PiperOrigin-RevId: 192756152 --- tensorflow/contrib/kfac/examples/convnet_mnist_single_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/kfac/examples/convnet_mnist_single_main.py b/tensorflow/contrib/kfac/examples/convnet_mnist_single_main.py index 3aa52aff19..2c1f099360 100644 --- a/tensorflow/contrib/kfac/examples/convnet_mnist_single_main.py +++ b/tensorflow/contrib/kfac/examples/convnet_mnist_single_main.py @@ -32,7 +32,7 @@ flags.DEFINE_string("data_dir", "/tmp/mnist", "local mnist dir") def main(unused_argv): - convnet.train_mnist_single_gpu(FLAGS.data_dir, num_epochs=200) + convnet.train_mnist_single_machine(FLAGS.data_dir, num_epochs=200) if __name__ == "__main__": -- GitLab From bb8fcd516ebd0a11e1768d308d3aa265b9ad50d2 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 06:53:54 -0700 Subject: [PATCH 313/791] Keep function doc string at the top of the function. PiperOrigin-RevId: 192761604 --- .../autograph/converters/name_scopes.py | 20 +++++++++---- .../autograph/converters/name_scopes_test.py | 30 ++++++++++++++++--- 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/tensorflow/contrib/autograph/converters/name_scopes.py b/tensorflow/contrib/autograph/converters/name_scopes.py index 280bc4c314..dfee529aba 100644 --- a/tensorflow/contrib/autograph/converters/name_scopes.py +++ b/tensorflow/contrib/autograph/converters/name_scopes.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Wraps a function body with a `name_scope` of the function name. -""" +"""Wraps a function body with a `name_scope` of the function name.""" from __future__ import absolute_import from __future__ import division @@ -48,15 +47,26 @@ class FunctionNameScopeTransformer(transformer.Base): return name def visit_FunctionDef(self, node): - self.generic_visit(node) + node = self.generic_visit(node) + + unscoped_body = [] + scoped_body = node.body + if scoped_body: + first = scoped_body[0] + if isinstance(first, gast.Expr) and isinstance(first.value, gast.Str): + # Skip any docstring. + unscoped_body = scoped_body[:1] + scoped_body = scoped_body[1:] + template = """ with tf.name_scope(scope_name): body """ - node.body = templates.replace( + scoped_body = templates.replace( template, scope_name=gast.Str(self._name_for_current_scope()), - body=node.body) + body=scoped_body) + node.body = unscoped_body + scoped_body return node diff --git a/tensorflow/contrib/autograph/converters/name_scopes_test.py b/tensorflow/contrib/autograph/converters/name_scopes_test.py index 2c2b6bbbec..17692cbd88 100644 --- a/tensorflow/contrib/autograph/converters/name_scopes_test.py +++ b/tensorflow/contrib/autograph/converters/name_scopes_test.py @@ -27,9 +27,10 @@ from tensorflow.python.platform import test class FunctionNameScopeTransformer(converter_test_base.TestCase): - def test_basic_name(self): + def test_basic(self): def test_fn(l): + """This should stay here.""" a = 5 l += a return l @@ -41,7 +42,28 @@ class FunctionNameScopeTransformer(converter_test_base.TestCase): result_op = result.test_fn(constant_op.constant(1)) self.assertIn('test_fn/', result_op.op.name) - def test_nested_name(self): + self.assertEqual('This should stay here.', result.test_fn.__doc__) + + def test_long_docstring(self): + + def test_fn(l): + """Multi-line docstring. + + Args: + l: A thing. + Returns: + l + """ + return l + + node = self.parse_and_analyze(test_fn, {}) + node = name_scopes.transform(node, self.ctx) + + with self.compiled(node, ops.name_scope) as result: + self.assertIn('Multi-line', result.test_fn.__doc__) + self.assertIn('Returns:', result.test_fn.__doc__) + + def test_nested_functions(self): def test_fn(l): @@ -62,7 +84,7 @@ class FunctionNameScopeTransformer(converter_test_base.TestCase): self.assertNotIn('inner_fn', first_result_input_name) self.assertIn('test_fn/inner_fn/', second_result_input_name) - def test_class_name(self): + def test_method(self): class TestClass(object): @@ -87,7 +109,7 @@ class FunctionNameScopeTransformer(converter_test_base.TestCase): self.assertNotIn('inner_fn', first_result_input_name) self.assertIn('TestClass/test_fn/inner_fn/', second_result_input_name) - def test_special_name(self): + def test_operator(self): class TestClass(object): -- GitLab From 8c47ec384eb28639934f8aee1a179b5b3d814af8 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 07:24:15 -0700 Subject: [PATCH 314/791] Demo: RNN colorbot with Estimators. PiperOrigin-RevId: 192765203 --- .../notebooks/rnn_colorbot_estimator.ipynb | 1421 +++++++++++++++++ 1 file changed, 1421 insertions(+) create mode 100644 tensorflow/contrib/autograph/examples/notebooks/rnn_colorbot_estimator.ipynb diff --git a/tensorflow/contrib/autograph/examples/notebooks/rnn_colorbot_estimator.ipynb b/tensorflow/contrib/autograph/examples/notebooks/rnn_colorbot_estimator.ipynb new file mode 100644 index 0000000000..7f5e4d4ac1 --- /dev/null +++ b/tensorflow/contrib/autograph/examples/notebooks/rnn_colorbot_estimator.ipynb @@ -0,0 +1,1421 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "LqNpENf-ec0X", + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [], + "source": [ + "!pip install -U tf-nightly" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "Pa2qpEmoVOGe", + "slideshow": { + "slide_type": "-" + } + }, + "outputs": [], + "source": [ + "import os\n", + "import time\n", + "\n", + "import tensorflow as tf\n", + "from tensorflow.contrib import autograph\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import six\n", + "\n", + "from google.colab import widgets" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "HNqUFL4deCsL", + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "# Case study: building an RNN\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "YkC1k4HEQ7rw", + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "In this section, we show how you can use AutoGraph to build RNNColorbot, an RNN that takes as input names of colors and predicts their corresponding RGB tuples. The model will be trained by a [custom Estimator](https://www.tensorflow.org/get_started/custom_estimators)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "7nkPDl5CTCNb", + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "To get started, set up the dataset. The following cells defines methods that download and format the data needed for RNNColorbot; the details aren't important (read them in the privacy of your own home if you so wish), but make sure to run the cells before proceeding." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "A0uREmVXCQEw", + "slideshow": { + "slide_type": "-" + } + }, + "outputs": [], + "source": [ + "def parse(line):\n", + " \"\"\"Parses a line from the colors dataset.\"\"\"\n", + " items = tf.string_split([line], \",\").values\n", + " rgb = tf.string_to_number(items[1:], out_type=tf.float32) / 255.0\n", + " color_name = items[0]\n", + " chars = tf.one_hot(tf.decode_raw(color_name, tf.uint8), depth=256)\n", + " length = tf.cast(tf.shape(chars)[0], dtype=tf.int64)\n", + " return rgb, chars, length\n", + "\n", + "def load_dataset(data_dir, url, batch_size, training=True):\n", + " \"\"\"Loads the colors data at path into a tf.PaddedDataset.\"\"\"\n", + " path = tf.keras.utils.get_file(os.path.basename(url), url, cache_dir=data_dir)\n", + " dataset = tf.data.TextLineDataset(path)\n", + " dataset = dataset.skip(1)\n", + " dataset = dataset.map(parse)\n", + " dataset = dataset.cache()\n", + " dataset = dataset.repeat()\n", + " if training:\n", + " dataset = dataset.shuffle(buffer_size=3000)\n", + " dataset = dataset.padded_batch(\n", + " batch_size, padded_shapes=([None], [None, None], []))\n", + " return dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "waZ89t3DTUla", + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "To show the use of control flow, we write the RNN loop by hand, rather than using a pre-built RNN model.\n", + "\n", + "Note how we write the model code in Eager style, with regular `if` and `while` statements. Then, we annotate the functions with `@autograph.convert` to have them automatically compiled to run in graph mode." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "9v8AJouiC44V", + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [], + "source": [ + "class RnnColorbot(object):\n", + " \"\"\"Holds the parameters of the colorbot model.\"\"\"\n", + "\n", + " def __init__(self):\n", + " self.lower_cell = tf.contrib.rnn.LSTMBlockCell(256)\n", + " self.upper_cell = tf.contrib.rnn.LSTMBlockCell(128)\n", + " self.relu_layer = tf.layers.Dense(3, activation=tf.nn.relu)\n", + "\n", + " self.lower_cell.build(tf.TensorShape((None, 256)))\n", + " self.upper_cell.build(tf.TensorShape((None, 256)))\n", + " self.relu_layer.build(tf.TensorShape((None, 128)))\n", + "\n", + "\n", + "def rnn_layer(chars, cell, batch_size, training):\n", + " \"\"\"A simple RNN layer.\n", + " \n", + " Args:\n", + " chars: A Tensor of shape (max_sequence_length, batch_size, input_size)\n", + " cell: An object of type tf.contrib.rnn.LSTMBlockCell\n", + " batch_size: Int, the batch size to use\n", + " training: Boolean, whether the layer is used for training\n", + "\n", + " Returns:\n", + " A Tensor of shape (max_sequence_length, batch_size, output_size).\n", + " \"\"\"\n", + " hidden_outputs = []\n", + " autograph.utils.set_element_type(hidden_outputs, tf.float32)\n", + " state, output = cell.zero_state(batch_size, tf.float32)\n", + " for ch in chars:\n", + " cell_output, (state, output) = cell.call(ch, (state, output))\n", + " hidden_outputs.append(cell_output)\n", + " hidden_outputs = hidden_outputs.stack()\n", + " if training:\n", + " hidden_outputs = tf.nn.dropout(hidden_outputs, 0.5)\n", + " return hidden_outputs\n", + "\n", + "\n", + "@autograph.convert(recursive=True)\n", + "def model(inputs, colorbot, batch_size, training):\n", + " \"\"\"RNNColorbot model.\n", + " \n", + " The model consists of two RNN layers (made by lower_cell and upper_cell),\n", + " followed by a fully connected layer with ReLU activation.\n", + " \n", + " Args:\n", + " inputs: A tuple (chars, length)\n", + " colorbot: An object of type RnnColorbot\n", + " batch_size: Int, the batch size to use\n", + " training: Boolean, whether the layer is used for training\n", + " \n", + " Returns:\n", + " A Tensor of shape (batch_size, 3) - the model predictions.\n", + " \"\"\"\n", + " (chars, length) = inputs\n", + " seq = tf.transpose(chars, [1, 0, 2])\n", + " seq.set_shape((None, batch_size, 256))\n", + "\n", + " seq = rnn_layer(seq, colorbot.lower_cell, batch_size, training)\n", + " seq = rnn_layer(seq, colorbot.upper_cell, batch_size, training)\n", + "\n", + " # Grab just the end-of-sequence from each output.\n", + " indices = tf.stack([length - 1, range(batch_size)], axis=1)\n", + " sequence_ends = tf.gather_nd(seq, indices)\n", + " return colorbot.relu_layer(sequence_ends)\n", + "\n", + "@autograph.convert()\n", + "def loss_fn(labels, predictions):\n", + " return tf.reduce_mean((predictions - labels) ** 2)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "JjK4gXFvFsf4", + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "We will now create the model function for the estimator.\n", + "\n", + "In the model function, we simply call the converted functions that we defined above - that's it!" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "-yso_Nx23Gy1", + "slideshow": { + "slide_type": "-" + } + }, + "outputs": [], + "source": [ + "def model_fn(features, labels, mode, params):\n", + " \"\"\"Estimator model function.\"\"\"\n", + " chars = features['chars']\n", + " sequence_length = features['sequence_length']\n", + " inputs = (chars, sequence_length)\n", + "\n", + " # Create the model components.\n", + " # Simply calling the AutoGraph-ed functions and objects just works!\n", + " colorbot = RnnColorbot()\n", + " \n", + " batch_size = params['batch_size']\n", + "\n", + " if mode == tf.estimator.ModeKeys.TRAIN:\n", + " predictions = model(inputs, colorbot, batch_size, training=True)\n", + " loss = loss_fn(labels, predictions)\n", + "\n", + " learning_rate = params['learning_rate']\n", + " optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", + " global_step = tf.train.get_global_step()\n", + " train_op = optimizer.minimize(loss, global_step=global_step)\n", + " return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)\n", + "\n", + " elif mode == tf.estimator.ModeKeys.EVAL:\n", + " predictions = model(inputs, colorbot, batch_size, training=False)\n", + " loss = loss_fn(labels, predictions)\n", + "\n", + " return tf.estimator.EstimatorSpec(mode, loss=loss)\n", + " \n", + " elif mode == tf.estimator.ModeKeys.PREDICT:\n", + " # For prediction, we expect single tensors.\n", + " predictions = model(inputs, colorbot, 1, training=False)\n", + "\n", + " predictions = tf.minimum(predictions, 1.0)\n", + " return tf.estimator.EstimatorSpec(mode, predictions=predictions)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "HOQfoBnHC9CP", + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "We'll create an input function that will feed our training and eval data." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "FJZlx7yG2MP0", + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [], + "source": [ + "def input_fn(data_dir, data_url, params, training=True):\n", + " \"\"\"An input function for training\"\"\"\n", + " batch_size = params['batch_size']\n", + " \n", + " # load_dataset defined above\n", + " dataset = load_dataset(data_dir, data_url, batch_size, training=training)\n", + "\n", + " # Package the pipeline end in a format suitable for the estimator.\n", + " labels, chars, sequence_length = dataset.make_one_shot_iterator().get_next()\n", + " features = {\n", + " 'chars': chars,\n", + " 'sequence_length': sequence_length\n", + " }\n", + "\n", + " return features, labels" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "qsvv-lzbDqXd", + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "We now have everything in place to build our custom estimator and use it for training and eval!" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 35 + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 10064, + "status": "ok", + "timestamp": 1523580419240, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "2pg1AfbxBJQq", + "outputId": "41894b16-3d3a-4e30-f6e4-5a9c837a2210", + "slideshow": { + "slide_type": "-" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss at step 100: 0.0665446\n" + ] + } + ], + "source": [ + "params = {\n", + " 'batch_size': 64,\n", + " 'learning_rate': 0.01,\n", + "}\n", + "\n", + "train_url = \"https://raw.githubusercontent.com/random-forests/tensorflow-workshop/master/extras/colorbot/data/train.csv\"\n", + "test_url = \"https://raw.githubusercontent.com/random-forests/tensorflow-workshop/master/extras/colorbot/data/test.csv\"\n", + "data_dir = \"tmp/rnn/data\"\n", + "\n", + "regressor = tf.estimator.Estimator(\n", + " model_fn=model_fn,\n", + " params=params)\n", + "\n", + "regressor.train(\n", + " input_fn=lambda: input_fn(data_dir, train_url, params),\n", + " steps=100)\n", + "eval_results = regressor.evaluate(\n", + " input_fn=lambda: input_fn(data_dir, test_url, params, training=False),\n", + " steps=2\n", + ")\n", + "\n", + "print('Eval loss at step %d: %s' % (eval_results['global_step'], eval_results['loss']))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "zG1YAjB_cUnQ", + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "And here's the same estimator used for inference." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 343 + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 31286, + "status": "ok", + "timestamp": 1523580450579, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "dxHex2tUN_10", + "outputId": "b3dc558d-b800-4e9b-e60e-3441124e80d8", + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "\u003clink rel=stylesheet type=text/css href='/nbextensions/google.colab/tabbar.css'\u003e\u003c/link\u003e" + ], + "text/plain": [ + "\u003cIPython.core.display.HTML at 0x7f4112527e90\u003e" + ] + }, + "metadata": { + "tags": [ + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\u003cscript src='/nbextensions/google.colab/tabbar_main.min.js'\u003e\u003c/script\u003e" + ], + "text/plain": [ + "\u003cIPython.core.display.HTML at 0x7f4112527f10\u003e" + ] + }, + "metadata": { + "tags": [ + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\u003cdiv id=\"id1\"\u003e\u003c/div\u003e" + ], + "text/plain": [ + "\u003cIPython.core.display.HTML at 0x7f4112527f50\u003e" + ] + }, + "metadata": { + "tags": [ + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"2c60f474-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = colab_lib.createTabBar({\"initialSelection\": 0, \"location\": \"top\", \"contentHeight\": [\"initial\"], \"borderColor\": [\"#a7a7a7\"], \"contentBorder\": [\"0px\"], \"tabNames\": [\"RNN Colorbot\"], \"elementId\": \"id1\"});\n", + "//# sourceURL=js_a0db480422" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f410f8fd1d0\u003e" + ] + }, + "metadata": { + "tags": [ + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"2c60f475-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", + "//# sourceURL=js_d2a46ea291" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f410f8fd0d0\u003e" + ] + }, + "metadata": { + "tags": [ + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"2c60f476-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", + "//# sourceURL=js_0a8262c6e9" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f410f8fd390\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"2c60f477-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = document.querySelector(\"#id1_content_0\");\n", + "//# sourceURL=js_e32f85ccd2" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f410f8fd490\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"2c60f478-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"2c60f477-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_eaee748b21" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f410f8fd550\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"2c60f479-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", + "//# sourceURL=js_2befe06587" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f4112527f10\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"354d7b1a-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"2c60f476-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_8ec4aeeb25" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f410f8fd690\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"354d7b1b-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", + "//# sourceURL=js_9f9f4574f1" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f410f8fd350\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"354d7b1c-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = document.querySelector(\"#id1_content_0\");\n", + "//# sourceURL=js_bcccd8f300" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f410f8fd6d0\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"354d7b1d-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"354d7b1c-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_2c056cee72" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f410f8fd490\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"354d7b1e-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", + "//# sourceURL=js_c853c3f58b" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f410f8fd610\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"354d7b1f-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"354d7b1b-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_e5730ab00d" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f41127a2050\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"354d7b20-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", + "//# sourceURL=js_a897ef7e24" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f41127a2250\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"354d7b21-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = document.querySelector(\"#id1_content_0\");\n", + "//# sourceURL=js_565fa3d154" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f4113124d90\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"354d7b22-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"354d7b21-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_222e0dc6af" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f4113124c10\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"354d7b23-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", + "//# sourceURL=js_831db7458f" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f4113124310\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3803fab4-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"354d7b20-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_adb576c6eb" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f410f990850\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3803fab5-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", + "//# sourceURL=js_9418f2d32f" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f410f990850\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3803fab6-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = document.querySelector(\"#id1_content_0\");\n", + "//# sourceURL=js_3fad25f306" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f4112527ed0\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3803fab7-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"3803fab6-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_45b9340e7b" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f410f990c90\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3803fab8-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", + "//# sourceURL=js_bec9896d44" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f410f990a10\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3803fab9-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"3803fab5-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_460b91ad4a" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f41b21d3a10\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3803faba-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", + "//# sourceURL=js_7dedd0b037" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f41b21d3890\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3803fabb-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = document.querySelector(\"#id1_content_0\");\n", + "//# sourceURL=js_4b1c977dc7" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f41b21d3bd0\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3803fabc-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"3803fabb-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_d64fedfcf9" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f41b21d3410\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3803fabd-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", + "//# sourceURL=js_3e8c929c3f" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f41b21d3c50\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3b9b986c-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"3803faba-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_9f9cf2b76f" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f410f8fd590\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3b9b986d-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", + "//# sourceURL=js_b402e6b587" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f41b21d3d90\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3b9b986e-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = document.querySelector(\"#id1_content_0\");\n", + "//# sourceURL=js_9b7d66db72" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f41b21d3b10\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3b9b986f-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"3b9b986e-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_11ec213a3f" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f41b21d3950\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3b9b9870-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", + "//# sourceURL=js_9c055e4bc0" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f41b21d3850\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQwAAAENCAYAAAD60Fs2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAACMRJREFUeJzt3F+IlfW+x/Gvp3FECyIqU4PCO7EgZnQtnUJ0JJGoTDoY\n/dGrMBJhosggIgK7KwwiMdxRF11F/0AJvIisLBqcguxCjEAkmNQGcRvVwIzm71zsc4Yje7P3x9h7\nz97u1+tqrYdnPeu7nos3v2f9m9FaawUQ+K/pHgD49yEYQEwwgJhgADHBAGKCAcQEg2nx9NNPV7fb\nrfvuu69GRkZq5cqV0z0SAcG4xK1evbqGh4ene4wLfPXVVzU8PFyfffZZvf3221VVNWPGjGmeioRg\n8E/122+/1Q8//FDXX399zZo1a7rH4SIJxiXsqaeeqhMnTtSWLVuqv7+/Xn/99frmm2/q/vvvr06n\nU+vXr6+RkZGp/Tdt2lQvv/xyPfDAA9Xf318PP/xwnTlzpqqqJicna9u2bbVs2bLqdDq1YcOGOn36\ndFVVjY2N1ZYtW2rZsmW1du3aeuedd6aOuXPnzhoaGqpt27bV0qVL67333qtnn322Dh06VP39/bVz\n584/m/vo0aO1adOm6nQ6dffdd9f+/furqmp0dLQ6nc7Ufs8880zdeuutU/e3bdtWb7755t/3JHKh\nxiVtcHCwDQ8Pt9ZaO3nyZOt2u+3AgQOttda++OKL1u122+nTp1trrW3cuLGtWbOmff/9921iYqJt\n3Lix7dixo7XW2ltvvdUeffTRNjEx0c6fP98OHz7cfvnll9Zaaw899FDbvn17m5ycbEeOHGnLly+f\nes5XXnml3XTTTe2jjz5qrbU2MTHR3n///fbggw9OzXjw4MG2cuXK1lprZ8+ebWvWrGm7d+9uZ8+e\nbcPDw62vr68dO3Zs6vUcPny4tdba2rVr2+23396OHj3aWmtt1apV7ciRI/+oU0lrzQrjP0D7358L\n7d27t1atWlUrVqyoqqqBgYG6+eab69NPP53a9957760bbrihent764477qgjR45UVVVPT0+dOXOm\njh07VjNmzKjFixfX5ZdfXidPnqyvv/66nnzyyZo5c2YtWrSoNmzYUHv27Jk6Zl9fX61evbqqqnp7\ne//qrIcOHarx8fF65JFHqqenp5YvX16Dg4P1wQcfVFXV0qVLa2RkpE6dOlVVVWvXrq0vv/yyRkdH\n69dff61Fixb9nc4af0nPdA/AP8/x48dr37599fHHH1fVn0Jy7ty5GhgYmNrnmmuumbo9e/bsGh8f\nr6qqe+65p06ePFlPPPFE/fzzz7Vu3bp6/PHHa2xsrK688sqaPXv21OMWLFhQhw8fnro/b968eMax\nsbGaP3/+BdsWLFhQY2NjVVXV6XRq//79dd1111W3261ut1t79uyp3t7eWrJkyUWcDX4PwbjE/f9P\nH+bPn1/r16+v7du3X/Rxenp6auvWrbV169Y6fvx4bd68uRYuXFi33XZb/fTTTzU+Pl5z5sypqqoT\nJ07U3Llz/+IMf8vcuXPrxIkTF2w7fvx4LVy4sKqqut1uvfjiizV//vzqdDrV399fzz33XPX29la3\n273o18XFcUlyibv22mtrdHS0qqrWrVtX+/fvr88//7zOnz9fExMTNTIyUj/++OPfPM7Bgwfru+++\nq/Pnz9ecOXOqp6enLrvsspo3b1719fXVSy+9VJOTk/Xtt9/Wu+++W+vWrftd895yyy01Z86ceu21\n1+rcuXN18ODB+uSTT+rOO++sqqobb7yxZs2aVXv37q1Op1NXXHFFXX311fXhhx9e8IYo/xiCcYnb\nvHlz7dq1q7rdbu3bt6927dpVu3fvroGBgRocHKw33nhj6j2Ov7YSOHXqVA0NDdWSJUvqrrvuqmXL\nlk1FYceOHTU6OlorVqyooaGheuyxxy64zLkYM2fOrFdffbUOHDhQy5cvr+eff75eeOGFqRVG1Z9W\nGVddddXUpc7/hWLx4sW/6znJzWjNH+gAGSsMICYYQEwwgJhgALF/2e9h/PEP/z3dI8B/tKseee/P\ntllhADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEA\nYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOI\nCQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAm\nGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhg\nADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIB\nxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOICQYQ\nEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBM\nMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHB\nAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQD\niAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwg\nJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICY\nYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKC\nAcQEA4gJBhATDCA2o7XWpnsI4N+DFQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEww\ngJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHE/gfh60wGjfc7LQAAAABJRU5ErkJg\ngg==\n", + "text/plain": [ + "\u003cmatplotlib.figure.Figure at 0x7f4113124310\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1", + "user_output" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3b9b9871-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"3b9b986d-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_ba6a061307" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f410f8fd890\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3b9b9872-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", + "//# sourceURL=js_83e3496927" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f410f8fd590\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3b9b9873-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = document.querySelector(\"#id1_content_0\");\n", + "//# sourceURL=js_f437bab20d" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f41127a22d0\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3b9b9874-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"3b9b9873-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_93aa63450e" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f41127a2b90\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3b9b9875-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", + "//# sourceURL=js_aca189bea5" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f410f8fd4d0\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\u003cdiv class=id_100313201 style=\"margin-right:10px; display:flex;align-items:center;\"\u003e\u003cspan style=\"margin-right: 3px;\"\u003e\u003c/span\u003e\u003c/div\u003e" + ], + "text/plain": [ + "\u003cIPython.core.display.HTML at 0x7f410f990a90\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1", + "user_output" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3b9b9876-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = jQuery(\".id_100313201 span\");\n", + "//# sourceURL=js_5df1fe383e" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f410f8fd490\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1", + "user_output" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3b9b9877-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = window[\"3b9b9876-3eb4-11e8-91ec-c8d3ffb5fbe0\"].text(\"Give me a color name (or press 'enter' to exit): \");\n", + "//# sourceURL=js_c62c7174ad" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f41127a2390\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1", + "user_output" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3ed76584-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = jQuery(\".id_100313201 input\");\n", + "//# sourceURL=js_2e2201ddc4" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f41127a2810\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1", + "user_output" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3ed76585-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = window[\"3ed76584-3eb4-11e8-91ec-c8d3ffb5fbe0\"].remove();\n", + "//# sourceURL=js_288e5283d6" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f41127a26d0\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1", + "user_output" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3ed76586-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = jQuery(\".id_100313201 span\");\n", + "//# sourceURL=js_2f31d19cde" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f41127a2fd0\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1", + "user_output" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3ed76587-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = window[\"3ed76586-3eb4-11e8-91ec-c8d3ffb5fbe0\"].text(\"Give me a color name (or press 'enter' to exit): \");\n", + "//# sourceURL=js_2fbbcda050" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f4112527e90\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1", + "user_output" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"3ed76588-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"3b9b9872-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_f94d975cf3" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f41127a2fd0\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1" + ] + }, + "output_type": "display_data" + } + ], + "source": [ + "def predict_input_fn(color_name):\n", + " \"\"\"An input function for prediction.\"\"\"\n", + " _, chars, sequence_length = parse(color_name)\n", + " \n", + " # We create a batch of a single element.\n", + " features = {\n", + " 'chars': tf.expand_dims(chars, 0),\n", + " 'sequence_length': tf.expand_dims(sequence_length, 0)\n", + " }\n", + " return features, None\n", + "\n", + "\n", + "def draw_prediction(color_name, pred):\n", + " pred = pred * 255\n", + " pred = pred.astype(np.uint8)\n", + " plt.axis('off')\n", + " plt.imshow(pred)\n", + " plt.title(color_name)\n", + " plt.show()\n", + "\n", + "\n", + "def predict_with_estimator(color_name, regressor):\n", + " predictions = regressor.predict(\n", + " input_fn=lambda:predict_input_fn(color_name))\n", + " pred = next(predictions)\n", + " predictions.close()\n", + " pred = np.minimum(pred, 1.0)\n", + " pred = np.expand_dims(np.expand_dims(pred, 0), 0)\n", + "\n", + " draw_prediction(color_name, pred)\n", + "\n", + "tb = widgets.TabBar([\"RNN Colorbot\"])\n", + "while True:\n", + " with tb.output_to(0):\n", + " try:\n", + " color_name = six.moves.input(\"Give me a color name (or press 'enter' to exit): \")\n", + " except (EOFError, KeyboardInterrupt):\n", + " break\n", + " if not color_name:\n", + " break\n", + " with tb.output_to(0):\n", + " tb.clear_tab()\n", + " predict_with_estimator(color_name, regressor)\n", + " " + ] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [], + "default_view": {}, + "name": "RNN Colorbot using Estimators", + "provenance": [ + { + "file_id": "1CtzefX39ffFibX_BqE6cRbT0UW_DdVKl", + "timestamp": 1523579810961 + }, + { + "file_id": "1DcfimonWU11tmyivKBGVrbpAl3BIOaRG", + "timestamp": 1523016192637 + }, + { + "file_id": "1wCZUh73zTNs1jzzYjqoxMIdaBWCdKJ2K", + "timestamp": 1522238054357 + }, + { + "file_id": "1_HpC-RrmIv4lNaqeoslUeWaX8zH5IXaJ", + "timestamp": 1521743157199 + }, + { + "file_id": "1mjO2fQ2F9hxpAzw2mnrrUkcgfb7xSGW-", + "timestamp": 1520522344607 + } + ], + "version": "0.3.2", + "views": {} + }, + "kernelspec": { + "display_name": "Python 2", + "name": "python2" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} -- GitLab From 3eb4e4f82c3d91586b2510d3fb769d6683a4c5f3 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 07:55:46 -0700 Subject: [PATCH 315/791] Split byte_order.h off cpu_info.h PiperOrigin-RevId: 192768744 --- tensorflow/compiler/aot/test.cc | 1 + tensorflow/compiler/xla/service/backend.cc | 1 + tensorflow/compiler/xla/shape_util.h | 1 + .../xla/tests/local_client_test_base.cc | 2 +- .../factorization/kernels/clustering_ops.cc | 1 + .../contrib/ffmpeg/default/ffmpeg_lib.cc | 2 +- tensorflow/core/BUILD | 6 ++- .../core/common_runtime/direct_session.cc | 2 +- .../kernel_benchmark_testlib.cc | 1 + .../core/common_runtime/local_device.cc | 1 + .../core/common_runtime/process_util.cc | 1 + tensorflow/core/framework/bfloat16.h | 1 + tensorflow/core/grappler/clusters/utils.cc | 1 + tensorflow/core/grappler/costs/utils.cc | 2 +- tensorflow/core/grappler/devices.cc | 1 + .../grappler/optimizers/constant_folding.cc | 1 + .../adaptive_shared_batch_scheduler.h | 1 + .../batching_util/shared_batch_scheduler.h | 1 + tensorflow/core/kernels/cast_op.h | 2 +- tensorflow/core/kernels/decode_raw_op.cc | 2 +- .../core/kernels/mkl_input_conversion_op.cc | 1 + tensorflow/core/kernels/mkl_tfconv_op.h | 1 + tensorflow/core/kernels/sparse_matmul_op.h | 1 + tensorflow/core/lib/bfloat16/bfloat16.h | 3 +- tensorflow/core/lib/core/coding.cc | 2 +- tensorflow/core/lib/core/raw_coding.h | 2 +- tensorflow/core/lib/gtl/inlined_vector.h | 2 +- tensorflow/core/lib/png/png_io.cc | 2 +- tensorflow/core/lib/wav/wav_io.cc | 2 +- tensorflow/core/platform/byte_order.h | 37 +++++++++++++++++++ tensorflow/core/platform/cpu_feature_guard.cc | 1 + tensorflow/core/platform/cpu_info.h | 3 -- tensorflow/core/platform/denormal.cc | 3 +- tensorflow/core/platform/windows/cpu_info.h | 9 ----- 34 files changed, 72 insertions(+), 28 deletions(-) create mode 100644 tensorflow/core/platform/byte_order.h diff --git a/tensorflow/compiler/aot/test.cc b/tensorflow/compiler/aot/test.cc index 47ef5f82cb..6b098049cb 100644 --- a/tensorflow/compiler/aot/test.cc +++ b/tensorflow/compiler/aot/test.cc @@ -35,6 +35,7 @@ limitations under the License. // clang-format on #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/test.h" #include "tensorflow/core/platform/test_benchmark.h" diff --git a/tensorflow/compiler/xla/service/backend.cc b/tensorflow/compiler/xla/service/backend.cc index 05f2d06278..0b36b67251 100644 --- a/tensorflow/compiler/xla/service/backend.cc +++ b/tensorflow/compiler/xla/service/backend.cc @@ -31,6 +31,7 @@ limitations under the License. #include "tensorflow/core/common_runtime/eigen_thread_pool.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/threadpool.h" +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/logging.h" diff --git a/tensorflow/compiler/xla/shape_util.h b/tensorflow/compiler/xla/shape_util.h index 6d228eff46..f2790ba293 100644 --- a/tensorflow/compiler/xla/shape_util.h +++ b/tensorflow/compiler/xla/shape_util.h @@ -31,6 +31,7 @@ limitations under the License. #include "tensorflow/core/lib/core/threadpool.h" #include "tensorflow/core/lib/gtl/array_slice.h" #include "tensorflow/core/lib/gtl/optional.h" +#include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/types.h" diff --git a/tensorflow/compiler/xla/tests/local_client_test_base.cc b/tensorflow/compiler/xla/tests/local_client_test_base.cc index 96b976d25d..12979a0473 100644 --- a/tensorflow/compiler/xla/tests/local_client_test_base.cc +++ b/tensorflow/compiler/xla/tests/local_client_test_base.cc @@ -27,7 +27,7 @@ limitations under the License. #include "tensorflow/compiler/xla/test_helpers.h" #include "tensorflow/core/common_runtime/eigen_thread_pool.h" #include "tensorflow/core/lib/core/threadpool.h" -#include "tensorflow/core/platform/cpu_info.h" +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/logging.h" diff --git a/tensorflow/contrib/factorization/kernels/clustering_ops.cc b/tensorflow/contrib/factorization/kernels/clustering_ops.cc index 2a6c97e8b9..025534d540 100644 --- a/tensorflow/contrib/factorization/kernels/clustering_ops.cc +++ b/tensorflow/contrib/factorization/kernels/clustering_ops.cc @@ -32,6 +32,7 @@ #include "tensorflow/core/lib/gtl/top_n.h" #include "tensorflow/core/lib/random/philox_random.h" #include "tensorflow/core/lib/random/simple_philox.h" +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/logging.h" diff --git a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc index 35341406a0..cca1a05419 100644 --- a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc +++ b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc @@ -28,7 +28,7 @@ #include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/lib/strings/str_util.h" -#include "tensorflow/core/platform/cpu_info.h" +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/env.h" using tensorflow::strings::StrCat; diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index c461f9ed2f..01fe61eeac 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -282,7 +282,7 @@ PLATFORM_BASE_HDRS = [ "platform/logging.h", "platform/macros.h", "platform/types.h", - "platform/cpu_info.h", + "platform/byte_order.h", ] PLATFORM_OTHER_HDRS = [ @@ -290,6 +290,7 @@ PLATFORM_OTHER_HDRS = [ "platform/stacktrace.h", "platform/stacktrace_handler.h", "platform/context.h", + "platform/cpu_info.h", "platform/cpu_feature_guard.h", "platform/dynamic_annotations.h", "platform/env.h", @@ -318,7 +319,6 @@ cc_library( srcs = glob([ "platform/*/integral_types.h", "platform/*/logging.h", - "platform/*/cpu_info.h", ]), hdrs = PLATFORM_BASE_HDRS, deps = [ @@ -666,6 +666,7 @@ cc_library( "framework/tensor_types.h", "framework/type_traits.h", "lib/bfloat16/bfloat16.h", + "platform/byte_order.h", "platform/default/dynamic_annotations.h", "platform/default/integral_types.h", "platform/default/logging.h", @@ -1906,6 +1907,7 @@ cc_library( "lib/core/casts.h", "lib/core/stringpiece.h", "lib/png/png_io.h", + "platform/byte_order.h", "platform/cpu_info.h", "platform/default/integral_types.h", "platform/default/logging.h", diff --git a/tensorflow/core/common_runtime/direct_session.cc b/tensorflow/core/common_runtime/direct_session.cc index 0479061daf..0afbd02e86 100644 --- a/tensorflow/core/common_runtime/direct_session.cc +++ b/tensorflow/core/common_runtime/direct_session.cc @@ -54,7 +54,7 @@ limitations under the License. #include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/platform/cpu_info.h" +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/device_tracer.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/mutex.h" diff --git a/tensorflow/core/common_runtime/kernel_benchmark_testlib.cc b/tensorflow/core/common_runtime/kernel_benchmark_testlib.cc index 64d8849475..7de1b80e2d 100644 --- a/tensorflow/core/common_runtime/kernel_benchmark_testlib.cc +++ b/tensorflow/core/common_runtime/kernel_benchmark_testlib.cc @@ -28,6 +28,7 @@ limitations under the License. #include "tensorflow/core/lib/core/notification.h" #include "tensorflow/core/lib/core/threadpool.h" #include "tensorflow/core/lib/strings/str_util.h" +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/test_benchmark.h" diff --git a/tensorflow/core/common_runtime/local_device.cc b/tensorflow/core/common_runtime/local_device.cc index ca7f1614f1..873182371e 100644 --- a/tensorflow/core/common_runtime/local_device.cc +++ b/tensorflow/core/common_runtime/local_device.cc @@ -19,6 +19,7 @@ limitations under the License. #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/common_runtime/eigen_thread_pool.h" #include "tensorflow/core/lib/core/threadpool.h" +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_feature_guard.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/logging.h" diff --git a/tensorflow/core/common_runtime/process_util.cc b/tensorflow/core/common_runtime/process_util.cc index d5bd7f8b98..cf8e11c9c8 100644 --- a/tensorflow/core/common_runtime/process_util.cc +++ b/tensorflow/core/common_runtime/process_util.cc @@ -18,6 +18,7 @@ limitations under the License. #include #include "tensorflow/core/lib/core/threadpool.h" +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/tracing.h" diff --git a/tensorflow/core/framework/bfloat16.h b/tensorflow/core/framework/bfloat16.h index 968c18bdd2..2f79d0fa70 100644 --- a/tensorflow/core/framework/bfloat16.h +++ b/tensorflow/core/framework/bfloat16.h @@ -17,6 +17,7 @@ limitations under the License. #define TENSORFLOW_FRAMEWORK_BFLOAT16_H_ #include "tensorflow/core/framework/numeric_types.h" +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/types.h" #if defined(PLATFORM_WINDOWS) diff --git a/tensorflow/core/grappler/clusters/utils.cc b/tensorflow/core/grappler/clusters/utils.cc index 50d6e6468f..a7519725a5 100644 --- a/tensorflow/core/grappler/clusters/utils.cc +++ b/tensorflow/core/grappler/clusters/utils.cc @@ -32,6 +32,7 @@ limitations under the License. #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/mem.h" diff --git a/tensorflow/core/grappler/costs/utils.cc b/tensorflow/core/grappler/costs/utils.cc index f318e3911c..be54d98534 100644 --- a/tensorflow/core/grappler/costs/utils.cc +++ b/tensorflow/core/grappler/costs/utils.cc @@ -44,7 +44,7 @@ limitations under the License. #include "tensorflow/core/lib/core/bits.h" #include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/platform/cpu_info.h" +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/protobuf.h" diff --git a/tensorflow/core/grappler/devices.cc b/tensorflow/core/grappler/devices.cc index b318ac22d4..2be894a08b 100644 --- a/tensorflow/core/grappler/devices.cc +++ b/tensorflow/core/grappler/devices.cc @@ -16,6 +16,7 @@ limitations under the License. #include #include "tensorflow/core/grappler/devices.h" +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #if GOOGLE_CUDA diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc index e29aaa25fe..45bb188e8d 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding.cc +++ b/tensorflow/core/grappler/optimizers/constant_folding.cc @@ -36,6 +36,7 @@ limitations under the License. #include "tensorflow/core/lib/gtl/inlined_vector.h" #include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/denormal.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/setround.h" diff --git a/tensorflow/core/kernels/batching_util/adaptive_shared_batch_scheduler.h b/tensorflow/core/kernels/batching_util/adaptive_shared_batch_scheduler.h index 339d792302..f5ced95feb 100644 --- a/tensorflow/core/kernels/batching_util/adaptive_shared_batch_scheduler.h +++ b/tensorflow/core/kernels/batching_util/adaptive_shared_batch_scheduler.h @@ -28,6 +28,7 @@ limitations under the License. #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/threadpool.h" +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/thread_annotations.h" diff --git a/tensorflow/core/kernels/batching_util/shared_batch_scheduler.h b/tensorflow/core/kernels/batching_util/shared_batch_scheduler.h index b77289aded..edc88a0384 100644 --- a/tensorflow/core/kernels/batching_util/shared_batch_scheduler.h +++ b/tensorflow/core/kernels/batching_util/shared_batch_scheduler.h @@ -30,6 +30,7 @@ limitations under the License. #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/thread_annotations.h" diff --git a/tensorflow/core/kernels/cast_op.h b/tensorflow/core/kernels/cast_op.h index fd4e75d26f..16d2e0e0a5 100644 --- a/tensorflow/core/kernels/cast_op.h +++ b/tensorflow/core/kernels/cast_op.h @@ -21,7 +21,7 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/framework/types.h" -#include "tensorflow/core/platform/cpu_info.h" +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/types.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/decode_raw_op.cc b/tensorflow/core/kernels/decode_raw_op.cc index bacacb94ae..eaef5a6097 100644 --- a/tensorflow/core/kernels/decode_raw_op.cc +++ b/tensorflow/core/kernels/decode_raw_op.cc @@ -21,7 +21,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/platform/cpu_info.h" +#include "tensorflow/core/platform/byte_order.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/mkl_input_conversion_op.cc b/tensorflow/core/kernels/mkl_input_conversion_op.cc index 68d3e1c9ab..3245625a32 100644 --- a/tensorflow/core/kernels/mkl_input_conversion_op.cc +++ b/tensorflow/core/kernels/mkl_input_conversion_op.cc @@ -24,6 +24,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/kernels/ops_util.h" +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/util/tensor_format.h" diff --git a/tensorflow/core/kernels/mkl_tfconv_op.h b/tensorflow/core/kernels/mkl_tfconv_op.h index ddea9e281b..4120f013ac 100644 --- a/tensorflow/core/kernels/mkl_tfconv_op.h +++ b/tensorflow/core/kernels/mkl_tfconv_op.h @@ -27,6 +27,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/kernels/ops_util.h" +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/util/tensor_format.h" diff --git a/tensorflow/core/kernels/sparse_matmul_op.h b/tensorflow/core/kernels/sparse_matmul_op.h index 14ef2ed704..e89280724e 100644 --- a/tensorflow/core/kernels/sparse_matmul_op.h +++ b/tensorflow/core/kernels/sparse_matmul_op.h @@ -17,6 +17,7 @@ limitations under the License. #define TENSORFLOW_KERNELS_SPARSE_MATMUL_OP_H_ #include "third_party/eigen3/Eigen/Core" +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/types.h" #if defined(PLATFORM_WINDOWS) diff --git a/tensorflow/core/lib/bfloat16/bfloat16.h b/tensorflow/core/lib/bfloat16/bfloat16.h index 126e5a17af..e7c24387a4 100644 --- a/tensorflow/core/lib/bfloat16/bfloat16.h +++ b/tensorflow/core/lib/bfloat16/bfloat16.h @@ -19,8 +19,7 @@ limitations under the License. #include #include -// We need cpu_info.h here in order to pick up __BYTE_ORDER__. -#include "tensorflow/core/platform/cpu_info.h" +#include "tensorflow/core/platform/byte_order.h" #ifdef __CUDACC__ // All functions callable from CUDA code must be qualified with __device__ diff --git a/tensorflow/core/lib/core/coding.cc b/tensorflow/core/lib/core/coding.cc index bb95c27410..50872eef83 100644 --- a/tensorflow/core/lib/core/coding.cc +++ b/tensorflow/core/lib/core/coding.cc @@ -15,7 +15,7 @@ limitations under the License. #include "tensorflow/core/lib/core/coding.h" -#include "tensorflow/core/platform/cpu_info.h" +#include "tensorflow/core/platform/byte_order.h" namespace tensorflow { namespace core { diff --git a/tensorflow/core/lib/core/raw_coding.h b/tensorflow/core/lib/core/raw_coding.h index bbfd33d303..37201b755d 100644 --- a/tensorflow/core/lib/core/raw_coding.h +++ b/tensorflow/core/lib/core/raw_coding.h @@ -17,7 +17,7 @@ limitations under the License. #define TENSORFLOW_LIB_CORE_RAW_CODING_H_ #include -#include "tensorflow/core/platform/cpu_info.h" +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/types.h" namespace tensorflow { diff --git a/tensorflow/core/lib/gtl/inlined_vector.h b/tensorflow/core/lib/gtl/inlined_vector.h index 6e3cb2206d..2011f7d4a1 100644 --- a/tensorflow/core/lib/gtl/inlined_vector.h +++ b/tensorflow/core/lib/gtl/inlined_vector.h @@ -43,7 +43,7 @@ limitations under the License. #include #include "tensorflow/core/lib/gtl/manual_constructor.h" -#include "tensorflow/core/platform/cpu_info.h" +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/mem.h" #include "tensorflow/core/platform/types.h" diff --git a/tensorflow/core/lib/png/png_io.cc b/tensorflow/core/lib/png/png_io.cc index cba473927d..62c803afb2 100644 --- a/tensorflow/core/lib/png/png_io.cc +++ b/tensorflow/core/lib/png/png_io.cc @@ -26,7 +26,7 @@ limitations under the License. #include "tensorflow/core/lib/core/casts.h" #include "tensorflow/core/lib/png/png_io.h" -#include "tensorflow/core/platform/cpu_info.h" // endian +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/png.h" diff --git a/tensorflow/core/lib/wav/wav_io.cc b/tensorflow/core/lib/wav/wav_io.cc index 51b9c6cd82..3f7dbcee85 100644 --- a/tensorflow/core/lib/wav/wav_io.cc +++ b/tensorflow/core/lib/wav/wav_io.cc @@ -23,7 +23,7 @@ limitations under the License. #include "tensorflow/core/lib/core/coding.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/wav/wav_io.h" -#include "tensorflow/core/platform/cpu_info.h" +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" diff --git a/tensorflow/core/platform/byte_order.h b/tensorflow/core/platform/byte_order.h new file mode 100644 index 0000000000..aab6535e4b --- /dev/null +++ b/tensorflow/core/platform/byte_order.h @@ -0,0 +1,37 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_CORE_PLATFORM_BYTE_ORDER_H_ +#define TENSORFLOW_CORE_PLATFORM_BYTE_ORDER_H_ + +// Byte order defines provided by gcc. MSVC doesn't define those so +// we define them here. +// We assume that all windows platform out there are little endian. +#if defined(_MSC_VER) && !defined(__clang__) +#define __ORDER_LITTLE_ENDIAN__ 0x4d2 +#define __ORDER_BIG_ENDIAN__ 0x10e1 +#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ +#endif + +namespace tensorflow { +namespace port { + +// TODO(jeff,sanjay): Make portable +constexpr bool kLittleEndian = __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__; + +} // namespace port +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_PLATFORM_BYTE_ORDER_H_ diff --git a/tensorflow/core/platform/cpu_feature_guard.cc b/tensorflow/core/platform/cpu_feature_guard.cc index b570658158..9d00aa7b7f 100644 --- a/tensorflow/core/platform/cpu_feature_guard.cc +++ b/tensorflow/core/platform/cpu_feature_guard.cc @@ -18,6 +18,7 @@ limitations under the License. #include #include +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/logging.h" diff --git a/tensorflow/core/platform/cpu_info.h b/tensorflow/core/platform/cpu_info.h index bb77650e26..c42429a394 100644 --- a/tensorflow/core/platform/cpu_info.h +++ b/tensorflow/core/platform/cpu_info.h @@ -25,9 +25,6 @@ limitations under the License. namespace tensorflow { namespace port { -// TODO(jeff,sanjay): Make portable -constexpr bool kLittleEndian = __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__; - // Returns an estimate of the number of schedulable CPUs for this // process. Usually, it's constant throughout the lifetime of a // process, but it might change if the underlying cluster management diff --git a/tensorflow/core/platform/denormal.cc b/tensorflow/core/platform/denormal.cc index 82cbc43b4f..c510dc204f 100644 --- a/tensorflow/core/platform/denormal.cc +++ b/tensorflow/core/platform/denormal.cc @@ -15,8 +15,9 @@ limitations under the License. #include -#include "tensorflow/core/platform/denormal.h" +#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" +#include "tensorflow/core/platform/denormal.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/platform.h" // If we're on gcc 4.8 or older, there's a known bug that prevents the use of diff --git a/tensorflow/core/platform/windows/cpu_info.h b/tensorflow/core/platform/windows/cpu_info.h index f20939d3c0..ba2126abcf 100644 --- a/tensorflow/core/platform/windows/cpu_info.h +++ b/tensorflow/core/platform/windows/cpu_info.h @@ -19,13 +19,4 @@ limitations under the License. // included so __cpuidex function is available for GETCPUID on Windows #include -// Byte order defines provided by gcc. MSVC doesn't define those so -// we define them here. -// We assume that all windows platform out there are little endian. -#if defined(_MSC_VER) && !defined(__clang__) -#define __ORDER_LITTLE_ENDIAN__ 0x4d2 -#define __ORDER_BIG_ENDIAN__ 0x10e1 -#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ -#endif - #endif // TENSORFLOW_PLATFORM_WINDOWS_CPU_INFO_H_ -- GitLab From f9de043501e401af73aa02ab950864534f07c1df Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 08:10:57 -0700 Subject: [PATCH 316/791] Automated g4 rollback of changelist 192768744 PiperOrigin-RevId: 192770717 --- tensorflow/compiler/aot/test.cc | 1 - tensorflow/compiler/xla/service/backend.cc | 1 - tensorflow/compiler/xla/shape_util.h | 1 - .../xla/tests/local_client_test_base.cc | 2 +- .../factorization/kernels/clustering_ops.cc | 1 - .../contrib/ffmpeg/default/ffmpeg_lib.cc | 2 +- tensorflow/core/BUILD | 6 +-- .../core/common_runtime/direct_session.cc | 2 +- .../kernel_benchmark_testlib.cc | 1 - .../core/common_runtime/local_device.cc | 1 - .../core/common_runtime/process_util.cc | 1 - tensorflow/core/framework/bfloat16.h | 1 - tensorflow/core/grappler/clusters/utils.cc | 1 - tensorflow/core/grappler/costs/utils.cc | 2 +- tensorflow/core/grappler/devices.cc | 1 - .../grappler/optimizers/constant_folding.cc | 1 - .../adaptive_shared_batch_scheduler.h | 1 - .../batching_util/shared_batch_scheduler.h | 1 - tensorflow/core/kernels/cast_op.h | 2 +- tensorflow/core/kernels/decode_raw_op.cc | 2 +- .../core/kernels/mkl_input_conversion_op.cc | 1 - tensorflow/core/kernels/mkl_tfconv_op.h | 1 - tensorflow/core/kernels/sparse_matmul_op.h | 1 - tensorflow/core/lib/bfloat16/bfloat16.h | 3 +- tensorflow/core/lib/core/coding.cc | 2 +- tensorflow/core/lib/core/raw_coding.h | 2 +- tensorflow/core/lib/gtl/inlined_vector.h | 2 +- tensorflow/core/lib/png/png_io.cc | 2 +- tensorflow/core/lib/wav/wav_io.cc | 2 +- tensorflow/core/platform/byte_order.h | 37 ------------------- tensorflow/core/platform/cpu_feature_guard.cc | 1 - tensorflow/core/platform/cpu_info.h | 3 ++ tensorflow/core/platform/denormal.cc | 3 +- tensorflow/core/platform/windows/cpu_info.h | 9 +++++ 34 files changed, 28 insertions(+), 72 deletions(-) delete mode 100644 tensorflow/core/platform/byte_order.h diff --git a/tensorflow/compiler/aot/test.cc b/tensorflow/compiler/aot/test.cc index 6b098049cb..47ef5f82cb 100644 --- a/tensorflow/compiler/aot/test.cc +++ b/tensorflow/compiler/aot/test.cc @@ -35,7 +35,6 @@ limitations under the License. // clang-format on #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" -#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/test.h" #include "tensorflow/core/platform/test_benchmark.h" diff --git a/tensorflow/compiler/xla/service/backend.cc b/tensorflow/compiler/xla/service/backend.cc index 0b36b67251..05f2d06278 100644 --- a/tensorflow/compiler/xla/service/backend.cc +++ b/tensorflow/compiler/xla/service/backend.cc @@ -31,7 +31,6 @@ limitations under the License. #include "tensorflow/core/common_runtime/eigen_thread_pool.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/threadpool.h" -#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/logging.h" diff --git a/tensorflow/compiler/xla/shape_util.h b/tensorflow/compiler/xla/shape_util.h index f2790ba293..6d228eff46 100644 --- a/tensorflow/compiler/xla/shape_util.h +++ b/tensorflow/compiler/xla/shape_util.h @@ -31,7 +31,6 @@ limitations under the License. #include "tensorflow/core/lib/core/threadpool.h" #include "tensorflow/core/lib/gtl/array_slice.h" #include "tensorflow/core/lib/gtl/optional.h" -#include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/types.h" diff --git a/tensorflow/compiler/xla/tests/local_client_test_base.cc b/tensorflow/compiler/xla/tests/local_client_test_base.cc index 12979a0473..96b976d25d 100644 --- a/tensorflow/compiler/xla/tests/local_client_test_base.cc +++ b/tensorflow/compiler/xla/tests/local_client_test_base.cc @@ -27,7 +27,7 @@ limitations under the License. #include "tensorflow/compiler/xla/test_helpers.h" #include "tensorflow/core/common_runtime/eigen_thread_pool.h" #include "tensorflow/core/lib/core/threadpool.h" -#include "tensorflow/core/platform/byte_order.h" +#include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/logging.h" diff --git a/tensorflow/contrib/factorization/kernels/clustering_ops.cc b/tensorflow/contrib/factorization/kernels/clustering_ops.cc index 025534d540..2a6c97e8b9 100644 --- a/tensorflow/contrib/factorization/kernels/clustering_ops.cc +++ b/tensorflow/contrib/factorization/kernels/clustering_ops.cc @@ -32,7 +32,6 @@ #include "tensorflow/core/lib/gtl/top_n.h" #include "tensorflow/core/lib/random/philox_random.h" #include "tensorflow/core/lib/random/simple_philox.h" -#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/logging.h" diff --git a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc index cca1a05419..35341406a0 100644 --- a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc +++ b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc @@ -28,7 +28,7 @@ #include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/lib/strings/str_util.h" -#include "tensorflow/core/platform/byte_order.h" +#include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/env.h" using tensorflow::strings::StrCat; diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 01fe61eeac..c461f9ed2f 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -282,7 +282,7 @@ PLATFORM_BASE_HDRS = [ "platform/logging.h", "platform/macros.h", "platform/types.h", - "platform/byte_order.h", + "platform/cpu_info.h", ] PLATFORM_OTHER_HDRS = [ @@ -290,7 +290,6 @@ PLATFORM_OTHER_HDRS = [ "platform/stacktrace.h", "platform/stacktrace_handler.h", "platform/context.h", - "platform/cpu_info.h", "platform/cpu_feature_guard.h", "platform/dynamic_annotations.h", "platform/env.h", @@ -319,6 +318,7 @@ cc_library( srcs = glob([ "platform/*/integral_types.h", "platform/*/logging.h", + "platform/*/cpu_info.h", ]), hdrs = PLATFORM_BASE_HDRS, deps = [ @@ -666,7 +666,6 @@ cc_library( "framework/tensor_types.h", "framework/type_traits.h", "lib/bfloat16/bfloat16.h", - "platform/byte_order.h", "platform/default/dynamic_annotations.h", "platform/default/integral_types.h", "platform/default/logging.h", @@ -1907,7 +1906,6 @@ cc_library( "lib/core/casts.h", "lib/core/stringpiece.h", "lib/png/png_io.h", - "platform/byte_order.h", "platform/cpu_info.h", "platform/default/integral_types.h", "platform/default/logging.h", diff --git a/tensorflow/core/common_runtime/direct_session.cc b/tensorflow/core/common_runtime/direct_session.cc index 0afbd02e86..0479061daf 100644 --- a/tensorflow/core/common_runtime/direct_session.cc +++ b/tensorflow/core/common_runtime/direct_session.cc @@ -54,7 +54,7 @@ limitations under the License. #include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/platform/byte_order.h" +#include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/device_tracer.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/mutex.h" diff --git a/tensorflow/core/common_runtime/kernel_benchmark_testlib.cc b/tensorflow/core/common_runtime/kernel_benchmark_testlib.cc index 7de1b80e2d..64d8849475 100644 --- a/tensorflow/core/common_runtime/kernel_benchmark_testlib.cc +++ b/tensorflow/core/common_runtime/kernel_benchmark_testlib.cc @@ -28,7 +28,6 @@ limitations under the License. #include "tensorflow/core/lib/core/notification.h" #include "tensorflow/core/lib/core/threadpool.h" #include "tensorflow/core/lib/strings/str_util.h" -#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/test_benchmark.h" diff --git a/tensorflow/core/common_runtime/local_device.cc b/tensorflow/core/common_runtime/local_device.cc index 873182371e..ca7f1614f1 100644 --- a/tensorflow/core/common_runtime/local_device.cc +++ b/tensorflow/core/common_runtime/local_device.cc @@ -19,7 +19,6 @@ limitations under the License. #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/common_runtime/eigen_thread_pool.h" #include "tensorflow/core/lib/core/threadpool.h" -#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_feature_guard.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/logging.h" diff --git a/tensorflow/core/common_runtime/process_util.cc b/tensorflow/core/common_runtime/process_util.cc index cf8e11c9c8..d5bd7f8b98 100644 --- a/tensorflow/core/common_runtime/process_util.cc +++ b/tensorflow/core/common_runtime/process_util.cc @@ -18,7 +18,6 @@ limitations under the License. #include #include "tensorflow/core/lib/core/threadpool.h" -#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/tracing.h" diff --git a/tensorflow/core/framework/bfloat16.h b/tensorflow/core/framework/bfloat16.h index 2f79d0fa70..968c18bdd2 100644 --- a/tensorflow/core/framework/bfloat16.h +++ b/tensorflow/core/framework/bfloat16.h @@ -17,7 +17,6 @@ limitations under the License. #define TENSORFLOW_FRAMEWORK_BFLOAT16_H_ #include "tensorflow/core/framework/numeric_types.h" -#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/types.h" #if defined(PLATFORM_WINDOWS) diff --git a/tensorflow/core/grappler/clusters/utils.cc b/tensorflow/core/grappler/clusters/utils.cc index a7519725a5..50d6e6468f 100644 --- a/tensorflow/core/grappler/clusters/utils.cc +++ b/tensorflow/core/grappler/clusters/utils.cc @@ -32,7 +32,6 @@ limitations under the License. #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/mem.h" diff --git a/tensorflow/core/grappler/costs/utils.cc b/tensorflow/core/grappler/costs/utils.cc index be54d98534..f318e3911c 100644 --- a/tensorflow/core/grappler/costs/utils.cc +++ b/tensorflow/core/grappler/costs/utils.cc @@ -44,7 +44,7 @@ limitations under the License. #include "tensorflow/core/lib/core/bits.h" #include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/platform/byte_order.h" +#include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/protobuf.h" diff --git a/tensorflow/core/grappler/devices.cc b/tensorflow/core/grappler/devices.cc index 2be894a08b..b318ac22d4 100644 --- a/tensorflow/core/grappler/devices.cc +++ b/tensorflow/core/grappler/devices.cc @@ -16,7 +16,6 @@ limitations under the License. #include #include "tensorflow/core/grappler/devices.h" -#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #if GOOGLE_CUDA diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc index 45bb188e8d..e29aaa25fe 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding.cc +++ b/tensorflow/core/grappler/optimizers/constant_folding.cc @@ -36,7 +36,6 @@ limitations under the License. #include "tensorflow/core/lib/gtl/inlined_vector.h" #include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/denormal.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/setround.h" diff --git a/tensorflow/core/kernels/batching_util/adaptive_shared_batch_scheduler.h b/tensorflow/core/kernels/batching_util/adaptive_shared_batch_scheduler.h index f5ced95feb..339d792302 100644 --- a/tensorflow/core/kernels/batching_util/adaptive_shared_batch_scheduler.h +++ b/tensorflow/core/kernels/batching_util/adaptive_shared_batch_scheduler.h @@ -28,7 +28,6 @@ limitations under the License. #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/threadpool.h" -#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/thread_annotations.h" diff --git a/tensorflow/core/kernels/batching_util/shared_batch_scheduler.h b/tensorflow/core/kernels/batching_util/shared_batch_scheduler.h index edc88a0384..b77289aded 100644 --- a/tensorflow/core/kernels/batching_util/shared_batch_scheduler.h +++ b/tensorflow/core/kernels/batching_util/shared_batch_scheduler.h @@ -30,7 +30,6 @@ limitations under the License. #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/thread_annotations.h" diff --git a/tensorflow/core/kernels/cast_op.h b/tensorflow/core/kernels/cast_op.h index 16d2e0e0a5..fd4e75d26f 100644 --- a/tensorflow/core/kernels/cast_op.h +++ b/tensorflow/core/kernels/cast_op.h @@ -21,7 +21,7 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/framework/types.h" -#include "tensorflow/core/platform/byte_order.h" +#include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/types.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/decode_raw_op.cc b/tensorflow/core/kernels/decode_raw_op.cc index eaef5a6097..bacacb94ae 100644 --- a/tensorflow/core/kernels/decode_raw_op.cc +++ b/tensorflow/core/kernels/decode_raw_op.cc @@ -21,7 +21,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/platform/byte_order.h" +#include "tensorflow/core/platform/cpu_info.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/mkl_input_conversion_op.cc b/tensorflow/core/kernels/mkl_input_conversion_op.cc index 3245625a32..68d3e1c9ab 100644 --- a/tensorflow/core/kernels/mkl_input_conversion_op.cc +++ b/tensorflow/core/kernels/mkl_input_conversion_op.cc @@ -24,7 +24,6 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/kernels/ops_util.h" -#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/util/tensor_format.h" diff --git a/tensorflow/core/kernels/mkl_tfconv_op.h b/tensorflow/core/kernels/mkl_tfconv_op.h index 4120f013ac..ddea9e281b 100644 --- a/tensorflow/core/kernels/mkl_tfconv_op.h +++ b/tensorflow/core/kernels/mkl_tfconv_op.h @@ -27,7 +27,6 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/kernels/ops_util.h" -#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/util/tensor_format.h" diff --git a/tensorflow/core/kernels/sparse_matmul_op.h b/tensorflow/core/kernels/sparse_matmul_op.h index e89280724e..14ef2ed704 100644 --- a/tensorflow/core/kernels/sparse_matmul_op.h +++ b/tensorflow/core/kernels/sparse_matmul_op.h @@ -17,7 +17,6 @@ limitations under the License. #define TENSORFLOW_KERNELS_SPARSE_MATMUL_OP_H_ #include "third_party/eigen3/Eigen/Core" -#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/types.h" #if defined(PLATFORM_WINDOWS) diff --git a/tensorflow/core/lib/bfloat16/bfloat16.h b/tensorflow/core/lib/bfloat16/bfloat16.h index e7c24387a4..126e5a17af 100644 --- a/tensorflow/core/lib/bfloat16/bfloat16.h +++ b/tensorflow/core/lib/bfloat16/bfloat16.h @@ -19,7 +19,8 @@ limitations under the License. #include #include -#include "tensorflow/core/platform/byte_order.h" +// We need cpu_info.h here in order to pick up __BYTE_ORDER__. +#include "tensorflow/core/platform/cpu_info.h" #ifdef __CUDACC__ // All functions callable from CUDA code must be qualified with __device__ diff --git a/tensorflow/core/lib/core/coding.cc b/tensorflow/core/lib/core/coding.cc index 50872eef83..bb95c27410 100644 --- a/tensorflow/core/lib/core/coding.cc +++ b/tensorflow/core/lib/core/coding.cc @@ -15,7 +15,7 @@ limitations under the License. #include "tensorflow/core/lib/core/coding.h" -#include "tensorflow/core/platform/byte_order.h" +#include "tensorflow/core/platform/cpu_info.h" namespace tensorflow { namespace core { diff --git a/tensorflow/core/lib/core/raw_coding.h b/tensorflow/core/lib/core/raw_coding.h index 37201b755d..bbfd33d303 100644 --- a/tensorflow/core/lib/core/raw_coding.h +++ b/tensorflow/core/lib/core/raw_coding.h @@ -17,7 +17,7 @@ limitations under the License. #define TENSORFLOW_LIB_CORE_RAW_CODING_H_ #include -#include "tensorflow/core/platform/byte_order.h" +#include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/types.h" namespace tensorflow { diff --git a/tensorflow/core/lib/gtl/inlined_vector.h b/tensorflow/core/lib/gtl/inlined_vector.h index 2011f7d4a1..6e3cb2206d 100644 --- a/tensorflow/core/lib/gtl/inlined_vector.h +++ b/tensorflow/core/lib/gtl/inlined_vector.h @@ -43,7 +43,7 @@ limitations under the License. #include #include "tensorflow/core/lib/gtl/manual_constructor.h" -#include "tensorflow/core/platform/byte_order.h" +#include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/mem.h" #include "tensorflow/core/platform/types.h" diff --git a/tensorflow/core/lib/png/png_io.cc b/tensorflow/core/lib/png/png_io.cc index 62c803afb2..cba473927d 100644 --- a/tensorflow/core/lib/png/png_io.cc +++ b/tensorflow/core/lib/png/png_io.cc @@ -26,7 +26,7 @@ limitations under the License. #include "tensorflow/core/lib/core/casts.h" #include "tensorflow/core/lib/png/png_io.h" -#include "tensorflow/core/platform/byte_order.h" +#include "tensorflow/core/platform/cpu_info.h" // endian #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/png.h" diff --git a/tensorflow/core/lib/wav/wav_io.cc b/tensorflow/core/lib/wav/wav_io.cc index 3f7dbcee85..51b9c6cd82 100644 --- a/tensorflow/core/lib/wav/wav_io.cc +++ b/tensorflow/core/lib/wav/wav_io.cc @@ -23,7 +23,7 @@ limitations under the License. #include "tensorflow/core/lib/core/coding.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/wav/wav_io.h" -#include "tensorflow/core/platform/byte_order.h" +#include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" diff --git a/tensorflow/core/platform/byte_order.h b/tensorflow/core/platform/byte_order.h deleted file mode 100644 index aab6535e4b..0000000000 --- a/tensorflow/core/platform/byte_order.h +++ /dev/null @@ -1,37 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CORE_PLATFORM_BYTE_ORDER_H_ -#define TENSORFLOW_CORE_PLATFORM_BYTE_ORDER_H_ - -// Byte order defines provided by gcc. MSVC doesn't define those so -// we define them here. -// We assume that all windows platform out there are little endian. -#if defined(_MSC_VER) && !defined(__clang__) -#define __ORDER_LITTLE_ENDIAN__ 0x4d2 -#define __ORDER_BIG_ENDIAN__ 0x10e1 -#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ -#endif - -namespace tensorflow { -namespace port { - -// TODO(jeff,sanjay): Make portable -constexpr bool kLittleEndian = __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__; - -} // namespace port -} // namespace tensorflow - -#endif // TENSORFLOW_CORE_PLATFORM_BYTE_ORDER_H_ diff --git a/tensorflow/core/platform/cpu_feature_guard.cc b/tensorflow/core/platform/cpu_feature_guard.cc index 9d00aa7b7f..b570658158 100644 --- a/tensorflow/core/platform/cpu_feature_guard.cc +++ b/tensorflow/core/platform/cpu_feature_guard.cc @@ -18,7 +18,6 @@ limitations under the License. #include #include -#include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/logging.h" diff --git a/tensorflow/core/platform/cpu_info.h b/tensorflow/core/platform/cpu_info.h index c42429a394..bb77650e26 100644 --- a/tensorflow/core/platform/cpu_info.h +++ b/tensorflow/core/platform/cpu_info.h @@ -25,6 +25,9 @@ limitations under the License. namespace tensorflow { namespace port { +// TODO(jeff,sanjay): Make portable +constexpr bool kLittleEndian = __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__; + // Returns an estimate of the number of schedulable CPUs for this // process. Usually, it's constant throughout the lifetime of a // process, but it might change if the underlying cluster management diff --git a/tensorflow/core/platform/denormal.cc b/tensorflow/core/platform/denormal.cc index c510dc204f..82cbc43b4f 100644 --- a/tensorflow/core/platform/denormal.cc +++ b/tensorflow/core/platform/denormal.cc @@ -15,9 +15,8 @@ limitations under the License. #include -#include "tensorflow/core/platform/byte_order.h" -#include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/denormal.h" +#include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/platform.h" // If we're on gcc 4.8 or older, there's a known bug that prevents the use of diff --git a/tensorflow/core/platform/windows/cpu_info.h b/tensorflow/core/platform/windows/cpu_info.h index ba2126abcf..f20939d3c0 100644 --- a/tensorflow/core/platform/windows/cpu_info.h +++ b/tensorflow/core/platform/windows/cpu_info.h @@ -19,4 +19,13 @@ limitations under the License. // included so __cpuidex function is available for GETCPUID on Windows #include +// Byte order defines provided by gcc. MSVC doesn't define those so +// we define them here. +// We assume that all windows platform out there are little endian. +#if defined(_MSC_VER) && !defined(__clang__) +#define __ORDER_LITTLE_ENDIAN__ 0x4d2 +#define __ORDER_BIG_ENDIAN__ 0x10e1 +#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ +#endif + #endif // TENSORFLOW_PLATFORM_WINDOWS_CPU_INFO_H_ -- GitLab From 91c31997e6854a3d07acc76381cff7436df1c1dd Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 08:12:42 -0700 Subject: [PATCH 317/791] Add support to TFLite for dilated convolution. PiperOrigin-RevId: 192770919 --- tensorflow/contrib/lite/builtin_op_data.h | 2 + tensorflow/contrib/lite/kernels/conv.cc | 67 ++++++++++++------- tensorflow/contrib/lite/kernels/conv_test.cc | 8 ++- .../contrib/lite/kernels/depthwise_conv.cc | 6 +- tensorflow/contrib/lite/kernels/padding.h | 7 +- tensorflow/contrib/lite/kernels/pooling.cc | 4 +- tensorflow/contrib/lite/model.cc | 2 + tensorflow/contrib/lite/schema/schema.fbs | 2 + .../contrib/lite/schema/schema_generated.h | 38 +++++++++-- .../contrib/lite/testing/generate_examples.py | 3 + .../contrib/lite/toco/tflite/operator.cc | 6 +- 11 files changed, 104 insertions(+), 41 deletions(-) diff --git a/tensorflow/contrib/lite/builtin_op_data.h b/tensorflow/contrib/lite/builtin_op_data.h index f5fb2f15e3..4910c89eae 100644 --- a/tensorflow/contrib/lite/builtin_op_data.h +++ b/tensorflow/contrib/lite/builtin_op_data.h @@ -53,6 +53,8 @@ typedef struct { TfLitePadding padding; int stride_width; int stride_height; + int dilation_width_factor; + int dilation_height_factor; TfLiteFusedActivation activation; } TfLiteConvParams; diff --git a/tensorflow/contrib/lite/kernels/conv.cc b/tensorflow/contrib/lite/kernels/conv.cc index 18ff33bf9f..3b467b3aa2 100644 --- a/tensorflow/contrib/lite/kernels/conv.cc +++ b/tensorflow/contrib/lite/kernels/conv.cc @@ -225,22 +225,27 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { // Matching GetWindowedOutputSize in TensorFlow. auto padding = params->padding; - auto computeOutSize = [padding](int imageSize, int filterSize, - int stride) -> int { + auto computeOutSize = [padding](int imageSize, int filterSize, int stride, + int dilationRate) -> int { + int effectiveFilterSize = (filterSize - 1) * dilationRate + 1; return padding == kTfLitePaddingSame ? (imageSize + stride - 1) / stride : padding == kTfLitePaddingValid - ? (imageSize - filterSize + stride) / stride + ? (imageSize - effectiveFilterSize + stride) / stride : 0; }; - int outWidth = computeOutSize(width, filter_width, params->stride_width); - int outHeight = computeOutSize(height, filter_height, params->stride_height); + int outWidth = computeOutSize(width, filter_width, params->stride_width, + params->dilation_width_factor); + int outHeight = computeOutSize(height, filter_height, params->stride_height, + params->dilation_height_factor); data->padding.height = - ComputePadding(params->stride_height, height, filter_height, outHeight); + ComputePadding(params->stride_height, params->dilation_height_factor, + height, filter_height, outHeight); data->padding.width = - ComputePadding(params->stride_width, width, filter_width, outWidth); + ComputePadding(params->stride_width, params->dilation_width_factor, width, + filter_width, outWidth); TF_LITE_ENSURE(context, hasBias); @@ -375,28 +380,40 @@ void EvalFloat(TfLiteContext* context, TfLiteNode* node, float output_activation_min, output_activation_max; CalculateActivationRangeFloat(params->activation, &output_activation_min, &output_activation_max); - - switch (kernel_type) { + KernelType effective_kernel_type; + if (((kernel_type == kMultithreadOptimized) || + (kernel_type == kCblasOptimized)) && + ((params->dilation_width_factor != 1) || + (params->dilation_height_factor != 1))) { + // kMultithreadOptimized and kCblasOptimized do not support dilation. + // Therefore, fallback to optimized. + effective_kernel_type = kGenericOptimized; + } else { + effective_kernel_type = kernel_type; + } + switch (effective_kernel_type) { case kReference: { - reference_ops::Conv(GetTensorData(input), GetTensorDims(input), - GetTensorData(filter), GetTensorDims(filter), - GetTensorData(bias), GetTensorDims(bias), - params->stride_width, params->stride_height, 1, 1, - data->padding.width, data->padding.height, - output_activation_min, output_activation_max, - GetTensorData(output), GetTensorDims(output), - GetTensorData(im2col), GetTensorDims(im2col)); + reference_ops::Conv( + GetTensorData(input), GetTensorDims(input), + GetTensorData(filter), GetTensorDims(filter), + GetTensorData(bias), GetTensorDims(bias), params->stride_width, + params->stride_height, params->dilation_width_factor, + params->dilation_height_factor, data->padding.width, + data->padding.height, output_activation_min, output_activation_max, + GetTensorData(output), GetTensorDims(output), + GetTensorData(im2col), GetTensorDims(im2col)); break; } case kGenericOptimized: { - optimized_ops::Conv(GetTensorData(input), GetTensorDims(input), - GetTensorData(filter), GetTensorDims(filter), - GetTensorData(bias), GetTensorDims(bias), - params->stride_width, params->stride_height, 1, 1, - data->padding.width, data->padding.height, - output_activation_min, output_activation_max, - GetTensorData(output), GetTensorDims(output), - GetTensorData(im2col), GetTensorDims(im2col)); + optimized_ops::Conv( + GetTensorData(input), GetTensorDims(input), + GetTensorData(filter), GetTensorDims(filter), + GetTensorData(bias), GetTensorDims(bias), params->stride_width, + params->stride_height, params->dilation_width_factor, + params->dilation_height_factor, data->padding.width, + data->padding.height, output_activation_min, output_activation_max, + GetTensorData(output), GetTensorDims(output), + GetTensorData(im2col), GetTensorDims(im2col)); break; } case kMultithreadOptimized: { diff --git a/tensorflow/contrib/lite/kernels/conv_test.cc b/tensorflow/contrib/lite/kernels/conv_test.cc index d2393c3c97..0dcfc826fd 100644 --- a/tensorflow/contrib/lite/kernels/conv_test.cc +++ b/tensorflow/contrib/lite/kernels/conv_test.cc @@ -46,7 +46,8 @@ class BaseConvolutionOpModel : public SingleOpModel { TfLiteRegistration* registration, const TensorData& input, const TensorData& filter, const TensorData& output, int stride_width = 2, int stride_height = 2, enum Padding padding = Padding_VALID, - enum ActivationFunctionType activation = ActivationFunctionType_NONE) { + enum ActivationFunctionType activation = ActivationFunctionType_NONE, + int dilation_width_factor = 1, int dilation_height_factor = 1) { input_ = AddInput(input); filter_ = AddInput(filter); @@ -71,8 +72,9 @@ class BaseConvolutionOpModel : public SingleOpModel { } SetBuiltinOp(BuiltinOperator_CONV_2D, BuiltinOptions_Conv2DOptions, - CreateConv2DOptions(builder_, padding, stride_width, - stride_height, activation) + CreateConv2DOptions( + builder_, padding, stride_width, stride_height, activation, + dilation_width_factor, dilation_height_factor) .Union()); resolver_ = absl::make_unique(BuiltinOperator_CONV_2D, diff --git a/tensorflow/contrib/lite/kernels/depthwise_conv.cc b/tensorflow/contrib/lite/kernels/depthwise_conv.cc index cad9ce114c..eeda1bc3c5 100644 --- a/tensorflow/contrib/lite/kernels/depthwise_conv.cc +++ b/tensorflow/contrib/lite/kernels/depthwise_conv.cc @@ -140,10 +140,10 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { int out_height = compute_out_size(height, filter_height, params->stride_height); - data->padding.height = - ComputePadding(params->stride_height, height, filter_height, out_height); + data->padding.height = ComputePadding(params->stride_height, 1, height, + filter_height, out_height); data->padding.width = - ComputePadding(params->stride_width, width, filter_width, out_width); + ComputePadding(params->stride_width, 1, width, filter_width, out_width); // Note that quantized inference requires that all tensors have their // parameters set. This is usually done during quantized training. diff --git a/tensorflow/contrib/lite/kernels/padding.h b/tensorflow/contrib/lite/kernels/padding.h index 40b8476b37..e81b970e0f 100644 --- a/tensorflow/contrib/lite/kernels/padding.h +++ b/tensorflow/contrib/lite/kernels/padding.h @@ -17,9 +17,10 @@ limitations under the License. namespace tflite { -inline int ComputePadding(int stride, int in_size, int filter_size, - int out_size) { - int padding = ((out_size - 1) * stride + filter_size - in_size) / 2; +inline int ComputePadding(int stride, int dilation_rate, int in_size, + int filter_size, int out_size) { + int effective_filter_size = (filter_size - 1) * dilation_rate + 1; + int padding = ((out_size - 1) * stride + effective_filter_size - in_size) / 2; return padding > 0 ? padding : 0; } diff --git a/tensorflow/contrib/lite/kernels/pooling.cc b/tensorflow/contrib/lite/kernels/pooling.cc index b798801108..0bf27c34c1 100644 --- a/tensorflow/contrib/lite/kernels/pooling.cc +++ b/tensorflow/contrib/lite/kernels/pooling.cc @@ -94,9 +94,9 @@ TfLiteStatus GenericPrepare(TfLiteContext* context, TfLiteNode* node) { int outHeight = computeOutSize(height, params->filter_height, params->stride_height); - data->padding.height = ComputePadding(params->stride_height, height, + data->padding.height = ComputePadding(params->stride_height, 1, height, params->filter_height, outHeight); - data->padding.width = ComputePadding(params->stride_width, width, + data->padding.width = ComputePadding(params->stride_width, 1, width, params->filter_width, outWidth); if (input->type == kTfLiteUInt8) { diff --git a/tensorflow/contrib/lite/model.cc b/tensorflow/contrib/lite/model.cc index 54b1460173..2dd6d67e07 100644 --- a/tensorflow/contrib/lite/model.cc +++ b/tensorflow/contrib/lite/model.cc @@ -333,6 +333,8 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, params->stride_height = conv_params->stride_h(); params->activation = parse_activation(conv_params->fused_activation_function()); + params->dilation_width_factor = conv_params->dilation_w_factor(); + params->dilation_height_factor = conv_params->dilation_h_factor(); } *builtin_data = reinterpret_cast(params); break; diff --git a/tensorflow/contrib/lite/schema/schema.fbs b/tensorflow/contrib/lite/schema/schema.fbs index 93980b15f0..2b62c257d8 100644 --- a/tensorflow/contrib/lite/schema/schema.fbs +++ b/tensorflow/contrib/lite/schema/schema.fbs @@ -199,6 +199,8 @@ table Conv2DOptions { stride_w:int; stride_h:int; fused_activation_function:ActivationFunctionType; + dilation_w_factor:int = 1; + dilation_h_factor:int = 1; } table Pool2DOptions { diff --git a/tensorflow/contrib/lite/schema/schema_generated.h b/tensorflow/contrib/lite/schema/schema_generated.h index b2a799d0ef..0b9961d606 100755 --- a/tensorflow/contrib/lite/schema/schema_generated.h +++ b/tensorflow/contrib/lite/schema/schema_generated.h @@ -1478,11 +1478,15 @@ struct Conv2DOptionsT : public flatbuffers::NativeTable { int32_t stride_w; int32_t stride_h; ActivationFunctionType fused_activation_function; + int32_t dilation_w_factor; + int32_t dilation_h_factor; Conv2DOptionsT() : padding(Padding_SAME), stride_w(0), stride_h(0), - fused_activation_function(ActivationFunctionType_NONE) { + fused_activation_function(ActivationFunctionType_NONE), + dilation_w_factor(0), + dilation_h_factor(0) { } }; @@ -1492,7 +1496,9 @@ struct Conv2DOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { VT_PADDING = 4, VT_STRIDE_W = 6, VT_STRIDE_H = 8, - VT_FUSED_ACTIVATION_FUNCTION = 10 + VT_FUSED_ACTIVATION_FUNCTION = 10, + VT_DILATION_W_FACTOR = 12, + VT_DILATION_H_FACTOR = 14 }; Padding padding() const { return static_cast(GetField(VT_PADDING, 0)); @@ -1506,12 +1512,20 @@ struct Conv2DOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { ActivationFunctionType fused_activation_function() const { return static_cast(GetField(VT_FUSED_ACTIVATION_FUNCTION, 0)); } + int32_t dilation_w_factor() const { + return GetField(VT_DILATION_W_FACTOR, 0); + } + int32_t dilation_h_factor() const { + return GetField(VT_DILATION_H_FACTOR, 0); + } bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && VerifyField(verifier, VT_PADDING) && VerifyField(verifier, VT_STRIDE_W) && VerifyField(verifier, VT_STRIDE_H) && VerifyField(verifier, VT_FUSED_ACTIVATION_FUNCTION) && + VerifyField(verifier, VT_DILATION_W_FACTOR) && + VerifyField(verifier, VT_DILATION_H_FACTOR) && verifier.EndTable(); } Conv2DOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; @@ -1534,6 +1548,12 @@ struct Conv2DOptionsBuilder { void add_fused_activation_function(ActivationFunctionType fused_activation_function) { fbb_.AddElement(Conv2DOptions::VT_FUSED_ACTIVATION_FUNCTION, static_cast(fused_activation_function), 0); } + void add_dilation_w_factor(int32_t dilation_w_factor) { + fbb_.AddElement(Conv2DOptions::VT_DILATION_W_FACTOR, dilation_w_factor, 0); + } + void add_dilation_h_factor(int32_t dilation_h_factor) { + fbb_.AddElement(Conv2DOptions::VT_DILATION_H_FACTOR, dilation_h_factor, 0); + } explicit Conv2DOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); @@ -1551,8 +1571,12 @@ inline flatbuffers::Offset CreateConv2DOptions( Padding padding = Padding_SAME, int32_t stride_w = 0, int32_t stride_h = 0, - ActivationFunctionType fused_activation_function = ActivationFunctionType_NONE) { + ActivationFunctionType fused_activation_function = ActivationFunctionType_NONE, + int32_t dilation_w_factor = 0, + int32_t dilation_h_factor = 0) { Conv2DOptionsBuilder builder_(_fbb); + builder_.add_dilation_h_factor(dilation_h_factor); + builder_.add_dilation_w_factor(dilation_w_factor); builder_.add_stride_h(stride_h); builder_.add_stride_w(stride_w); builder_.add_fused_activation_function(fused_activation_function); @@ -4885,6 +4909,8 @@ inline void Conv2DOptions::UnPackTo(Conv2DOptionsT *_o, const flatbuffers::resol { auto _e = stride_w(); _o->stride_w = _e; }; { auto _e = stride_h(); _o->stride_h = _e; }; { auto _e = fused_activation_function(); _o->fused_activation_function = _e; }; + { auto _e = dilation_w_factor(); _o->dilation_w_factor = _e; }; + { auto _e = dilation_h_factor(); _o->dilation_h_factor = _e; }; } inline flatbuffers::Offset Conv2DOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const Conv2DOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) { @@ -4899,12 +4925,16 @@ inline flatbuffers::Offset CreateConv2DOptions(flatbuffers::FlatB auto _stride_w = _o->stride_w; auto _stride_h = _o->stride_h; auto _fused_activation_function = _o->fused_activation_function; + auto _dilation_w_factor = _o->dilation_w_factor; + auto _dilation_h_factor = _o->dilation_h_factor; return tflite::CreateConv2DOptions( _fbb, _padding, _stride_w, _stride_h, - _fused_activation_function); + _fused_activation_function, + _dilation_w_factor, + _dilation_h_factor); } inline Pool2DOptionsT *Pool2DOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { diff --git a/tensorflow/contrib/lite/testing/generate_examples.py b/tensorflow/contrib/lite/testing/generate_examples.py index 53b41d2358..e045c27427 100644 --- a/tensorflow/contrib/lite/testing/generate_examples.py +++ b/tensorflow/contrib/lite/testing/generate_examples.py @@ -1039,6 +1039,7 @@ def make_conv_tests(zip_path): "input_shape": [[1, 3, 4, 3]], "filter_shape": [[1, 1, 3, 2]], "strides": [[1, 1, 1, 1], [1, 2, 3, 1]], + "dilations": [[1, 1, 1, 1], [1, 3, 2, 1], [1, 2, 2, 1]], "padding": ["SAME", "VALID"], "data_format": ["NHWC"], # TODO(aselle): NCHW would be good "constant_filter": [True, False], @@ -1047,6 +1048,7 @@ def make_conv_tests(zip_path): "input_shape": [[2, 14, 14, 2]], "filter_shape": [[6, 6, 2, 2]], "strides": [[1, 1, 1, 1], [1, 2, 3, 1]], + "dilations": [[1, 1, 1, 1], [1, 2, 2, 1]], "padding": ["SAME", "VALID"], "data_format": ["NHWC"], # TODO(aselle): NCHW would be good "constant_filter": [True, False], @@ -1072,6 +1074,7 @@ def make_conv_tests(zip_path): input_tensor, filter_input, strides=parameters["strides"], + dilations=parameters["dilations"], padding=parameters["padding"], data_format=parameters["data_format"]) return input_tensors, [out] diff --git a/tensorflow/contrib/lite/toco/tflite/operator.cc b/tensorflow/contrib/lite/toco/tflite/operator.cc index f41a312b47..d2e14ac5e0 100644 --- a/tensorflow/contrib/lite/toco/tflite/operator.cc +++ b/tensorflow/contrib/lite/toco/tflite/operator.cc @@ -68,7 +68,9 @@ class Convolution auto activation_function = ActivationFunction::Serialize(op.fused_activation_function); return ::tflite::CreateConv2DOptions(*builder, padding, op.stride_width, - op.stride_height, activation_function); + op.stride_height, activation_function, + op.dilation_width_factor, + op.dilation_height_factor); } void ReadOptions(const TfLiteOptions& options, @@ -76,6 +78,8 @@ class Convolution op->padding.type = Padding::Deserialize(options.padding()); op->stride_width = options.stride_w(); op->stride_height = options.stride_h(); + op->dilation_width_factor = options.dilation_w_factor(); + op->dilation_height_factor = options.dilation_h_factor(); op->fused_activation_function = ActivationFunction::Deserialize(options.fused_activation_function()); } -- GitLab From 17aa70e87ad9818f8918534ac4a567c3a3ef4550 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 08:17:49 -0700 Subject: [PATCH 318/791] Refactor to remove the duplicate calls to obtain a function's namespace. This removes the need to explicitly import internal components (barring the tf module which cannot be imported directly). PiperOrigin-RevId: 192771440 --- tensorflow/contrib/autograph/impl/api.py | 14 +++--- tensorflow/contrib/autograph/impl/api_test.py | 2 - tensorflow/contrib/autograph/impl/config.py | 6 --- .../contrib/autograph/impl/conversion.py | 48 ++++++++++--------- .../contrib/autograph/impl/conversion_test.py | 9 ++-- 5 files changed, 37 insertions(+), 42 deletions(-) diff --git a/tensorflow/contrib/autograph/impl/api.py b/tensorflow/contrib/autograph/impl/api.py index a00d9c68dc..f97a33326e 100644 --- a/tensorflow/contrib/autograph/impl/api.py +++ b/tensorflow/contrib/autograph/impl/api.py @@ -235,7 +235,8 @@ def to_graph(e, nocompile_decorators=(convert, do_not_convert, converted_call), partial_types=partial_types, api_module=tf_inspect.getmodule(to_graph)) - _, name = conversion.entity_to_graph(e, conversion_map, arg_values, arg_types) + _, name, namespace = conversion.entity_to_graph(e, conversion_map, arg_values, + arg_types) module = gast.Module([]) for import_line in config.COMPILED_IMPORT_STATEMENTS: @@ -244,13 +245,12 @@ def to_graph(e, module.body.append(dep) compiled_node, compiled_src = compiler.ast_to_object(module) - # The compiled code should see everything the entry function saw. + # The compiled code should see everything the entry entity saw. # TODO(mdan): This might not work well if the call tree spans modules? - if tf_inspect.isfunction(e): - for key, val in inspect_utils.getnamespace(e).items(): - # Avoid overwriting entities that have been transformed. - if key not in compiled_node.__dict__: - compiled_node.__dict__[key] = val + for key, val in namespace.items(): + # Avoid overwriting entities that have been transformed. + if key not in compiled_node.__dict__: + compiled_node.__dict__[key] = val compiled_fn = getattr(compiled_node, name) if verbose: diff --git a/tensorflow/contrib/autograph/impl/api_test.py b/tensorflow/contrib/autograph/impl/api_test.py index 2e09d19621..a7737b7f44 100644 --- a/tensorflow/contrib/autograph/impl/api_test.py +++ b/tensorflow/contrib/autograph/impl/api_test.py @@ -39,8 +39,6 @@ class ApiTest(test.TestCase): 'from __future__ import print_function', 'from tensorflow.contrib.autograph import utils' ' as autograph_utils', - 'from tensorflow.contrib.autograph import operators' - ' as __ops', 'tf = autograph_utils.fake_tf()', ) diff --git a/tensorflow/contrib/autograph/impl/config.py b/tensorflow/contrib/autograph/impl/config.py index 26326465e2..2600088595 100644 --- a/tensorflow/contrib/autograph/impl/config.py +++ b/tensorflow/contrib/autograph/impl/config.py @@ -46,10 +46,4 @@ NO_SIDE_EFFECT_CONSTRUCTORS = set(('tensorflow',)) COMPILED_IMPORT_STATEMENTS = ( 'from __future__ import print_function', 'import tensorflow as tf', - 'from tensorflow.contrib.autograph.impl import api' - ' as autograph_api', - 'from tensorflow.contrib.autograph import utils' - ' as autograph_utils', - 'from tensorflow.contrib.autograph import operators' - ' as __ops', ) diff --git a/tensorflow/contrib/autograph/impl/conversion.py b/tensorflow/contrib/autograph/impl/conversion.py index 3bacc94300..373dc1602b 100644 --- a/tensorflow/contrib/autograph/impl/conversion.py +++ b/tensorflow/contrib/autograph/impl/conversion.py @@ -20,6 +20,7 @@ from __future__ import print_function import gast +from tensorflow.contrib.autograph import operators from tensorflow.contrib.autograph import utils from tensorflow.contrib.autograph.converters import asserts from tensorflow.contrib.autograph.converters import break_statements @@ -138,20 +139,22 @@ def entity_to_graph(o, conversion_map, arg_values, arg_types): parameters. Returns: - A tuple (ast, new_name): + A tuple (ast, new_name, namespace): * ast: An AST representing an entity with interface equivalent to `o`, but which when executed it creates TF a graph. * new_name: The symbol name under which the new entity can be found. + * namespace: A dict mapping all symbols visible to the converted entity, + keyed by their symbol name. Raises: ValueError: if the entity type is not supported. """ if tf_inspect.isclass(o): - node, new_name = class_to_graph(o, conversion_map) + node, name, ns = class_to_graph(o, conversion_map) elif tf_inspect.isfunction(o): - node, new_name = function_to_graph(o, conversion_map, arg_values, arg_types) + node, name, ns = function_to_graph(o, conversion_map, arg_values, arg_types) elif tf_inspect.ismethod(o): - node, new_name = function_to_graph(o, conversion_map, arg_values, arg_types) + node, name, ns = function_to_graph(o, conversion_map, arg_values, arg_types) else: raise ValueError( 'Entity "%s" has unsupported type "%s". Only functions and classes are ' @@ -174,7 +177,7 @@ def entity_to_graph(o, conversion_map, arg_values, arg_types): continue entity_to_graph(candidate, conversion_map, {}, {}) - return node, new_name + return node, name, ns def class_to_graph(c, conversion_map): @@ -185,17 +188,18 @@ def class_to_graph(c, conversion_map): if not members: raise ValueError('Cannot convert %s: it has no member methods.' % c) - class_namespace = None + class_namespace = {} for _, m in members: - node, _ = function_to_graph( + node, _, namespace = function_to_graph( m, conversion_map=conversion_map, arg_values={}, arg_types={'self': (c.__name__, c)}, owner_type=c) - # TODO(mdan): Do not assume all members have the same view of globals. if class_namespace is None: - class_namespace = inspect_utils.getnamespace(m) + class_namespace = namespace + else: + class_namespace.update(namespace) converted_members[m] = node namer = conversion_map.new_namer(class_namespace) class_name = namer.compiled_class_name(c.__name__, c) @@ -206,25 +210,23 @@ def class_to_graph(c, conversion_map): body=list(converted_members.values()), decorator_list=[]) - return node, class_name + return node, class_name, class_namespace + + +def _add_reserved_symbol(namespace, name, entity): + if name not in namespace: + namespace[name] = entity + elif namespace[name] != entity: + raise ValueError('The name "%s" is reserved and may not be used.' % name) def _add_self_references(namespace, api_module): - """Self refs are only required for analysis and are not used directly.""" # Manually add the utils namespace which may be used from generated code. - if 'autograph_util' not in namespace: - namespace['autograph_utils'] = utils - elif namespace['autograph_utils'] != utils: - raise ValueError( - 'The module name "autograph_utils" is reserved and may not be used.') - + _add_reserved_symbol(namespace, 'autograph_utils', utils) + _add_reserved_symbol(namespace, '__ops', operators) # We also make reference to the api module for dynamic conversion, but # to avoid circular references we don't import it here. - if 'autograph_api' not in namespace: - namespace['autograph_api'] = api_module - elif namespace['autograph_api'] != api_module: - raise ValueError( - 'The module name "autograph_api" is reserved and may not be used.') + _add_reserved_symbol(namespace, 'autograph_api', api_module) def function_to_graph(f, conversion_map, arg_values, arg_types, @@ -261,7 +263,7 @@ def function_to_graph(f, conversion_map, arg_values, arg_types, # TODO(mdan): Use this at compilation. conversion_map.additional_imports.update(deps) - return node, new_name + return node, new_name, namespace def _static_analysis_pass(node, ctx): diff --git a/tensorflow/contrib/autograph/impl/conversion_test.py b/tensorflow/contrib/autograph/impl/conversion_test.py index 7066739eb8..962009c71f 100644 --- a/tensorflow/contrib/autograph/impl/conversion_test.py +++ b/tensorflow/contrib/autograph/impl/conversion_test.py @@ -43,14 +43,15 @@ class ConversionTest(test.TestCase): conversion.entity_to_graph('dummy', conversion_map, None, None) def test_entity_to_graph_callable(self): - + b = 2 def f(a): - return a + return a + b conversion_map = conversion.ConversionMap(True, (), (), None) - ast, new_name = conversion.entity_to_graph(f, conversion_map, None, None) + ast, name, ns = conversion.entity_to_graph(f, conversion_map, None, None) self.assertTrue(isinstance(ast, gast.FunctionDef), ast) - self.assertEqual('tf__f', new_name) + self.assertEqual('tf__f', name) + self.assertTrue(ns['b'] is b) def test_entity_to_graph_call_tree(self): -- GitLab From 554c587c54d0725d6da0ce39557d17b8393c35bc Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 08:22:06 -0700 Subject: [PATCH 319/791] Experiment with pre-shuffled fully-connected weights PiperOrigin-RevId: 192771889 --- .../lite/kernels/internal/compatibility.h | 1 + .../internal/optimized/optimized_ops.h | 136 ++++++++++++++++++ .../internal/reference/reference_ops.h | 61 ++++++++ tensorflow/contrib/lite/toco/BUILD | 1 + .../experimental_shuffle_fc_weights.cc | 135 +++++++++++++++++ .../graph_transformations.h | 1 + .../graph_transformations/identify_lstm.cc | 6 + tensorflow/contrib/lite/toco/model.h | 1 + 8 files changed, 342 insertions(+) create mode 100644 tensorflow/contrib/lite/toco/graph_transformations/experimental_shuffle_fc_weights.cc diff --git a/tensorflow/contrib/lite/kernels/internal/compatibility.h b/tensorflow/contrib/lite/kernels/internal/compatibility.h index 51426bb1c5..93fc6b6a76 100644 --- a/tensorflow/contrib/lite/kernels/internal/compatibility.h +++ b/tensorflow/contrib/lite/kernels/internal/compatibility.h @@ -77,6 +77,7 @@ limitations under the License. #endif // TODO(ahentz): Clean up. +using int8 = std::int8_t; using uint8 = std::uint8_t; using int16 = std::int16_t; using uint16 = std::uint16_t; diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h index fa91db7fe1..7fc6615965 100644 --- a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h +++ b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h @@ -1203,6 +1203,142 @@ void FullyConnected(const uint8* input_data, const Dims<4>& input_dims, output_activation_max, output_data, output_dims, gemm_context); } +inline void ExperimentalShuffledFullyConnected( + const uint8* input_data, const Dims<4>& input_dims, + const uint8* shuffled_weights_data, const Dims<4>& weights_dims, + const int32* bias_data, const Dims<4>& bias_dims, int32 output_multiplier, + int output_shift, int32 output_activation_min, int32 output_activation_max, + int16* output_data, const Dims<4>& output_dims, + gemmlowp::GemmContext* gemm_context) { + gemmlowp::ScopedProfilingLabel label( + "ExperimentalShuffledFullyConnected/8bit"); + (void)gemm_context; // only used in optimized code. + TFLITE_DCHECK_EQ(output_activation_min, -32768); + TFLITE_DCHECK_EQ(output_activation_max, 32767); + // TODO(benoitjacob): This really should be: + // const int batches = ArraySize(output_dims, 1); + // but the current --variable_batch hack consists in overwriting the 3rd + // dimension with the runtime batch size, as we don't keep track for each + // array of which dimension is the batch dimension in it. + const int batches = ArraySize(output_dims, 1) * ArraySize(output_dims, 2) * + ArraySize(output_dims, 3); + const int output_depth = MatchingArraySize(weights_dims, 1, output_dims, 0); + const int accum_depth = ArraySize(weights_dims, 0); + TFLITE_DCHECK(IsPackedWithoutStrides(input_dims)); + TFLITE_DCHECK(IsPackedWithoutStrides(weights_dims)); + // The experimental shuffling is an optimization for matrix*vector product. + // We aren't interested in supporting non-matrix*vector-product cases, i.e. + // batches>1. + TFLITE_DCHECK_EQ(batches, 1); + // Shuffled weights have had their sign bit (0x80) pre-flipped (xor'd) + // so that just reinterpreting them as int8 values is equivalent to + // subtracting 128 from them, thus implementing for free the subtraction of + // the zero_point value 128. + const int8* shuffled_weights_ptr = + reinterpret_cast(shuffled_weights_data); +#if defined USE_NEON + // We'll only need to xor signbit to the input activation values, as + // that xor-ing is pre-built into the shuffled weights values. + const uint8x16_t signbit = vdupq_n_u8(0x80); + const int right_shift = output_shift > 0 ? output_shift : 0; + const int left_shift = output_shift > 0 ? 0 : -output_shift; + for (int c = 0; c < output_depth; c += 4) { + // Accumulation loop. + int32x4_t row_accum0 = vdupq_n_s32(0); + int32x4_t row_accum1 = vdupq_n_s32(0); + int32x4_t row_accum2 = vdupq_n_s32(0); + int32x4_t row_accum3 = vdupq_n_s32(0); + for (int d = 0; d < accum_depth; d += 16) { + int8x16_t weights0 = vld1q_s8(shuffled_weights_ptr + 0); + int8x16_t weights1 = vld1q_s8(shuffled_weights_ptr + 16); + int8x16_t weights2 = vld1q_s8(shuffled_weights_ptr + 32); + int8x16_t weights3 = vld1q_s8(shuffled_weights_ptr + 48); + shuffled_weights_ptr += 64; + int8x16_t input = + vreinterpretq_s8_u8(veorq_u8(signbit, vld1q_u8(input_data + d))); + int16x8_t local_accum0 = + vmull_s8(vget_low_s8(weights0), vget_low_s8(input)); + int16x8_t local_accum1 = + vmull_s8(vget_low_s8(weights1), vget_low_s8(input)); + int16x8_t local_accum2 = + vmull_s8(vget_low_s8(weights2), vget_low_s8(input)); + int16x8_t local_accum3 = + vmull_s8(vget_low_s8(weights3), vget_low_s8(input)); + local_accum0 = + vmlal_s8(local_accum0, vget_high_s8(weights0), vget_high_s8(input)); + local_accum1 = + vmlal_s8(local_accum1, vget_high_s8(weights1), vget_high_s8(input)); + local_accum2 = + vmlal_s8(local_accum2, vget_high_s8(weights2), vget_high_s8(input)); + local_accum3 = + vmlal_s8(local_accum3, vget_high_s8(weights3), vget_high_s8(input)); + row_accum0 = vpadalq_s16(row_accum0, local_accum0); + row_accum1 = vpadalq_s16(row_accum1, local_accum1); + row_accum2 = vpadalq_s16(row_accum2, local_accum2); + row_accum3 = vpadalq_s16(row_accum3, local_accum3); + } + // Horizontally reduce accumulators + int32x2_t pairwise_reduced_acc_0, pairwise_reduced_acc_1, + pairwise_reduced_acc_2, pairwise_reduced_acc_3; + pairwise_reduced_acc_0 = + vpadd_s32(vget_low_s32(row_accum0), vget_high_s32(row_accum0)); + pairwise_reduced_acc_1 = + vpadd_s32(vget_low_s32(row_accum1), vget_high_s32(row_accum1)); + pairwise_reduced_acc_2 = + vpadd_s32(vget_low_s32(row_accum2), vget_high_s32(row_accum2)); + pairwise_reduced_acc_3 = + vpadd_s32(vget_low_s32(row_accum3), vget_high_s32(row_accum3)); + const int32x2_t reduced_lo = + vpadd_s32(pairwise_reduced_acc_0, pairwise_reduced_acc_1); + const int32x2_t reduced_hi = + vpadd_s32(pairwise_reduced_acc_2, pairwise_reduced_acc_3); + int32x4_t reduced = vcombine_s32(reduced_lo, reduced_hi); + // Add bias values. + int32x4_t bias_vec = vld1q_s32(bias_data + c); + reduced = vaddq_s32(reduced, bias_vec); + reduced = vshlq_s32(reduced, vdupq_n_s32(left_shift)); + // Multiply by the fixed-point multiplier. + reduced = vqrdmulhq_n_s32(reduced, output_multiplier); + // Rounding-shift-right. + using gemmlowp::RoundingDivideByPOT; + reduced = RoundingDivideByPOT(reduced, right_shift); + // Narrow values down to 16 bit signed. + const int16x4_t res16 = vqmovn_s32(reduced); + vst1_s16(output_data + c, res16); + } +#else + for (int c = 0; c < output_depth; c += 4) { + // Internal accumulation. + // Initialize accumulator with the bias-value. + int32 accum[4] = {0}; + // Accumulation loop. + for (int d = 0; d < accum_depth; d += 16) { + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 16; j++) { + int8 input_val = input_data[d + j] - 128; + int8 weights_val = *shuffled_weights_ptr++; + accum[i] += weights_val * input_val; + } + } + } + for (int i = 0; i < 4; i++) { + // Add bias value + int acc = accum[i] + bias_data[c + i]; + // Down-scale the final int32 accumulator to the scale used by our + // (16-bit, typically 3 integer bits) fixed-point format. The quantized + // multiplier and shift here have been pre-computed offline + // (e.g. by toco). + acc = + MultiplyByQuantizedMultiplier(acc, output_multiplier, -output_shift); + // Saturate, cast to int16, and store to output array. + acc = std::max(acc, output_activation_min); + acc = std::min(acc, output_activation_max); + output_data[c + i] = acc; + } + } +#endif +} + template inline void ExtractPatchIntoBufferColumn( const Dims<4>& input_dims, int w, int h, int b, int kheight, int kwidth, diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h index 6a89dbc803..791fb52391 100644 --- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h +++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h @@ -602,6 +602,67 @@ inline void FullyConnected(const uint8* input_data, const Dims<4>& input_dims, } } +inline void ExperimentalShuffledFullyConnected( + const uint8* input_data, const Dims<4>& input_dims, + const uint8* shuffled_weights_data, const Dims<4>& weights_dims, + const int32* bias_data, const Dims<4>& bias_dims, int32 output_multiplier, + int output_shift, int32 output_activation_min, int32 output_activation_max, + int16* output_data, const Dims<4>& output_dims, + gemmlowp::GemmContext* gemm_context) { + (void)gemm_context; // only used in optimized code. + TFLITE_DCHECK_LE(output_activation_min, output_activation_max); + // TODO(benoitjacob): This really should be: + // const int batches = ArraySize(output_dims, 1); + // but the current --variable_batch hack consists in overwriting the 3rd + // dimension with the runtime batch size, as we don't keep track for each + // array of which dimension is the batch dimension in it. + const int batches = ArraySize(output_dims, 1) * ArraySize(output_dims, 2) * + ArraySize(output_dims, 3); + const int output_depth = MatchingArraySize(weights_dims, 1, output_dims, 0); + const int accum_depth = ArraySize(weights_dims, 0); + TFLITE_DCHECK(IsPackedWithoutStrides(input_dims)); + TFLITE_DCHECK(IsPackedWithoutStrides(weights_dims)); + // The experimental shuffling is an optimization for matrix*vector product. + // We aren't interested in supporting non-matrix*vector-product cases, i.e. + // batches>1. + TFLITE_DCHECK_EQ(batches, 1); + // Shuffled weights have had their sign bit (0x80) pre-flipped (xor'd) + // so that just reinterpreting them as int8 values is equivalent to + // subtracting 128 from them, thus implementing for free the subtraction of + // the zero_point value 128. + const int8* shuffled_weights_ptr = + reinterpret_cast(shuffled_weights_data); + for (int c = 0; c < output_depth; c += 4) { + // Internal accumulation. + // Initialize accumulator with the bias-value. + int32 accum[4] = {0}; + // Accumulation loop. + for (int d = 0; d < accum_depth; d += 16) { + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 16; j++) { + int8 input_val = input_data[d + j] - 128; + int8 weights_val = *shuffled_weights_ptr++; + accum[i] += weights_val * input_val; + } + } + } + for (int i = 0; i < 4; i++) { + // Add bias value + int acc = accum[i] + bias_data[c + i]; + // Down-scale the final int32 accumulator to the scale used by our + // (16-bit, typically 3 integer bits) fixed-point format. The quantized + // multiplier and shift here have been pre-computed offline + // (e.g. by toco). + acc = + MultiplyByQuantizedMultiplier(acc, output_multiplier, -output_shift); + // Saturate, cast to int16, and store to output array. + acc = std::max(acc, output_activation_min); + acc = std::min(acc, output_activation_max); + output_data[c + i] = acc; + } + } +} + // legacy, for compatibility with old checked-in code template void FullyConnected(const uint8* input_data, const Dims<4>& input_dims, diff --git a/tensorflow/contrib/lite/toco/BUILD b/tensorflow/contrib/lite/toco/BUILD index 4c8652d62e..5b86e4e5ae 100644 --- a/tensorflow/contrib/lite/toco/BUILD +++ b/tensorflow/contrib/lite/toco/BUILD @@ -219,6 +219,7 @@ cc_library( "graph_transformations/drop_fake_quant.cc", "graph_transformations/drop_im2col_arrays.cc", "graph_transformations/ensure_bias_vectors.cc", + "graph_transformations/experimental_shuffle_fc_weights.cc", "graph_transformations/fuse_activation_functions.cc", "graph_transformations/fuse_binary_into_following_affine.cc", "graph_transformations/fuse_binary_into_preceding_affine.cc", diff --git a/tensorflow/contrib/lite/toco/graph_transformations/experimental_shuffle_fc_weights.cc b/tensorflow/contrib/lite/toco/graph_transformations/experimental_shuffle_fc_weights.cc new file mode 100644 index 0000000000..f098981a5c --- /dev/null +++ b/tensorflow/contrib/lite/toco/graph_transformations/experimental_shuffle_fc_weights.cc @@ -0,0 +1,135 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include +#include +#include + +#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h" +#include "tensorflow/contrib/lite/toco/model.h" +#include "tensorflow/contrib/lite/toco/tooling_util.h" +#include "tensorflow/core/platform/logging.h" + +namespace toco { + +bool ExperimentalShuffleFCWeights::Run(Model* model, std::size_t op_index) { + Operator* op = model->operators[op_index].get(); + if (op->type != OperatorType::kFullyConnected) { + return false; + } + FullyConnectedOperator* fc_op = static_cast(op); + // Exit if this FC op already has shuffled weights + if (fc_op->experimental_shuffled_weights) { + return false; + } + const Array& input_array = model->GetArray(fc_op->inputs[0]); + const string& weights_name = fc_op->inputs[1]; + Array& weights_array = model->GetArray(weights_name); + const Array& output_array = model->GetArray(fc_op->outputs[0]); + // Exit if this FC op isn't quantized with uint8 inputs and int16 outputs, + // the only case where we are currently interested in providing a fast path + // with shuffled weights. + if (input_array.data_type != ArrayDataType::kUint8 || + weights_array.data_type != ArrayDataType::kUint8 || + output_array.data_type != ArrayDataType::kInt16 || + !input_array.quantization_params || !weights_array.quantization_params || + !output_array.quantization_params) { + return false; + } + // Exit if the shapes aren't known + if (!input_array.has_shape() || !weights_array.has_shape()) { + return false; + } + // Exit if, based on the known shapes, this FC op is not a GEMV. + // The shuffling of FC weights is only useful to enable fast GEMV paths. + const Shape& input_shape = input_array.shape(); + for (int i = 0; i < input_shape.dimensions_count() - 1; i++) { + if (input_shape.dims(i) != 1) { + // The input activations, shaped as a matrix, have multiple columns. + // This FC op isn't a matrix*vector multiplication. + AddMessageF( + "Not applying experimental shuffling to the weights of %s because " + "it's not a matrix*vector product", + LogName(*op)); + return false; + } + } + // Exit if the weights shape isn't an integral multiple of the shuffled + // block shape, 4x16. We don't want to have to write code dealing with + // odd sizes, that would go un-exercised at the moment as the models + // for which we need this shuffling have shapes that are multiples of that + // 4x16 block size. In fact, much of the rationale for this shuffling is + // to avoid cache aliasin issue with large power-of-two depths, with our + // models motivating this shuffling having FC weights shapes like + // 4096x2048. Thus, if some model doesn't get the shuffling because of that + // size requirement, that might be just fine --- that model might just not + // suffer from that cache aliasing issue that we have with large powers of + // two. + const Shape& weights_shape = weights_array.shape(); + if (weights_shape.dimensions_count() != 2) { + return false; + } + const int rows = weights_shape.dims(0); + const int cols = weights_shape.dims(1); + if (rows % 4 || cols % 16) { + AddMessageF( + "Not applying experimental shuffling to the weights of %s because its " + "shape isn't a multiple of the shuffling block shape, 4x16", + LogName(*op)); + return false; + } + // Exit if the weights aren't already a constant array. + if (!weights_array.buffer) { + return false; + } + // Exit if the weights are used by more than one op. + if (CountOpsWithInput(*model, weights_name) != 1) { + AddMessageF( + "Not applying experimental shuffling to the weights of %s because that " + "array is consumed by other operators", + LogName(*op)); + return false; + } + // Compute the shuffled weights + auto& weights_data = + weights_array.GetMutableBuffer().data; + CHECK_EQ(rows * cols, weights_data.size()); + std::vector shuffled_data(weights_data.size()); + uint8* shuffled_data_ptr = shuffled_data.data(); + for (int r = 0; r < rows; r += 4) { + for (int c = 0; c < cols; c += 16) { + for (int i = 0; i < 4; i++) { + const uint8* src_data_ptr = weights_data.data() + (r + i) * cols + c; + for (int j = 0; j < 16; j++) { + uint8 src_val = *src_data_ptr++; + // Flip the sign bit, so that the runtime will only need to + // reinterpret these uint8 values as int8, getting for free the + // subtraction of the zero_point value 128. + uint8 dst_val = src_val ^ 0x80; + *shuffled_data_ptr++ = dst_val; + } + } + } + } + CHECK_EQ(shuffled_data_ptr, shuffled_data.data() + rows * cols); + // Switch this FC op to using the shuffled weights. + weights_data = std::move(shuffled_data); + fc_op->experimental_shuffled_weights = true; + AddMessageF("Applied experimental shuffling to the weights of %s", + LogName(*op)); + return true; +} + +} // namespace toco diff --git a/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h b/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h index 384bd85b81..dbf029a853 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h +++ b/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h @@ -187,6 +187,7 @@ DECLARE_GRAPH_TRANSFORMATION(ResolveConstantGather) DECLARE_GRAPH_TRANSFORMATION(ResolveMultiplyByZero) DECLARE_GRAPH_TRANSFORMATION(Dequantize) DECLARE_GRAPH_TRANSFORMATION(UnpartitionEmbeddingLookup) +DECLARE_GRAPH_TRANSFORMATION(ExperimentalShuffleFCWeights) class ResolveReshapeAttributes : public GraphTransformation { public: diff --git a/tensorflow/contrib/lite/toco/graph_transformations/identify_lstm.cc b/tensorflow/contrib/lite/toco/graph_transformations/identify_lstm.cc index c363b93394..e9842524c8 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/identify_lstm.cc +++ b/tensorflow/contrib/lite/toco/graph_transformations/identify_lstm.cc @@ -306,6 +306,12 @@ bool IdentifyLstmCell::Run(Model* model, std::size_t op_index) { return false; } + if (static_cast(fully_connected) + ->experimental_shuffled_weights) { + // Not yet implemented: experimental shuffled weights in fused LSTM cell. + return false; + } + // Emplace a new LSTM cell operator auto* lstm_cell_op = new LstmCellOperator; lstm_cell_op->inputs.resize(LstmCellOperator::NUM_INPUTS); diff --git a/tensorflow/contrib/lite/toco/model.h b/tensorflow/contrib/lite/toco/model.h index 716a579d22..1c4c96ae70 100644 --- a/tensorflow/contrib/lite/toco/model.h +++ b/tensorflow/contrib/lite/toco/model.h @@ -425,6 +425,7 @@ struct SpaceToDepthOperator : Operator { // input activations as a matrix, followed by a MatMul node. struct FullyConnectedOperator : Operator { FullyConnectedOperator() : Operator(OperatorType::kFullyConnected) {} + bool experimental_shuffled_weights = false; }; // Dequantization operator, converting a quantized array of integers with -- GitLab From cd7ba4390360e1860cd57a6674a8423cf56b55bd Mon Sep 17 00:00:00 2001 From: Guangda Lai Date: Fri, 13 Apr 2018 10:02:25 -0700 Subject: [PATCH 320/791] Add debugging checks for setting cuda stream, so it will check fail if the stream is not set or set to a wrong one when running cudnn methods that conceptually require a stream. Also add missing cudnnSetStream()s for DoRnnForwardImpl() and DoRnnBackwardImpl(). Implementation details: 1. a current_cudnn_stream_ member is added which will be set in cudnnSetStream() 2. a different macro is used to wrap cudnn methods that require a stream in order to verify whether the provided stream is same as current_cudnn_stream_, and the program will check fail if not PiperOrigin-RevId: 192783913 --- tensorflow/stream_executor/cuda/cuda_dnn.cc | 215 ++++++++++++-------- tensorflow/stream_executor/cuda/cuda_dnn.h | 24 ++- 2 files changed, 151 insertions(+), 88 deletions(-) diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc index 1dc7f991b3..4a6b2bf5d7 100644 --- a/tensorflow/stream_executor/cuda/cuda_dnn.cc +++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc @@ -169,11 +169,34 @@ static port::ThreadPool* GetCudaThreadpool() { } \ } __name; +#define PERFTOOLS_GPUTOOLS_CUDNN_WRAP_WITH_CHECKED_STREAM(__name) \ + struct WrapperShim__##__name { \ + template \ + cudnnStatus_t operator()(CudnnSupport* dnn, Stream* s, Args... args) \ + SHARED_LOCKS_REQUIRED(dnn->dnn_handle_mutex_) { \ + CHECK_NOTNULL(s); \ + CHECK_EQ(s, dnn->GetCurrentDnnStream()) \ + << "Stream is not set correctly!"; \ + cuda::ScopedActivateExecutorContext sac{dnn->GetParentExecutor()}; \ + cudnnStatus_t retval = ::__name(args...); \ + return retval; \ + } \ + } __name; + +// Handles cudnnSetStream differently in order to add debug information. +struct WrapperShim__cudnnSetStream { + cudnnStatus_t operator()(CudnnSupport* dnn, Stream* stream, + cudnnHandle_t handle) + EXCLUSIVE_LOCKS_REQUIRED(dnn->dnn_handle_mutex_) { + dnn->SetCurrentDnnStream(stream); + cuda::ScopedActivateExecutorContext sac{dnn->GetParentExecutor()}; + cudnnStatus_t retval = ::cudnnSetStream(handle, AsCUDAStreamValue(stream)); + return retval; + } +} cudnnSetStream; + // clang-format off #define CUDNN_DNN_ROUTINE_EACH(__macro) \ - __macro(cudnnBatchNormalizationBackward) \ - __macro(cudnnBatchNormalizationForwardInference) \ - __macro(cudnnBatchNormalizationForwardTraining) \ __macro(cudnnGetConvolutionNdForwardOutputDim) \ __macro(cudnnGetConvolutionForwardAlgorithm) \ __macro(cudnnCreateTensorDescriptor) \ @@ -190,16 +213,25 @@ static port::ThreadPool* GetCudaThreadpool() { __macro(cudnnDestroyConvolutionDescriptor) \ __macro(cudnnCreate) \ __macro(cudnnDestroy) \ - __macro(cudnnSetStream) \ - __macro(cudnnActivationForward) \ - __macro(cudnnConvolutionForward) \ - __macro(cudnnConvolutionBackwardBias) \ __macro(cudnnGetConvolutionForwardWorkspaceSize) \ - __macro(cudnnTransformTensor) \ __macro(cudnnSetConvolutionNdDescriptor) \ __macro(cudnnSetTensor4dDescriptor) \ __macro(cudnnSetTensorNdDescriptor) \ - __macro(cudnnSetFilterNdDescriptor) \ + __macro(cudnnSetFilterNdDescriptor) + +// clang-format on +CUDNN_DNN_ROUTINE_EACH(PERFTOOLS_GPUTOOLS_CUDNN_WRAP) +#undef CUDNN_DNN_ROUTINE_EACH + +// clang-format off +#define CUDNN_DNN_ROUTINE_EACH_WITH_STREAM(__macro) \ + __macro(cudnnBatchNormalizationBackward) \ + __macro(cudnnBatchNormalizationForwardInference) \ + __macro(cudnnBatchNormalizationForwardTraining) \ + __macro(cudnnActivationForward) \ + __macro(cudnnConvolutionForward) \ + __macro(cudnnConvolutionBackwardBias) \ + __macro(cudnnTransformTensor) \ __macro(cudnnPoolingForward) \ __macro(cudnnPoolingBackward) \ __macro(cudnnLRNCrossChannelForward) \ @@ -207,9 +239,11 @@ static port::ThreadPool* GetCudaThreadpool() { __macro(cudnnAddTensor) \ __macro(cudnnConvolutionBackwardData) \ __macro(cudnnConvolutionBackwardFilter) -// clang-format on -CUDNN_DNN_ROUTINE_EACH(PERFTOOLS_GPUTOOLS_CUDNN_WRAP) +// clang-format on +CUDNN_DNN_ROUTINE_EACH_WITH_STREAM( + PERFTOOLS_GPUTOOLS_CUDNN_WRAP_WITH_CHECKED_STREAM) +#undef CUDNN_DNN_ROUTINE_EACH_WITH_STREAM // APIs available after R3: #if CUDNN_VERSION >= 3000 @@ -225,14 +259,15 @@ CUDNN_DNN_ROUTINE_EACH_AFTER_R3(PERFTOOLS_GPUTOOLS_CUDNN_WRAP) // APIs in R3 but not in R5 // clang-format off #if CUDNN_VERSION >= 3000 && CUDNN_VERSION < 5000 -#define CUDNN_DNN_ROUTINE_EACH_R3(__macro) \ +#define CUDNN_DNN_ROUTINE_EACH_R3_WITH_STREAM(__macro) \ __macro(cudnnAddTensor_v3) \ __macro(cudnnConvolutionBackwardData_v3) \ __macro(cudnnConvolutionBackwardFilter_v3) // clang-format on -CUDNN_DNN_ROUTINE_EACH_R3(PERFTOOLS_GPUTOOLS_CUDNN_WRAP) -#undef CUDNN_DNN_ROUTINE_EACH_R3 +CUDNN_DNN_ROUTINE_EACH_R3_WITH_STREAM( + PERFTOOLS_GPUTOOLS_CUDNN_WRAP_WITH_CHECKED_STREAM) +#undef CUDNN_DNN_ROUTINE_EACH_R3_WITH_STREAM #endif // APIs in R5 @@ -254,29 +289,44 @@ CUDNN_DNN_ROUTINE_EACH_R3(PERFTOOLS_GPUTOOLS_CUDNN_WRAP) __macro(cudnnGetRNNTrainingReserveSize) \ __macro(cudnnGetRNNLinLayerMatrixParams) \ __macro(cudnnGetRNNLinLayerBiasParams) \ - __macro(cudnnRNNForwardInference) \ - __macro(cudnnRNNForwardTraining) \ - __macro(cudnnRNNBackwardData) \ - __macro(cudnnRNNBackwardWeights) \ __macro(cudnnSetRNNDescriptor) \ __macro(cudnnGetFilterNdDescriptor) // clang-format on - CUDNN_DNN_ROUTINE_EACH_R5(PERFTOOLS_GPUTOOLS_CUDNN_WRAP) #undef CUDNN_DNN_ROUTINE_EACH_R5 + +// clang-format off +#define CUDNN_DNN_ROUTINE_EACH_R5_WITH_STREAM(__macro) \ + __macro(cudnnRNNForwardInference) \ + __macro(cudnnRNNForwardTraining) \ + __macro(cudnnRNNBackwardData) \ + __macro(cudnnRNNBackwardWeights) + +// clang-format on +CUDNN_DNN_ROUTINE_EACH_R5_WITH_STREAM( + PERFTOOLS_GPUTOOLS_CUDNN_WRAP_WITH_CHECKED_STREAM) +#undef CUDNN_DNN_ROUTINE_EACH_R5_WITH_STREAM #endif // APIs in R6 // clang-format off #if CUDNN_VERSION >= 6000 #define CUDNN_DNN_ROUTINE_EACH_R6(__macro) \ - __macro(cudnnConvolutionBiasActivationForward) \ __macro(cudnnSetRNNDescriptor_v6) // clang-format on CUDNN_DNN_ROUTINE_EACH_R6(PERFTOOLS_GPUTOOLS_CUDNN_WRAP) #undef CUDNN_DNN_ROUTINE_EACH_R6 + +// clang-format off +#define CUDNN_DNN_ROUTINE_EACH_R6_WITH_STREAM(__macro) \ + __macro(cudnnConvolutionBiasActivationForward) + +// clang-format on +CUDNN_DNN_ROUTINE_EACH_R6_WITH_STREAM( + PERFTOOLS_GPUTOOLS_CUDNN_WRAP_WITH_CHECKED_STREAM) +#undef CUDNN_DNN_ROUTINE_EACH_R6_WITH_STREAM #endif // APIs in R7 @@ -291,8 +341,6 @@ CUDNN_DNN_ROUTINE_EACH_R7(PERFTOOLS_GPUTOOLS_CUDNN_WRAP) #undef CUDNN_DNN_ROUTINE_EACH_R7 #endif -#undef CUDNN_DNN_ROUTINE_EACH - } // namespace wrap namespace { @@ -419,7 +467,7 @@ port::Status GetLoadedCudnnVersion(CudnnVersion* version) { } // namespace CudnnSupport::CudnnSupport(CUDAExecutor* parent) - : parent_(parent), dnn_handle_(nullptr) {} + : parent_(parent), dnn_handle_(nullptr), current_dnn_stream_(nullptr) {} CudnnSupport::~CudnnSupport() { auto status = wrap::cudnnDestroy(parent_, ToHandle(dnn_handle_)); @@ -1660,6 +1708,12 @@ bool CudnnSupport::DoRnnForwardImpl( // check params size mutex_lock lock{dnn_handle_mutex_}; + auto set_stream_status = + wrap::cudnnSetStream(this, stream, ToHandle(dnn_handle_)); + if (set_stream_status != CUDNN_STATUS_SUCCESS) { + LOG(FATAL) << "failed to set stream for cudnn handle: " + << ToString(set_stream_status); + } if (!CheckRNNParameterSize(parent_, ToHandle(dnn_handle_), rnn_desc, input_desc)) { @@ -1720,7 +1774,7 @@ bool CudnnSupport::DoRnnForwardImpl( cudnnStatus_t status; if (!is_training) { status = wrap::cudnnRNNForwardInference( - parent_, ToHandle(dnn_handle_) /*handle*/, + this, stream, ToHandle(dnn_handle_) /*handle*/, rnn_desc.handle() /*rnnDesc*/, model_dims.seq_length /*seqLength*/, input_desc.handles() /*xDesc*/, input_data.opaque() /*x*/, input_h_desc.handle() /*hxDesc*/, input_h_data.opaque() /*hx*/, @@ -1733,7 +1787,7 @@ bool CudnnSupport::DoRnnForwardImpl( workspace.size() /*workSpaceSizeInBytes*/); } else { status = wrap::cudnnRNNForwardTraining( - parent_, ToHandle(dnn_handle_) /*handle*/, + this, stream, ToHandle(dnn_handle_) /*handle*/, rnn_desc.handle() /*rnnDesc*/, model_dims.seq_length /*seqLength*/, input_desc.handles() /*xDesc*/, input_data.opaque() /*x*/, input_h_desc.handle() /*hxDesc*/, input_h_data.opaque() /*hx*/, @@ -1810,6 +1864,12 @@ bool CudnnSupport::DoRnnBackwardImpl( // check params size mutex_lock lock{dnn_handle_mutex_}; + auto set_stream_status = + wrap::cudnnSetStream(this, stream, ToHandle(dnn_handle_)); + if (set_stream_status != CUDNN_STATUS_SUCCESS) { + LOG(FATAL) << "failed to set stream for cudnn handle: " + << ToString(set_stream_status); + } if (!CheckRNNParameterSize(parent_, ToHandle(dnn_handle_), rnn_desc, input_desc)) { @@ -1841,10 +1901,11 @@ bool CudnnSupport::DoRnnBackwardImpl( } // make the backward data call cudnnStatus_t status = wrap::cudnnRNNBackwardData( - parent_, ToHandle(dnn_handle_) /*handle*/, rnn_desc.handle() /*rnnDesc*/, - model_dims.seq_length /*seqLength*/, output_desc.handles() /*yDesc*/, - output_data.opaque() /*y*/, output_desc.handles() /*dyDesc*/, - output_backprop_data.opaque() /*dy*/, output_h_desc.handle() /*dhyDesc*/, + this, stream, ToHandle(dnn_handle_) /*handle*/, + rnn_desc.handle() /*rnnDesc*/, model_dims.seq_length /*seqLength*/, + output_desc.handles() /*yDesc*/, output_data.opaque() /*y*/, + output_desc.handles() /*dyDesc*/, output_backprop_data.opaque() /*dy*/, + output_h_desc.handle() /*dhyDesc*/, output_h_backprop_data.opaque() /*dhy*/, output_c_desc.handle() /*dcyDesc*/, output_c_backprop_data.opaque() /*dcy*/, @@ -1873,7 +1934,7 @@ bool CudnnSupport::DoRnnBackwardImpl( stream->ThenMemZero(params_backprop_data, params_backprop_data->size()); // make the backward weight call status = wrap::cudnnRNNBackwardWeights( - parent_, ToHandle(dnn_handle_) /*handle*/, + this, stream, ToHandle(dnn_handle_) /*handle*/, rnn_desc.handle() /*rnnDesc*/, model_dims.seq_length /*seqLength*/, input_desc.handles() /*xDesc*/, input_data.opaque() /*x*/, input_h_desc.handle() /*hxDesc*/, input_h_data.opaque() /*hx*/, @@ -2517,8 +2578,7 @@ bool CudnnSupport::DoConvolveImpl( GetConvComputeType()}; mutex_lock lock{dnn_handle_mutex_}; - auto status = wrap::cudnnSetStream(parent_, ToHandle(dnn_handle_), - AsCUDAStreamValue(stream)); + auto status = wrap::cudnnSetStream(this, stream, ToHandle(dnn_handle_)); if (status != CUDNN_STATUS_SUCCESS) { LOG(FATAL) << "failed to set stream for cudnn handle: " << ToString(status); } @@ -2668,7 +2728,7 @@ bool CudnnSupport::DoConvolveImpl( } } status = wrap::cudnnConvolutionForward( - parent_, ToHandle(dnn_handle_), + this, stream, ToHandle(dnn_handle_), /*alpha=*/alpha, /*srcDesc=*/input_nd.handle(), /*srcData=*/input_data.opaque(), /*filterDesc=*/filter.handle(), /*filterData=*/filter_data.opaque(), /*convDesc=*/conv.handle(), @@ -2737,8 +2797,7 @@ bool CudnnSupport::DoFusedConvolveImpl( static_cast(cudnn_compute_type)}; mutex_lock lock{dnn_handle_mutex_}; - auto status = wrap::cudnnSetStream(parent_, ToHandle(dnn_handle_), - AsCUDAStreamValue(stream)); + auto status = wrap::cudnnSetStream(this, stream, ToHandle(dnn_handle_)); CHECK(status == CUDNN_STATUS_SUCCESS) << "failed to set stream for cudnn handle: " << ToString(status); @@ -2804,7 +2863,7 @@ bool CudnnSupport::DoFusedConvolveImpl( << "\noutput_data->opaque() = " << output_data->opaque(); status = wrap::cudnnConvolutionBiasActivationForward( - parent_, ToHandle(dnn_handle_), /*alpha1=*/&conv_input_scale, + this, stream, ToHandle(dnn_handle_), /*alpha1=*/&conv_input_scale, /*srcDesc=*/conv_input_nd.handle(), /*srcData=*/conv_input_data.opaque(), /*filterDesc=*/filter.handle(), /*filterData=*/filter_data.opaque(), /*convDesc=*/conv.handle(), algo, /*workSpace=*/scratch.opaque(), @@ -3009,8 +3068,7 @@ bool CudnnSupport::DoBatchNormalizationForwardImpl( bool is_training, std::function&()> var_to_inv_var, std::function inv_var_to_var) { mutex_lock lock{dnn_handle_mutex_}; - auto status = wrap::cudnnSetStream(parent_, ToHandle(dnn_handle_), - AsCUDAStreamValue(stream)); + auto status = wrap::cudnnSetStream(this, stream, ToHandle(dnn_handle_)); if (status != CUDNN_STATUS_SUCCESS) { LOG(ERROR) << "failed to set stream for cudnn handle: " << ToString(status); return false; @@ -3046,7 +3104,7 @@ bool CudnnSupport::DoBatchNormalizationForwardImpl( } status = wrap::cudnnBatchNormalizationForwardTraining( - parent_, ToHandle(dnn_handle_), mode, &one, &zero, + this, stream, ToHandle(dnn_handle_), mode, &one, &zero, x_descriptor.handle(), x.opaque(), x_descriptor.handle(), y->opaque(), scale_offset_descriptor.handle(), scale.opaque(), offset.opaque(), 1.0, batch_mean_opaque, batch_var_opaque, epsilon, saved_mean->opaque(), @@ -3063,7 +3121,7 @@ bool CudnnSupport::DoBatchNormalizationForwardImpl( const void* maybe_inv_var = estimated_variance.opaque(); #endif status = wrap::cudnnBatchNormalizationForwardInference( - parent_, ToHandle(dnn_handle_), mode, &one, &zero, + this, stream, ToHandle(dnn_handle_), mode, &one, &zero, x_descriptor.handle(), x.opaque(), x_descriptor.handle(), y->opaque(), scale_offset_descriptor.handle(), scale.opaque(), offset.opaque(), estimated_mean.opaque(), maybe_inv_var, epsilon); @@ -3114,8 +3172,7 @@ bool CudnnSupport::DoBatchNormalizationBackwardImpl( DeviceMemory* x_backprop, DeviceMemory* scale_backprop, DeviceMemory* offset_backprop) { mutex_lock lock{dnn_handle_mutex_}; - auto status = wrap::cudnnSetStream(parent_, ToHandle(dnn_handle_), - AsCUDAStreamValue(stream)); + auto status = wrap::cudnnSetStream(this, stream, ToHandle(dnn_handle_)); if (status != CUDNN_STATUS_SUCCESS) { LOG(ERROR) << "failed to set stream for cudnn handle: " << ToString(status); return false; @@ -3136,7 +3193,7 @@ bool CudnnSupport::DoBatchNormalizationBackwardImpl( float zero = 0.0; status = wrap::cudnnBatchNormalizationBackward( - parent_, ToHandle(dnn_handle_), mode, &one, &zero, &one, &zero, + this, stream, ToHandle(dnn_handle_), mode, &one, &zero, &one, &zero, x_descriptor.handle(), x.opaque(), x_descriptor.handle(), y_backprop.opaque(), x_descriptor.handle(), x_backprop->opaque(), scale_offset_descriptor.handle(), scale.opaque(), @@ -3326,7 +3383,7 @@ DeviceMemory CudnnSupport::MaybeTransformLayout( float alpha = 1.0f; float beta = 0.0f; auto status = wrap::cudnnTransformTensor( - parent_, ToHandle(dnn_handle_), &alpha, orig_out_back_nd.handle(), + this, stream, ToHandle(dnn_handle_), &alpha, orig_out_back_nd.handle(), backward_output_data.opaque(), &beta, transformed_out_back_nd.handle(), (*transform_scratch)->mutable_device_memory()->opaque()); @@ -3345,8 +3402,7 @@ bool CudnnSupport::DoTransformTensor(Stream* stream, dnn::DataType output_type, float scale, DeviceMemoryBase* output_data) { mutex_lock lock{dnn_handle_mutex_}; - cudnnStatus_t status = wrap::cudnnSetStream(parent_, ToHandle(dnn_handle_), - AsCUDAStreamValue(stream)); + auto status = wrap::cudnnSetStream(this, stream, ToHandle(dnn_handle_)); if (status != CUDNN_STATUS_SUCCESS) { LOG(FATAL) << "failed to set stream for cudnn handle: " << ToString(status); } @@ -3357,7 +3413,7 @@ bool CudnnSupport::DoTransformTensor(Stream* stream, ScopedTensorDescriptor output_tensor_desc( parent_, output_desc, ToCudnnDataType(output_type, output_desc.layout())); status = wrap::cudnnTransformTensor( - parent_, ToHandle(dnn_handle_), &scale, input_tensor_desc.handle(), + this, stream, ToHandle(dnn_handle_), &scale, input_tensor_desc.handle(), input_data.opaque(), &beta, output_tensor_desc.handle(), output_data->opaque()); if (status != CUDNN_STATUS_SUCCESS) { @@ -3384,8 +3440,7 @@ bool CudnnSupport::DoConvolveBackwardDataImpl( const dnn::AlgorithmConfig& algorithm_config, dnn::ProfileResult* output_profile_result) { mutex_lock lock{dnn_handle_mutex_}; - auto status = wrap::cudnnSetStream(parent_, ToHandle(dnn_handle_), - AsCUDAStreamValue(stream)); + auto status = wrap::cudnnSetStream(this, stream, ToHandle(dnn_handle_)); if (status != CUDNN_STATUS_SUCCESS) { LOG(FATAL) << "failed to set stream for cudnn handle: " << ToString(status); } @@ -3554,7 +3609,7 @@ bool CudnnSupport::DoConvolveBackwardDataImpl( #else status = wrap::cudnnConvolutionBackwardData_v3( #endif - parent_, ToHandle(dnn_handle_), + this, stream, ToHandle(dnn_handle_), /*alpha=*/alpha, /*filterDesc=*/filter.handle(), /*filterData=*/filter_data.opaque(), @@ -3655,8 +3710,7 @@ bool CudnnSupport::DoConvolveBackwardFilterImpl( const dnn::AlgorithmConfig& algorithm_config, dnn::ProfileResult* output_profile_result) { mutex_lock lock{dnn_handle_mutex_}; - auto status = wrap::cudnnSetStream(parent_, ToHandle(dnn_handle_), - AsCUDAStreamValue(stream)); + auto status = wrap::cudnnSetStream(this, stream, ToHandle(dnn_handle_)); if (status != CUDNN_STATUS_SUCCESS) { LOG(FATAL) << "failed to set stream for cudnn handle: " << ToString(status); } @@ -3826,7 +3880,7 @@ bool CudnnSupport::DoConvolveBackwardFilterImpl( #else status = wrap::cudnnConvolutionBackwardFilter_v3( #endif - parent_, ToHandle(dnn_handle_), /*alpha=*/alpha, + this, stream, ToHandle(dnn_handle_), /*alpha=*/alpha, /*srcDesc=*/input_nd.handle(), /*srcData=*/input_data.opaque(), /*diffDesc=*/out_back_nd.handle(), @@ -3922,8 +3976,7 @@ bool CudnnSupport::DoConvolveBackwardBiasImpl( const dnn::BatchDescriptor& bias_descriptor, DeviceMemory* backward_bias_data) { mutex_lock lock{dnn_handle_mutex_}; - auto status = wrap::cudnnSetStream(parent_, ToHandle(dnn_handle_), - AsCUDAStreamValue(stream)); + auto status = wrap::cudnnSetStream(this, stream, ToHandle(dnn_handle_)); if (status != CUDNN_STATUS_SUCCESS) { LOG(FATAL) << "failed to set stream for cudnn handle: " << ToString(status); } @@ -3938,7 +3991,7 @@ bool CudnnSupport::DoConvolveBackwardBiasImpl( float beta = 0.0; status = wrap::cudnnConvolutionBackwardBias( - parent_, ToHandle(dnn_handle_), &alpha, input_nd.handle(), + this, stream, ToHandle(dnn_handle_), &alpha, input_nd.handle(), input_data.opaque(), &beta, bias_nd.handle(), backward_bias_data->opaque()); if (status != CUDNN_STATUS_SUCCESS) { @@ -4143,8 +4196,7 @@ bool CudnnSupport::DoBiasAdd(Stream* stream, } mutex_lock lock{dnn_handle_mutex_}; - auto status = wrap::cudnnSetStream(parent_, ToHandle(dnn_handle_), - AsCUDAStreamValue(stream)); + auto status = wrap::cudnnSetStream(this, stream, ToHandle(dnn_handle_)); if (status != CUDNN_STATUS_SUCCESS) { LOG(ERROR) << "failed to set stream for cudnn handle: " << ToString(status); return false; @@ -4158,7 +4210,7 @@ bool CudnnSupport::DoBiasAdd(Stream* stream, #else status = wrap::cudnnAddTensor_v3( #endif - parent_, ToHandle(dnn_handle_), &alpha, bias_descriptor.handle(), + this, stream, ToHandle(dnn_handle_), &alpha, bias_descriptor.handle(), biases.opaque(), &beta, input_descriptor.handle(), output_data->opaque()); if (status != CUDNN_STATUS_SUCCESS) { @@ -4176,8 +4228,7 @@ bool CudnnSupport::DoActivate(Stream* stream, DeviceMemory* output_data, uint64 options) { mutex_lock lock{dnn_handle_mutex_}; - auto status = wrap::cudnnSetStream(parent_, ToHandle(dnn_handle_), - AsCUDAStreamValue(stream)); + auto status = wrap::cudnnSetStream(this, stream, ToHandle(dnn_handle_)); if (status != CUDNN_STATUS_SUCCESS) { LOG(ERROR) << "failed to set stream for cudnn handle: " << ToString(status); return false; @@ -4221,7 +4272,7 @@ bool CudnnSupport::DoActivate(Stream* stream, // Beta is the output scaling factor. float beta = 0.0; status = wrap::cudnnActivationForward( - parent_, ToHandle(dnn_handle_), + this, stream, ToHandle(dnn_handle_), #if CUDNN_VERSION >= 5000 activation_desc.handle(), #else @@ -4245,8 +4296,7 @@ bool CudnnSupport::DoPoolForward( const dnn::BatchDescriptor& output_dimensions, DeviceMemory* output_data) { mutex_lock lock{dnn_handle_mutex_}; - auto status = wrap::cudnnSetStream(parent_, ToHandle(dnn_handle_), - AsCUDAStreamValue(stream)); + auto status = wrap::cudnnSetStream(this, stream, ToHandle(dnn_handle_)); if (status != CUDNN_STATUS_SUCCESS) { LOG(ERROR) << "failed to set stream for cudnn handle: " << ToString(status); return false; @@ -4262,7 +4312,7 @@ bool CudnnSupport::DoPoolForward( CUDNN_DATA_DOUBLE}; ScopedPoolingDescriptor pooling_desc{parent_, pooling_dimensions}; status = wrap::cudnnPoolingForward( - parent_, ToHandle(dnn_handle_), pooling_desc.handle(), &alpha, + this, stream, ToHandle(dnn_handle_), pooling_desc.handle(), &alpha, src_desc.handle(), input_data.opaque(), &beta, dest_desc.handle(), output_data->opaque()); if (status != CUDNN_STATUS_SUCCESS) { @@ -4280,8 +4330,7 @@ bool CudnnSupport::DoPoolForward( const dnn::BatchDescriptor& output_dimensions, DeviceMemory* output_data) { mutex_lock lock{dnn_handle_mutex_}; - auto status = wrap::cudnnSetStream(parent_, ToHandle(dnn_handle_), - AsCUDAStreamValue(stream)); + auto status = wrap::cudnnSetStream(this, stream, ToHandle(dnn_handle_)); if (status != CUDNN_STATUS_SUCCESS) { LOG(ERROR) << "failed to set stream for cudnn handle: " << ToString(status); return false; @@ -4297,7 +4346,7 @@ bool CudnnSupport::DoPoolForward( CUDNN_DATA_FLOAT}; ScopedPoolingDescriptor pooling_desc{parent_, pooling_dimensions}; status = wrap::cudnnPoolingForward( - parent_, ToHandle(dnn_handle_), pooling_desc.handle(), &alpha, + this, stream, ToHandle(dnn_handle_), pooling_desc.handle(), &alpha, src_desc.handle(), input_data.opaque(), &beta, dest_desc.handle(), output_data->opaque()); if (status != CUDNN_STATUS_SUCCESS) { @@ -4315,8 +4364,7 @@ bool CudnnSupport::DoPoolForward( const dnn::BatchDescriptor& output_dimensions, DeviceMemory* output_data) { mutex_lock lock{dnn_handle_mutex_}; - auto status = wrap::cudnnSetStream(parent_, ToHandle(dnn_handle_), - AsCUDAStreamValue(stream)); + auto status = wrap::cudnnSetStream(this, stream, ToHandle(dnn_handle_)); if (status != CUDNN_STATUS_SUCCESS) { LOG(ERROR) << "failed to set stream for cudnn handle: " << ToString(status); return false; @@ -4331,7 +4379,7 @@ bool CudnnSupport::DoPoolForward( ScopedTensorDescriptor dest_desc{parent_, output_dimensions, CUDNN_DATA_HALF}; ScopedPoolingDescriptor pooling_desc{parent_, pooling_dimensions}; status = wrap::cudnnPoolingForward( - parent_, ToHandle(dnn_handle_), pooling_desc.handle(), &alpha, + this, stream, ToHandle(dnn_handle_), pooling_desc.handle(), &alpha, src_desc.handle(), input_data.opaque(), &beta, dest_desc.handle(), output_data->opaque()); if (status != CUDNN_STATUS_SUCCESS) { @@ -4351,8 +4399,7 @@ bool CudnnSupport::DoPoolBackward( const DeviceMemory& input_diff_data, DeviceMemory* output_diff_data) { mutex_lock lock{dnn_handle_mutex_}; - auto status = wrap::cudnnSetStream(parent_, ToHandle(dnn_handle_), - AsCUDAStreamValue(stream)); + auto status = wrap::cudnnSetStream(this, stream, ToHandle(dnn_handle_)); if (status != CUDNN_STATUS_SUCCESS) { LOG(ERROR) << "failed to set stream for cudnn handle: " << ToString(status); return false; @@ -4368,7 +4415,7 @@ bool CudnnSupport::DoPoolBackward( CUDNN_DATA_DOUBLE}; ScopedPoolingDescriptor pooling_desc{parent_, pooling_dimensions}; status = wrap::cudnnPoolingBackward( - parent_, ToHandle(dnn_handle_), pooling_desc.handle(), &alpha, + this, stream, ToHandle(dnn_handle_), pooling_desc.handle(), &alpha, dest_desc.handle(), output_data.opaque(), dest_desc.handle(), input_diff_data.opaque(), src_desc.handle(), input_data.opaque(), &beta, src_desc.handle(), output_diff_data->opaque()); @@ -4389,8 +4436,7 @@ bool CudnnSupport::DoPoolBackward( const DeviceMemory& input_diff_data, DeviceMemory* output_diff_data) { mutex_lock lock{dnn_handle_mutex_}; - auto status = wrap::cudnnSetStream(parent_, ToHandle(dnn_handle_), - AsCUDAStreamValue(stream)); + auto status = wrap::cudnnSetStream(this, stream, ToHandle(dnn_handle_)); if (status != CUDNN_STATUS_SUCCESS) { LOG(ERROR) << "failed to set stream for cudnn handle: " << ToString(status); return false; @@ -4406,7 +4452,7 @@ bool CudnnSupport::DoPoolBackward( CUDNN_DATA_FLOAT}; ScopedPoolingDescriptor pooling_desc{parent_, pooling_dimensions}; status = wrap::cudnnPoolingBackward( - parent_, ToHandle(dnn_handle_), pooling_desc.handle(), &alpha, + this, stream, ToHandle(dnn_handle_), pooling_desc.handle(), &alpha, dest_desc.handle(), output_data.opaque(), dest_desc.handle(), input_diff_data.opaque(), src_desc.handle(), input_data.opaque(), &beta, src_desc.handle(), output_diff_data->opaque()); @@ -4427,8 +4473,7 @@ bool CudnnSupport::DoPoolBackward( const DeviceMemory& input_diff_data, DeviceMemory* output_diff_data) { mutex_lock lock{dnn_handle_mutex_}; - auto status = wrap::cudnnSetStream(parent_, ToHandle(dnn_handle_), - AsCUDAStreamValue(stream)); + auto status = wrap::cudnnSetStream(this, stream, ToHandle(dnn_handle_)); if (status != CUDNN_STATUS_SUCCESS) { LOG(ERROR) << "failed to set stream for cudnn handle: " << ToString(status); return false; @@ -4443,7 +4488,7 @@ bool CudnnSupport::DoPoolBackward( ScopedTensorDescriptor dest_desc{parent_, output_dimensions, CUDNN_DATA_HALF}; ScopedPoolingDescriptor pooling_desc{parent_, pooling_dimensions}; status = wrap::cudnnPoolingBackward( - parent_, ToHandle(dnn_handle_), pooling_desc.handle(), &alpha, + this, stream, ToHandle(dnn_handle_), pooling_desc.handle(), &alpha, dest_desc.handle(), output_data.opaque(), dest_desc.handle(), input_diff_data.opaque(), src_desc.handle(), input_data.opaque(), &beta, src_desc.handle(), output_diff_data->opaque()); @@ -4478,8 +4523,7 @@ bool CudnnSupport::DoNormalizeWithDimensions( // Launch the normalization. mutex_lock lock{dnn_handle_mutex_}; - auto status = wrap::cudnnSetStream(parent_, ToHandle(dnn_handle_), - AsCUDAStreamValue(stream)); + auto status = wrap::cudnnSetStream(this, stream, ToHandle(dnn_handle_)); if (status != CUDNN_STATUS_SUCCESS) { LOG(ERROR) << "failed to set stream for cudnn handle: " << ToString(status); return false; @@ -4494,7 +4538,7 @@ bool CudnnSupport::DoNormalizeWithDimensions( float beta = 0.0f; status = wrap::cudnnLRNCrossChannelForward( - parent_, ToHandle(dnn_handle_), normalize.handle(), + this, stream, ToHandle(dnn_handle_), normalize.handle(), CUDNN_LRN_CROSS_CHANNEL_DIM1, &alpha, dims.handle(), input_data.opaque(), &beta, dims.handle(), output_data->opaque()); if (status != CUDNN_STATUS_SUCCESS) { @@ -4521,8 +4565,7 @@ bool CudnnSupport::DoNormalizeBackwardWithDimensions( } mutex_lock lock{dnn_handle_mutex_}; - auto status = wrap::cudnnSetStream(parent_, ToHandle(dnn_handle_), - AsCUDAStreamValue(stream)); + auto status = wrap::cudnnSetStream(this, stream, ToHandle(dnn_handle_)); if (status != CUDNN_STATUS_SUCCESS) { LOG(ERROR) << "failed to set stream for cudnn handle: " << ToString(status); return false; @@ -4535,7 +4578,7 @@ bool CudnnSupport::DoNormalizeBackwardWithDimensions( float beta = 0.0f; status = wrap::cudnnLRNCrossChannelBackward( - parent_, ToHandle(dnn_handle_), normalize.handle(), + this, stream, ToHandle(dnn_handle_), normalize.handle(), CUDNN_LRN_CROSS_CHANNEL_DIM1, &alpha, dims.handle(), normalized_data.opaque(), dims.handle(), normalized_variable_gradient.opaque(), dims.handle(), raw_data.opaque(), diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.h b/tensorflow/stream_executor/cuda/cuda_dnn.h index 0e5368aca8..7518b23757 100644 --- a/tensorflow/stream_executor/cuda/cuda_dnn.h +++ b/tensorflow/stream_executor/cuda/cuda_dnn.h @@ -625,10 +625,27 @@ class CudnnSupport : public dnn::DnnSupport { dnn::DataType output_type, float scale, DeviceMemoryBase* output_data) override; - private: - // Guards the enqueueing of DNN operations via the dnn_handle_ below. + const Stream* GetCurrentDnnStream() const + SHARED_LOCKS_REQUIRED(dnn_handle_mutex_) { + return current_dnn_stream_; + } + + void SetCurrentDnnStream(Stream* stream) + EXCLUSIVE_LOCKS_REQUIRED(dnn_handle_mutex_) { + current_dnn_stream_ = stream; + } + + CUDAExecutor* GetParentExecutor() { return parent_; } + + // Guards the enqueueing of DNN operations via the dnn_handle_ below, and + // access to current_dnn_stream_. + // + // This is a public member because we need to add thread safty annotations in + // the cudnn wrapper functions in the cc file, which need to access this + // mutex (the annotations require C++ permission checks). mutex dnn_handle_mutex_; + private: CUDAExecutor* parent_; // Parent executor object. Not owned. // cudnn library handle. cudnnHandle_t type is not present in this header to @@ -636,6 +653,9 @@ class CudnnSupport : public dnn::DnnSupport { // single cuda_dnn translation unit. void* dnn_handle_ GUARDED_BY(dnn_handle_mutex_); + // The current cudnn stream that is set by cudnnSetStream(). + Stream* current_dnn_stream_ GUARDED_BY(dnn_handle_mutex_); + // NOTE(keveman): Temporary data layout transformation until cuDNN supports // kBatchYXDepth for backward pass. This function allocates temporary memory, // lays out the source data into the temporary but in the kBatchDepthXY -- GitLab From 49f56ac87ee630cf4d15a161900e5a0bb631f563 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 10:07:10 -0700 Subject: [PATCH 321/791] Enable GCS remote cache in Windows Bazel Build PiperOrigin-RevId: 192784701 --- .../ci_build/windows/bazel/bazel_test_lib.sh | 7 +++++++ .../windows/cpu/pip/build_tf_windows.sh | 17 ++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh b/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh index d654b433e7..b2e16902d6 100644 --- a/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh +++ b/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh @@ -140,6 +140,13 @@ function run_configure_for_gpu_build { echo "" | ./configure } +function set_gcs_remote_cache_options { + echo "build --experimental_remote_spawn_cache" >> .bazelrc + echo "build --experimental_remote_platform_override='properties:{name:\"build\" value:\"windows-x64\"}'" >> .bazelrc + echo "build --remote_http_cache=https://storage.googleapis.com/$GCS_BUCKET_NAME" >> .bazelrc + echo "build --google_credentials=$GOOGLE_CLOUD_CREDENTIAL" >> .bazelrc +} + function create_python_test_dir() { rm -rf "$1" mkdir -p "$1" diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh index 5e9ae497e1..4657ff196b 100644 --- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh +++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh @@ -42,20 +42,27 @@ source "tensorflow/tools/ci_build/windows/bazel/common_env.sh" \ source "tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh" \ || { echo "Failed to source bazel_test_lib.sh" >&2; exit 1; } +# Recreate an empty bazelrc file under source root +rm -f .bazelrc +touch .bazelrc + skip_test=0 for ARG in "$@"; do if [[ "$ARG" == --skip_test ]]; then skip_test=1 + elif [[ "$ARG" == --enable_gcs_remote_cache ]]; then + set_gcs_remote_cache_options fi done -run_configure_for_cpu_build - # --define=override_eigen_strong_inline=true speeds up the compiling of conv_grad_ops_3d.cc and conv_ops_3d.cc # by 20 minutes. See https://github.com/tensorflow/tensorflow/issues/10521 -BUILD_OPTS="--define=override_eigen_strong_inline=true" -bazel build -c opt $BUILD_OPTS tensorflow/tools/pip_package:build_pip_package || exit $? +echo "build --define=override_eigen_strong_inline=true" >> .bazelrc + +run_configure_for_cpu_build + +bazel build -c opt tensorflow/tools/pip_package:build_pip_package || exit $? if [[ "$skip_test" == 1 ]]; then exit 0 @@ -73,7 +80,7 @@ reinstall_tensorflow_pip ${PIP_NAME} # Define no_tensorflow_py_deps=true so that every py_test has no deps anymore, # which will result testing system installed tensorflow -bazel test -c opt $BUILD_OPTS -k --test_output=errors \ +bazel test -c opt -k --test_output=errors \ --define=no_tensorflow_py_deps=true --test_lang_filters=py \ --test_tag_filters=-no_pip,-no_windows,-no_oss \ --build_tag_filters=-no_pip,-no_windows,-no_oss --build_tests_only \ -- GitLab From defc185d57233d5185c4d77c973d8e25256b1e73 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 10:27:11 -0700 Subject: [PATCH 322/791] DepthwiseConv Optimization Fixes PiperOrigin-RevId: 192787669 --- .../depthwiseconv_uint8_3x3_filter.h | 170 +++++++++--------- 1 file changed, 86 insertions(+), 84 deletions(-) diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h b/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h index cdcb166b2f..55e0d5c3aa 100644 --- a/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h +++ b/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h @@ -386,12 +386,13 @@ inline void DotProductAndStore2yStride1( } // A kernel that is optimized on the number of output cells in the x and y -// direction, and the stride. Assumes 3x3 filters of 16 depth. -template +// direction, and the stride. Assumes 3x3 filters of 8 depth. +template struct ConvKernel3x3FilterDepth8 {}; template <> -struct ConvKernel3x3FilterDepth8<8, 8, 1> { +struct ConvKernel3x3FilterDepth8<8, 8, 1, 1> { static inline void Run(const uint8* input_ptr, int input_depth, int32 input_offset, int input_row_size, const uint8* filter_ptr, int32 filter_offset, @@ -1642,7 +1643,7 @@ struct ConvKernel3x3FilterDepth8<8, 8, 1> { }; template <> -struct ConvKernel3x3FilterDepth8<4, 4, 1> { +struct ConvKernel3x3FilterDepth8<4, 4, 1, 1> { static inline void Run(const uint8* input_ptr, int input_depth, int32 input_offset, int input_row_size, const uint8* filter_ptr, int32 filter_offset, @@ -1957,7 +1958,7 @@ struct ConvKernel3x3FilterDepth8<4, 4, 1> { }; template <> -struct ConvKernel3x3FilterDepth8<4, 2, 1> { +struct ConvKernel3x3FilterDepth8<4, 2, 1, 1> { static inline void Run(const uint8* input_ptr, int input_depth, int32 input_offset, int input_row_size, const uint8* filter_ptr, int32 filter_offset, @@ -2123,7 +2124,7 @@ struct ConvKernel3x3FilterDepth8<4, 2, 1> { }; template <> -struct ConvKernel3x3FilterDepth8<4, 1, 1> { +struct ConvKernel3x3FilterDepth8<4, 1, 1, 1> { static inline void Run(const uint8* input_ptr, int input_depth, int32 input_offset, int input_row_size, const uint8* filter_ptr, int32 filter_offset, @@ -2235,7 +2236,7 @@ struct ConvKernel3x3FilterDepth8<4, 1, 1> { }; template <> -struct ConvKernel3x3FilterDepth8<2, 2, 1> { +struct ConvKernel3x3FilterDepth8<2, 2, 1, 1> { static inline void Run(const uint8* input_ptr, int input_depth, int32 input_offset, int input_row_size, const uint8* filter_ptr, int32 filter_offset, @@ -2373,7 +2374,7 @@ struct ConvKernel3x3FilterDepth8<2, 2, 1> { }; template <> -struct ConvKernel3x3FilterDepth8<2, 4, 1> { +struct ConvKernel3x3FilterDepth8<2, 4, 1, 1> { static inline void Run(const uint8* input_ptr, int input_depth, int32 input_offset, int input_row_size, const uint8* filter_ptr, int32 filter_offset, @@ -2554,7 +2555,7 @@ struct ConvKernel3x3FilterDepth8<2, 4, 1> { }; template <> -struct ConvKernel3x3FilterDepth8<1, 4, 1> { +struct ConvKernel3x3FilterDepth8<1, 4, 1, 1> { static inline void Run(const uint8* input_ptr, int input_depth, int32 input_offset, int input_row_size, const uint8* filter_ptr, int32 filter_offset, @@ -2669,7 +2670,7 @@ struct ConvKernel3x3FilterDepth8<1, 4, 1> { }; template <> -struct ConvKernel3x3FilterDepth8<2, 1, 1> { +struct ConvKernel3x3FilterDepth8<2, 1, 1, 1> { static inline void Run(const uint8* input_ptr, int input_depth, int32 input_offset, int input_row_size, const uint8* filter_ptr, int32 filter_offset, @@ -2746,7 +2747,7 @@ struct ConvKernel3x3FilterDepth8<2, 1, 1> { }; template <> -struct ConvKernel3x3FilterDepth8<4, 2, 2> { +struct ConvKernel3x3FilterDepth8<4, 2, 2, 2> { static inline void Run(const uint8* input_ptr, int input_depth, int32 input_offset, int input_row_size, const uint8* filter_ptr, int32 filter_offset, @@ -3063,7 +3064,7 @@ struct ConvKernel3x3FilterDepth8<4, 2, 2> { }; template <> -struct ConvKernel3x3FilterDepth8<4, 4, 2> { +struct ConvKernel3x3FilterDepth8<4, 4, 2, 2> { static inline void Run(const uint8* input_ptr, int input_depth, int32 input_offset, int input_row_size, const uint8* filter_ptr, int32 filter_offset, @@ -3073,13 +3074,13 @@ struct ConvKernel3x3FilterDepth8<4, 4, 2> { int32 output_activation_max, uint8* output_ptr, int output_depth, int output_width) { // Reuse 4x2 kernel twice. - ConvKernel3x3FilterDepth8<4, 2, 2>::Run( + ConvKernel3x3FilterDepth8<4, 2, 2, 2>::Run( input_ptr, input_depth, input_offset, input_row_size, filter_ptr, filter_offset, bias_ptr, output_offset, output_multiplier, output_shift, output_activation_min, output_activation_max, output_ptr, output_depth, output_width); - ConvKernel3x3FilterDepth8<4, 2, 2>::Run( + ConvKernel3x3FilterDepth8<4, 2, 2, 2>::Run( input_ptr + 4 * input_depth, input_depth, input_offset, input_row_size, filter_ptr, filter_offset, bias_ptr, output_offset, output_multiplier, output_shift, output_activation_min, output_activation_max, @@ -3088,7 +3089,7 @@ struct ConvKernel3x3FilterDepth8<4, 4, 2> { }; template <> -struct ConvKernel3x3FilterDepth8<4, 1, 2> { +struct ConvKernel3x3FilterDepth8<4, 1, 2, 2> { static inline void Run(const uint8* input_ptr, int input_depth, int32 input_offset, int input_row_size, const uint8* filter_ptr, int32 filter_offset, @@ -3243,7 +3244,7 @@ struct ConvKernel3x3FilterDepth8<4, 1, 2> { }; template <> -struct ConvKernel3x3FilterDepth8<2, 2, 2> { +struct ConvKernel3x3FilterDepth8<2, 2, 2, 2> { static inline void Run(const uint8* input_ptr, int input_depth, int32 input_offset, int input_row_size, const uint8* filter_ptr, int32 filter_offset, @@ -3433,7 +3434,7 @@ struct ConvKernel3x3FilterDepth8<2, 2, 2> { }; template <> -struct ConvKernel3x3FilterDepth8<2, 4, 2> { +struct ConvKernel3x3FilterDepth8<2, 4, 2, 2> { static inline void Run(const uint8* input_ptr, int input_depth, int32 input_offset, int input_row_size, const uint8* filter_ptr, int32 filter_offset, @@ -3443,13 +3444,13 @@ struct ConvKernel3x3FilterDepth8<2, 4, 2> { int32 output_activation_max, uint8* output_ptr, int output_depth, int output_width) { // Reuse 2x2 kernel twice. - ConvKernel3x3FilterDepth8<2, 2, 2>::Run( + ConvKernel3x3FilterDepth8<2, 2, 2, 2>::Run( input_ptr, input_depth, input_offset, input_row_size, filter_ptr, filter_offset, bias_ptr, output_offset, output_multiplier, output_shift, output_activation_min, output_activation_max, output_ptr, output_depth, output_width); - ConvKernel3x3FilterDepth8<2, 2, 2>::Run( + ConvKernel3x3FilterDepth8<2, 2, 2, 2>::Run( input_ptr + 4 * input_depth, input_depth, input_offset, input_row_size, filter_ptr, filter_offset, bias_ptr, output_offset, output_multiplier, output_shift, output_activation_min, output_activation_max, @@ -3458,7 +3459,7 @@ struct ConvKernel3x3FilterDepth8<2, 4, 2> { }; template <> -struct ConvKernel3x3FilterDepth8<2, 1, 2> { +struct ConvKernel3x3FilterDepth8<2, 1, 2, 2> { static inline void Run(const uint8* input_ptr, int input_depth, int32 input_offset, int input_row_size, const uint8* filter_ptr, int32 filter_offset, @@ -3551,7 +3552,7 @@ struct ConvKernel3x3FilterDepth8<2, 1, 2> { }; template <> -struct ConvKernel3x3FilterDepth8<1, 2, 2> { +struct ConvKernel3x3FilterDepth8<1, 2, 2, 2> { static inline void Run(const uint8* input_ptr, int input_depth, int32 input_offset, int input_row_size, const uint8* filter_ptr, int32 filter_offset, @@ -3643,7 +3644,7 @@ struct ConvKernel3x3FilterDepth8<1, 2, 2> { }; template <> -struct ConvKernel3x3FilterDepth8<1, 4, 2> { +struct ConvKernel3x3FilterDepth8<1, 4, 2, 2> { static inline void Run(const uint8* input_ptr, int input_depth, int32 input_offset, int input_row_size, const uint8* filter_ptr, int32 filter_offset, @@ -3798,8 +3799,8 @@ struct ConvKernel3x3FilterDepth8<1, 4, 2> { } }; -template <> -struct ConvKernel3x3FilterDepth8<1, 1> { +template +struct ConvKernel3x3FilterDepth8<1, 1, kFixedStrideWidth, kFixedStrideHeight> { static inline void Run(const uint8* input_ptr, int input_depth, int32 input_offset, int input_row_size, const uint8* filter_ptr, int32 filter_offset, @@ -3872,12 +3873,11 @@ inline void ShuffleInput(const uint8* input_ptr, int input_depth, } } -template +template struct ConvRow3x3FilterDepth8 {}; -template -struct ConvRow3x3FilterDepth8<1, kFixedStrideWidth> { +template +struct ConvRow3x3FilterDepth8<1, kFixedStrideWidth, kFixedStrideHeight> { static inline void Run(const uint8* input_data, int start_x, int start_y, int input_depth, int input_width, int input_height, int input_row_size, int32 input_offset, @@ -3899,11 +3899,11 @@ struct ConvRow3x3FilterDepth8<1, kFixedStrideWidth> { uint8* output_ptr = output_data; for (int depth = 0; depth <= output_depth - 8; depth += 8) { - ConvKernel3x3FilterDepth8<1, 4, kFixedStrideWidth>::Run( - input_ptr, input_depth, input_offset, input_row_size, filter_ptr, - filter_offset, bias_ptr, output_offset, output_multiplier, - output_shift, output_activation_min, output_activation_max, - output_ptr, output_depth, output_width); + ConvKernel3x3FilterDepth8<1, 4, kFixedStrideWidth, kFixedStrideHeight>:: + Run(input_ptr, input_depth, input_offset, input_row_size, + filter_ptr, filter_offset, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr, output_depth, output_width); input_ptr += 8; output_ptr += 8; @@ -3924,11 +3924,11 @@ struct ConvRow3x3FilterDepth8<1, kFixedStrideWidth> { uint8* output_ptr = output_data; for (int depth = 0; depth <= output_depth - 8; depth += 8) { - ConvKernel3x3FilterDepth8<1, 1>::Run( - input_ptr, input_depth, input_offset, input_row_size, filter_ptr, - filter_offset, bias_ptr, output_offset, output_multiplier, - output_shift, output_activation_min, output_activation_max, - output_ptr, output_depth, output_width); + ConvKernel3x3FilterDepth8<1, 1, kFixedStrideWidth, kFixedStrideHeight>:: + Run(input_ptr, input_depth, input_offset, input_row_size, + filter_ptr, filter_offset, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr, output_depth, output_width); input_ptr += 8; output_ptr += 8; @@ -3942,8 +3942,8 @@ struct ConvRow3x3FilterDepth8<1, kFixedStrideWidth> { } }; -template -struct ConvRow3x3FilterDepth8<2, kFixedStrideWidth> { +template +struct ConvRow3x3FilterDepth8<2, kFixedStrideWidth, kFixedStrideHeight> { static inline void Run(const uint8* input_data, int start_x, int start_y, int input_depth, int input_width, int input_height, int input_row_size, int32 input_offset, @@ -3965,11 +3965,11 @@ struct ConvRow3x3FilterDepth8<2, kFixedStrideWidth> { uint8* output_ptr = output_data; for (int depth = 0; depth <= output_depth - 8; depth += 8) { - ConvKernel3x3FilterDepth8<2, 4, kFixedStrideWidth>::Run( - input_ptr, input_depth, input_offset, input_row_size, filter_ptr, - filter_offset, bias_ptr, output_offset, output_multiplier, - output_shift, output_activation_min, output_activation_max, - output_ptr, output_depth, output_width); + ConvKernel3x3FilterDepth8<2, 4, kFixedStrideWidth, kFixedStrideHeight>:: + Run(input_ptr, input_depth, input_offset, input_row_size, + filter_ptr, filter_offset, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr, output_depth, output_width); input_ptr += 8; output_ptr += 8; @@ -3990,11 +3990,11 @@ struct ConvRow3x3FilterDepth8<2, kFixedStrideWidth> { uint8* output_ptr = output_data; for (int depth = 0; depth <= output_depth - 8; depth += 8) { - ConvKernel3x3FilterDepth8<2, 2, kFixedStrideWidth>::Run( - input_ptr, input_depth, input_offset, input_row_size, filter_ptr, - filter_offset, bias_ptr, output_offset, output_multiplier, - output_shift, output_activation_min, output_activation_max, - output_ptr, output_depth, output_width); + ConvKernel3x3FilterDepth8<2, 2, kFixedStrideWidth, kFixedStrideHeight>:: + Run(input_ptr, input_depth, input_offset, input_row_size, + filter_ptr, filter_offset, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr, output_depth, output_width); input_ptr += 8; output_ptr += 8; @@ -4015,11 +4015,11 @@ struct ConvRow3x3FilterDepth8<2, kFixedStrideWidth> { uint8* output_ptr = output_data; for (int depth = 0; depth <= output_depth - 8; depth += 8) { - ConvKernel3x3FilterDepth8<2, 1, kFixedStrideWidth>::Run( - input_ptr, input_depth, input_offset, input_row_size, filter_ptr, - filter_offset, bias_ptr, output_offset, output_multiplier, - output_shift, output_activation_min, output_activation_max, - output_ptr, output_depth, output_width); + ConvKernel3x3FilterDepth8<2, 1, kFixedStrideWidth, kFixedStrideHeight>:: + Run(input_ptr, input_depth, input_offset, input_row_size, + filter_ptr, filter_offset, bias_ptr, output_offset, + output_multiplier, output_shift, output_activation_min, + output_activation_max, output_ptr, output_depth, output_width); input_ptr += 8; output_ptr += 8; @@ -4034,7 +4034,7 @@ struct ConvRow3x3FilterDepth8<2, kFixedStrideWidth> { }; template <> -struct ConvRow3x3FilterDepth8<4, 1> { +struct ConvRow3x3FilterDepth8<4, 1, 1> { static inline void Run(const uint8* input_data, int start_x, int start_y, int input_depth, int input_width, int input_height, int input_row_size, int32 input_offset, @@ -4056,7 +4056,7 @@ struct ConvRow3x3FilterDepth8<4, 1> { uint8* output_ptr = output_data; for (int depth = 0; depth <= output_depth - 8; depth += 8) { - ConvKernel3x3FilterDepth8<4, 4, 1>::Run( + ConvKernel3x3FilterDepth8<4, 4, 1, 1>::Run( input_ptr, input_depth, input_offset, input_row_size, filter_ptr, filter_offset, bias_ptr, output_offset, output_multiplier, output_shift, output_activation_min, output_activation_max, @@ -4082,7 +4082,7 @@ struct ConvRow3x3FilterDepth8<4, 1> { uint8* output_ptr = output_data; for (int depth = 0; depth <= output_depth - 8; depth += 8) { - ConvKernel3x3FilterDepth8<4, 2, 1>::Run( + ConvKernel3x3FilterDepth8<4, 2, 1, 1>::Run( input_ptr, input_depth, input_offset, input_row_size, filter_ptr, filter_offset, bias_ptr, output_offset, output_multiplier, output_shift, output_activation_min, output_activation_max, @@ -4107,7 +4107,7 @@ struct ConvRow3x3FilterDepth8<4, 1> { uint8* output_ptr = output_data; for (int depth = 0; depth <= output_depth - 8; depth += 8) { - ConvKernel3x3FilterDepth8<4, 1, 1>::Run( + ConvKernel3x3FilterDepth8<4, 1, 1, 1>::Run( input_ptr, input_depth, input_offset, input_row_size, filter_ptr, filter_offset, bias_ptr, output_offset, output_multiplier, output_shift, output_activation_min, output_activation_max, @@ -4126,7 +4126,7 @@ struct ConvRow3x3FilterDepth8<4, 1> { }; template <> -struct ConvRow3x3FilterDepth8<4, 2> { +struct ConvRow3x3FilterDepth8<4, 2, 2> { // The buffer size of the shuffled input. static inline constexpr int ShuffleWorkspaceSize() { return 64 * 9 * 9; } @@ -4195,7 +4195,7 @@ struct ConvRow3x3FilterDepth8<4, 2> { const uint8* shuffled_ptr = &shuffle_workspace[0]; for (int micro_depth = 0; micro_depth <= 64 - 8; micro_depth += 8) { - ConvKernel3x3FilterDepth8<4, 4, 2>::Run( + ConvKernel3x3FilterDepth8<4, 4, 2, 2>::Run( shuffled_ptr, 64, input_offset, 64 * 9, filter_ptr, filter_offset, bias_ptr, output_offset, output_multiplier, output_shift, output_activation_min, output_activation_max, output_ptr, @@ -4221,7 +4221,7 @@ struct ConvRow3x3FilterDepth8<4, 2> { DEPTHWISECONV_PRELOAD_ROW(input_ptr, 8); for (; depth <= output_depth - 8; depth += 8) { - ConvKernel3x3FilterDepth8<4, 4, 2>::Run( + ConvKernel3x3FilterDepth8<4, 4, 2, 2>::Run( input_ptr, input_depth, input_offset, input_row_size, filter_ptr, filter_offset, bias_ptr, output_offset, output_multiplier, output_shift, output_activation_min, output_activation_max, @@ -4249,7 +4249,7 @@ struct ConvRow3x3FilterDepth8<4, 2> { uint8* output_ptr = output_data; for (int depth = 0; depth <= output_depth - 8; depth += 8) { - ConvKernel3x3FilterDepth8<4, 2, 2>::Run( + ConvKernel3x3FilterDepth8<4, 2, 2, 2>::Run( input_ptr, input_depth, input_offset, input_row_size, filter_ptr, filter_offset, bias_ptr, output_offset, output_multiplier, output_shift, output_activation_min, output_activation_max, @@ -4274,7 +4274,7 @@ struct ConvRow3x3FilterDepth8<4, 2> { uint8* output_ptr = output_data; for (int depth = 0; depth <= output_depth - 8; depth += 8) { - ConvKernel3x3FilterDepth8<4, 1, 2>::Run( + ConvKernel3x3FilterDepth8<4, 1, 2, 2>::Run( input_ptr, input_depth, input_offset, input_row_size, filter_ptr, filter_offset, bias_ptr, output_offset, output_multiplier, output_shift, output_activation_min, output_activation_max, @@ -4293,7 +4293,7 @@ struct ConvRow3x3FilterDepth8<4, 2> { }; template <> -struct ConvRow3x3FilterDepth8<8, 2> { +struct ConvRow3x3FilterDepth8<8, 2, 2> { static inline void Run(const uint8* input_data, int start_x, int start_y, int input_depth, int input_width, int input_height, int input_row_size, int32 input_offset, @@ -4305,14 +4305,14 @@ struct ConvRow3x3FilterDepth8<8, 2> { int output_depth, int output_width, uint8* shuffle_workspace) { // Reuse 4 row kernels twice. - ConvRow3x3FilterDepth8<4, 2>::Run( + ConvRow3x3FilterDepth8<4, 2, 2>::Run( input_data, start_x, start_y, input_depth, input_width, input_height, input_row_size, input_offset, filter_data, filter_offset, bias_data, output_offset, output_multiplier, output_shift, output_activation_min, output_activation_max, output_data, output_depth, output_width, shuffle_workspace); - ConvRow3x3FilterDepth8<4, 2>::Run( + ConvRow3x3FilterDepth8<4, 2, 2>::Run( input_data + 2 * 4 * input_row_size, start_x, start_y + 4, input_depth, input_width, input_height, input_row_size, input_offset, filter_data, filter_offset, bias_data, output_offset, output_multiplier, @@ -4323,7 +4323,7 @@ struct ConvRow3x3FilterDepth8<8, 2> { }; template <> -struct ConvRow3x3FilterDepth8<8, 1> { +struct ConvRow3x3FilterDepth8<8, 1, 1> { // The buffer size of the shuffled input. static inline constexpr int ShuffleWorkspaceSize() { return 64 * 10 * 10; } @@ -4359,7 +4359,7 @@ struct ConvRow3x3FilterDepth8<8, 1> { const uint8* shuffled_ptr = shuffle_workspace; for (int micro_depth = 0; micro_depth <= 64 - 8; micro_depth += 8) { - ConvKernel3x3FilterDepth8<8, 8, 1>::Run( + ConvKernel3x3FilterDepth8<8, 8, 1, 1>::Run( shuffled_ptr, 64, input_offset, 64 * 10, filter_ptr, filter_offset, bias_ptr, output_offset, output_multiplier, output_shift, output_activation_min, output_activation_max, @@ -4374,7 +4374,7 @@ struct ConvRow3x3FilterDepth8<8, 1> { } for (; depth <= output_depth - 8; depth += 8) { - ConvKernel3x3FilterDepth8<8, 8, 1>::Run( + ConvKernel3x3FilterDepth8<8, 8, 1, 1>::Run( input_ptr, input_depth, input_offset, input_row_size, filter_ptr, filter_offset, bias_ptr, output_offset, output_multiplier, output_shift, output_activation_min, output_activation_max, @@ -4391,14 +4391,14 @@ struct ConvRow3x3FilterDepth8<8, 1> { } // Handle the rest of the right side by re-using 4 row kernels twice. - ConvRow3x3FilterDepth8<4, 1>::Run( + ConvRow3x3FilterDepth8<4, 1, 1>::Run( input_data, out_x, start_y, input_depth, input_width, input_height, input_row_size, input_offset, filter_data, filter_offset, bias_data, output_offset, output_multiplier, output_shift, output_activation_min, output_activation_max, output_data, output_depth, output_width, shuffle_workspace); - ConvRow3x3FilterDepth8<4, 1>::Run( + ConvRow3x3FilterDepth8<4, 1, 1>::Run( input_data + 4 * input_row_size, out_x, start_y + 4, input_depth, input_width, input_height, input_row_size, input_offset, filter_data, filter_offset, bias_data, output_offset, output_multiplier, @@ -4426,7 +4426,8 @@ inline bool Fast3x3FilterKernelSupported(const Dims<4>& input_dims, depth_multiplier == 1 && (stride_width == 1 || stride_width == 2) && (stride_height == 1 || stride_height == 2) && - pad_width == 0 && pad_height == 0 && (input_depth % 8) == 0; + (stride_width == stride_height) && pad_width == 0 && + pad_height == 0 && (input_depth % 8) == 0; if (!supported) { return false; @@ -4477,23 +4478,24 @@ inline void DepthwiseConv3x3Filter( TFLITE_DCHECK(pad_width == 0); TFLITE_DCHECK(stride_height == 1 || stride_height == 2); TFLITE_DCHECK(stride_width == 1 || stride_width == 2); + TFLITE_DCHECK(stride_width == stride_height); const int input_row_size = input_depth * (input_width + 2 * pad_width); const int output_row_size = output_depth * output_width; const int input_batch_size = input_row_size * (input_height + 2 * pad_height); const int output_batch_size = output_depth * output_width * output_height; - using conv_row_func_t = decltype(&ConvRow3x3FilterDepth8<1, 1>::Run); - conv_row_func_t conv_1_output_row = ConvRow3x3FilterDepth8<1, 1>::Run; - conv_row_func_t conv_2_output_rows = ConvRow3x3FilterDepth8<2, 1>::Run; - conv_row_func_t conv_4_output_rows = ConvRow3x3FilterDepth8<4, 1>::Run; - conv_row_func_t conv_8_output_rows = ConvRow3x3FilterDepth8<8, 1>::Run; + using conv_row_func_t = decltype(&ConvRow3x3FilterDepth8<1, 1, 1>::Run); + conv_row_func_t conv_1_output_row = ConvRow3x3FilterDepth8<1, 1, 1>::Run; + conv_row_func_t conv_2_output_rows = ConvRow3x3FilterDepth8<2, 1, 1>::Run; + conv_row_func_t conv_4_output_rows = ConvRow3x3FilterDepth8<4, 1, 1>::Run; + conv_row_func_t conv_8_output_rows = ConvRow3x3FilterDepth8<8, 1, 1>::Run; if (stride_width == 2) { - conv_1_output_row = ConvRow3x3FilterDepth8<1, 2>::Run; - conv_2_output_rows = ConvRow3x3FilterDepth8<2, 2>::Run; - conv_4_output_rows = ConvRow3x3FilterDepth8<4, 2>::Run; - conv_8_output_rows = ConvRow3x3FilterDepth8<8, 2>::Run; + conv_1_output_row = ConvRow3x3FilterDepth8<1, 2, 2>::Run; + conv_2_output_rows = ConvRow3x3FilterDepth8<2, 2, 2>::Run; + conv_4_output_rows = ConvRow3x3FilterDepth8<4, 2, 2>::Run; + conv_8_output_rows = ConvRow3x3FilterDepth8<8, 2, 2>::Run; } // Allocate maximum memory needed for shuffled input. @@ -4505,10 +4507,10 @@ inline void DepthwiseConv3x3Filter( uint8 shuffle_workspace[DEPTHWISECONV_SHUFFLE_WORKSPACE_SIZE]; // Make sure the kernels using this buffer will not run out of bounds. - static_assert(ConvRow3x3FilterDepth8<8, 1>::ShuffleWorkspaceSize() <= + static_assert(ConvRow3x3FilterDepth8<8, 1, 1>::ShuffleWorkspaceSize() <= DEPTHWISECONV_SHUFFLE_WORKSPACE_SIZE, "Shuffle workspace size is too small."); - static_assert(ConvRow3x3FilterDepth8<4, 2>::ShuffleWorkspaceSize() <= + static_assert(ConvRow3x3FilterDepth8<4, 2, 2>::ShuffleWorkspaceSize() <= DEPTHWISECONV_SHUFFLE_WORKSPACE_SIZE, "Shuffle workspace size is too small."); -- GitLab From 6e8c908c8e299ddb46ac20b6a668e37ed37f24c0 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 10:30:32 -0700 Subject: [PATCH 323/791] Disable x * x -> square(x) Grapler rewrite for complex types unless the op is on CPU. Square is not registered for complex types on GPU, and doing so produces a crash in with CUDA_ILLEGAL_INSTRUCTION when running it on open source ubuntu. PiperOrigin-RevId: 192788160 --- .../optimizers/arithmetic_optimizer.cc | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc index 60b1af48ec..b80ae5fa40 100644 --- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc @@ -1782,13 +1782,22 @@ string ArithmeticOptimizer::TrySimplifyAndReplaceUses( if (node->op() == "Mul" && node->input(0) == node->input(1) && !OptimizedNodeExists(*node, "square")) { - NodeDef* new_square_node = AddNode(*node, "square", /*copy_node=*/true); - new_square_node->set_op("Square"); - for (int i = 1; i < new_square_node->input_size(); ++i) { - new_square_node->set_input(i - 1, new_square_node->input(i)); + const DataType type = GetDataTypeFromAttr(*node, "T"); + bool is_complex = (type == DT_COMPLEX64) || (type == DT_COMPLEX128); + string dontcare; + string device; + bool is_on_cpu = + DeviceNameUtils::SplitDeviceName(node->device(), &dontcare, &device) && + str_util::StrContains(device, DEVICE_CPU); + if (!is_complex || is_on_cpu) { + NodeDef* new_square_node = AddNode(*node, "square", /*copy_node=*/true); + new_square_node->set_op("Square"); + for (int i = 1; i < new_square_node->input_size(); ++i) { + new_square_node->set_input(i - 1, new_square_node->input(i)); + } + new_square_node->mutable_input()->RemoveLast(); + return new_square_node->name(); } - new_square_node->mutable_input()->RemoveLast(); - return new_square_node->name(); } if (IsAggregate(*node) && NumNonControlInputs(*node) > 0) { -- GitLab From 8303fa2a53071a7e4a346454f707d25abbd6e1b5 Mon Sep 17 00:00:00 2001 From: James Wexler Date: Fri, 13 Apr 2018 13:33:37 -0400 Subject: [PATCH 324/791] closure proto library for example protos --- WORKSPACE | 19 ++++++++++++------- tensorflow/core/BUILD | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 11c5cdb207..d37e213922 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,13 +1,18 @@ workspace(name = "org_tensorflow") -http_archive( +## DO NOT SUBMIT +#http_archive( +# name = "io_bazel_rules_closure", +# sha256 = "6691c58a2cd30a86776dd9bb34898b041e37136f2dc7e24cadaeaf599c95c657", +# strip_prefix = "rules_closure-08039ba8ca59f64248bb3b6ae016460fe9c9914f", +# urls = [ +# "https://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/08039ba8ca59f64248bb3b6ae016460fe9c9914f.tar.gz", +# "https://github.com/bazelbuild/rules_closure/archive/08039ba8ca59f64248bb3b6ae016460fe9c9914f.tar.gz", # 2018-01-16 +# ], +#) +local_repository( name = "io_bazel_rules_closure", - sha256 = "6691c58a2cd30a86776dd9bb34898b041e37136f2dc7e24cadaeaf599c95c657", - strip_prefix = "rules_closure-08039ba8ca59f64248bb3b6ae016460fe9c9914f", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/08039ba8ca59f64248bb3b6ae016460fe9c9914f.tar.gz", - "https://github.com/bazelbuild/rules_closure/archive/08039ba8ca59f64248bb3b6ae016460fe9c9914f.tar.gz", # 2018-01-16 - ], + path = "/usr/local/google/home/jwexler/jameswex/rules_closure", ) load("@io_bazel_rules_closure//closure:defs.bzl", "closure_repositories") diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index c5ca421ced..08884fa914 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -149,6 +149,7 @@ load( "//third_party/mkl:build_defs.bzl", "if_mkl", ) +load("@io_bazel_rules_closure//closure:defs.bzl","closure_proto_library") exports_files(["ops/ops.pbtxt"]) @@ -244,6 +245,21 @@ tf_nano_proto_library( deps = [":protos_all_cc"], ) +proto_library( + name = "example_protos", + srcs = [ + "example/example.proto", + "example/feature.proto", + ], + visibility = ["//visibility:public"], +) + +closure_proto_library( + name = "example_protos_closure", + deps = [":example_protos"], + visibility = ["//visibility:public"], +) + exports_files([ "framework/types.proto", ]) -- GitLab From b004e233da511e2692277d5a98d72ec40917b4b2 Mon Sep 17 00:00:00 2001 From: Anna R Date: Fri, 13 Apr 2018 10:52:56 -0700 Subject: [PATCH 325/791] Internal change. PiperOrigin-RevId: 192791493 --- tensorflow/python/kernel_tests/BUILD | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index e504a9fd21..e82d738f14 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -2669,10 +2669,6 @@ cuda_py_test( "//tensorflow/python:variables", ], shard_count = 50, - tags = [ - "manual", - "notap", # b/30226163 - ], ) cuda_py_test( -- GitLab From ff97232dbf44c8c5515e10f7d3d72f215381bd65 Mon Sep 17 00:00:00 2001 From: Mark Heffernan Date: Fri, 13 Apr 2018 11:02:08 -0700 Subject: [PATCH 326/791] Fix comment in xla_data.proto related to padding value for Windows. PiperOrigin-RevId: 192792971 --- tensorflow/compiler/xla/xla_data.proto | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto index 1f16e6d251..f18d53c608 100644 --- a/tensorflow/compiler/xla/xla_data.proto +++ b/tensorflow/compiler/xla/xla_data.proto @@ -355,17 +355,19 @@ message WindowDimension { // positions of the window in this dimension. int64 stride = 2; - // If positive, means the amount of padding with zeroes to add to the base - // area at the low end of this dimension; if negative, its negative means the - // number of elements removed from the low end of this dimension. For example, - // in the horizontal dimension of a rectangle, this would be the number of - // zeroes to pad on the left, given that indices increase when going right. + // If positive, means the amount of padding to add to the base area at the low + // end of this dimension; if negative, its negative means the number of + // elements removed from the low end of this dimension. For example, in the + // horizontal dimension of a rectangle, this would be the number of padding + // values to pad on the left, given that indices increase when going right. + // The actual padding value depends upon the context. Convolution pads with + // zeros. ReduceWindow and SelectAndScatter pads with the reduce function's + // init value. int64 padding_low = 3; - // As padding_low, but on the high end of this dimension. For - // example, in the horizontal dimension of a rectangle, this would - // be the number of zeroes to pad on the right, given that indices - // increase when going right. + // As padding_low, but on the high end of this dimension. For example, in the + // horizontal dimension of a rectangle, this would be the number of values to + // pad on the right, given that indices increase when going right. int64 padding_high = 4; // Dilation factor of the sliding window in this dimension. A dilation factor -- GitLab From 4fa6ca2bb74aa27ffb71a23e4a8d72810c377b07 Mon Sep 17 00:00:00 2001 From: James Wexler Date: Fri, 13 Apr 2018 14:09:42 -0400 Subject: [PATCH 327/791] review changes --- WORKSPACE | 19 +++++++------------ tensorflow/core/BUILD | 2 +- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index d37e213922..4ddfb9a383 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,18 +1,13 @@ workspace(name = "org_tensorflow") -## DO NOT SUBMIT -#http_archive( -# name = "io_bazel_rules_closure", -# sha256 = "6691c58a2cd30a86776dd9bb34898b041e37136f2dc7e24cadaeaf599c95c657", -# strip_prefix = "rules_closure-08039ba8ca59f64248bb3b6ae016460fe9c9914f", -# urls = [ -# "https://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/08039ba8ca59f64248bb3b6ae016460fe9c9914f.tar.gz", -# "https://github.com/bazelbuild/rules_closure/archive/08039ba8ca59f64248bb3b6ae016460fe9c9914f.tar.gz", # 2018-01-16 -# ], -#) -local_repository( +http_archive( name = "io_bazel_rules_closure", - path = "/usr/local/google/home/jwexler/jameswex/rules_closure", + sha256 = "a38539c5b5c358548e75b44141b4ab637bba7c4dc02b46b1f62a96d6433f56ae", + strip_prefix = "rules_closure-dbb96841cc0a5fb2664c37822803b06dab20c7d1", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/dbb96841cc0a5fb2664c37822803b06dab20c7d1.tar.gz", + "https://github.com/bazelbuild/rules_closure/archive/dbb96841cc0a5fb2664c37822803b06dab20c7d1.tar.gz", # 2018-04-13 + ], ) load("@io_bazel_rules_closure//closure:defs.bzl", "closure_repositories") diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 08884fa914..ab25283cc4 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -149,7 +149,7 @@ load( "//third_party/mkl:build_defs.bzl", "if_mkl", ) -load("@io_bazel_rules_closure//closure:defs.bzl","closure_proto_library") +load("@io_bazel_rules_closure//closure:defs.bzl", "closure_proto_library") exports_files(["ops/ops.pbtxt"]) -- GitLab From 8e2fd4b30210ef633153b65d3d45cc51a3d4f0cf Mon Sep 17 00:00:00 2001 From: Akshay Modi Date: Fri, 13 Apr 2018 11:09:58 -0700 Subject: [PATCH 328/791] Use eager compatible wrappers in load_library for custom ops --- tensorflow/python/BUILD | 1 + tensorflow/python/framework/load_library.py | 2 +- tensorflow/python/framework/python_op_gen.i | 8 ++-- .../tools/ci_build/builds/test_user_ops.sh | 41 +++++++++++-------- 4 files changed, 30 insertions(+), 22 deletions(-) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index db17a3fe02..9209ca4b96 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -3286,6 +3286,7 @@ tf_py_wrap_cc( "//tensorflow/core/profiler/internal:print_model_analysis", "//tensorflow/tools/graph_transforms:transform_graph_lib", "//tensorflow/python/eager:pywrap_tfe_lib", + "//tensorflow/python/eager:python_eager_op_gen", "//util/python:python_headers", ] + (tf_additional_lib_deps() + tf_additional_plugin_deps() + diff --git a/tensorflow/python/framework/load_library.py b/tensorflow/python/framework/load_library.py index 1f2aa264c1..4f349304d3 100644 --- a/tensorflow/python/framework/load_library.py +++ b/tensorflow/python/framework/load_library.py @@ -60,7 +60,7 @@ def load_op_library(library_filename): op_list_str = py_tf.TF_GetOpList(lib_handle) op_list = op_def_pb2.OpList() op_list.ParseFromString(compat.as_bytes(op_list_str)) - wrappers = py_tf.GetPythonWrappers(op_list_str) + wrappers = py_tf.GetEagerPythonWrappers(op_list_str) # Delete the library handle to release any memory held in C # that are no longer needed. diff --git a/tensorflow/python/framework/python_op_gen.i b/tensorflow/python/framework/python_op_gen.i index 26ec4e8e66..e39c425b05 100644 --- a/tensorflow/python/framework/python_op_gen.i +++ b/tensorflow/python/framework/python_op_gen.i @@ -16,10 +16,10 @@ limitations under the License. %include "tensorflow/python/platform/base.i" %{ -#include "tensorflow/python/framework/python_op_gen.h" +#include "tensorflow/python/eager/python_eager_op_gen.h" %} -// Input typemap for GetPythonWrappers. +// Input typemap for GetEagerPythonWrappers. // Accepts a python object of 'bytes' type, and converts it to // a const char* pointer and size_t length. The default typemap // going from python bytes to const char* tries to decode the @@ -37,5 +37,5 @@ limitations under the License. %ignoreall; -%unignore tensorflow::GetPythonWrappers; -%include "tensorflow/python/framework/python_op_gen.h" +%unignore tensorflow::GetEagerPythonWrappers; +%include "third_party/tensorflow/python/eager/python_eager_op_gen.h" diff --git a/tensorflow/tools/ci_build/builds/test_user_ops.sh b/tensorflow/tools/ci_build/builds/test_user_ops.sh index caa3a40817..c342367bac 100755 --- a/tensorflow/tools/ci_build/builds/test_user_ops.sh +++ b/tensorflow/tools/ci_build/builds/test_user_ops.sh @@ -213,27 +213,34 @@ USER_OP=$(echo "${USER_OP_SO}" | sed -e 's/\.so//') echo "Invoking user op ${USER_OP} defined in file ${USER_OP_SO} "\ "via pip installation" -ORIG_OUTPUT=$("${PYTHON_BIN_PATH}" -c "import tensorflow as tf; print(tf.Session('').run(tf.load_op_library('./${USER_OP_SO}').${USER_OP}(${OP_INPUT})))") - -# Format OUTPUT for analysis -if [[ -z $(echo "${ORIG_OUTPUT}" | grep -o ',') ]]; then - if [[ ${IS_MAC} == "1" ]]; then - OUTPUT=$(echo "${ORIG_OUTPUT}" | sed -E -e 's/[ \t]+/,/g') +function run_op() { + local ORIG_OUTPUT=$1 + local ADDITIONAL_LOG=$2 + + # Format OUTPUT for analysis + if [[ -z $(echo "${ORIG_OUTPUT}" | grep -o ',') ]]; then + if [[ ${IS_MAC} == "1" ]]; then + local OUTPUT=$(echo "${ORIG_OUTPUT}" | sed -E -e 's/[ \t]+/,/g') + else + local OUTPUT=$(echo "${ORIG_OUTPUT}" | sed -r -e 's/[ \t]+/,/g') + fi else - OUTPUT=$(echo "${ORIG_OUTPUT}" | sed -r -e 's/[ \t]+/,/g') + local OUTPUT="${ORIG_OUTPUT}" fi -else - OUTPUT="${ORIG_OUTPUT}" -fi -EQUALS_EXPECTED=$("${PYTHON_BIN_PATH}" -c "print(${OUTPUT} == ${EXPECTED_OUTPUT})") + local EQUALS_EXPECTED=$("${PYTHON_BIN_PATH}" -c "print(${OUTPUT} == ${EXPECTED_OUTPUT})") -if [[ "${EQUALS_EXPECTED}" != "True" ]]; then - die "FAILED: Output from user op (${OUTPUT}) does not match expected "\ -"output ${EXPECTED_OUTPUT}" -else - echo "Output from user op (${OUTPUT}) matches expected output" -fi + if [[ "${EQUALS_EXPECTED}" != "True" ]]; then + local ERROR="FAILED: Output from user op (${OUTPUT}) does not match expected "\ + "output ${EXPECTED_OUTPUT}"${ADDITIONAL_LOG} + die ${ERROR} + else + echo "Output from user op (${OUTPUT}) matches expected output" + fi +} + +run_op $("${PYTHON_BIN_PATH}" -c "import tensorflow as tf; print(tf.Session('').run(tf.load_op_library('./${USER_OP_SO}').${USER_OP}(${OP_INPUT})))") +run_op $("${PYTHON_BIN_PATH}" -c "import tensorflow as tf; tf.enable_eager_execution(); print(tf.load_op_library('./${USER_OP_SO}').${USER_OP}(${OP_INPUT}))") " in eager mode" popd -- GitLab From e42ebc5b95856760332443987292e5d750050531 Mon Sep 17 00:00:00 2001 From: James Qin Date: Fri, 13 Apr 2018 11:06:49 -0700 Subject: [PATCH 329/791] Add more logging for failure cases in CUDATimer PiperOrigin-RevId: 192793983 --- tensorflow/stream_executor/cuda/cuda_timer.cc | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/tensorflow/stream_executor/cuda/cuda_timer.cc b/tensorflow/stream_executor/cuda/cuda_timer.cc index 7d78601fb9..8532f08725 100644 --- a/tensorflow/stream_executor/cuda/cuda_timer.cc +++ b/tensorflow/stream_executor/cuda/cuda_timer.cc @@ -73,16 +73,22 @@ float CUDATimer::GetElapsedMilliseconds() const { return elapsed_milliseconds; } -bool CUDATimer::Start(CUDAStream *stream) { - return CUDADriver::RecordEvent(parent_->cuda_context(), start_event_, - stream->cuda_stream()) - .ok(); +bool CUDATimer::Start(CUDAStream* stream) { + port::Status status = CUDADriver::RecordEvent( + parent_->cuda_context(), start_event_, stream->cuda_stream()); + if (!status.ok()) { + LOG(ERROR) << status; + } + return status.ok(); } -bool CUDATimer::Stop(CUDAStream *stream) { - return CUDADriver::RecordEvent(parent_->cuda_context(), stop_event_, - stream->cuda_stream()) - .ok(); +bool CUDATimer::Stop(CUDAStream* stream) { + port::Status status = CUDADriver::RecordEvent( + parent_->cuda_context(), stop_event_, stream->cuda_stream()); + if (!status.ok()) { + LOG(ERROR) << status; + } + return status.ok(); } } // namespace cuda -- GitLab From 3dbd4518321088e2796e738fec2e253cdc6d3da1 Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Fri, 13 Apr 2018 11:14:09 -0700 Subject: [PATCH 330/791] [TF:XLA] Start a TensorFlow library that contains direct wrappers for XLA operators. Add new XlaReduceWindow and XlaDynamicUpdateSlice operators. Add new tests for the existing XlaWhile operator. Add wrappers for XlaSend and XlaRecv. PiperOrigin-RevId: 192795174 --- tensorflow/compiler/tests/BUILD | 43 ++++++ .../compiler/tests/dynamic_slice_ops_test.py | 93 ++++++++++++ .../compiler/tests/reduce_window_test.py | 102 +++++++++++++ tensorflow/compiler/tests/while_test.py | 130 +++++++++++++++++ tensorflow/compiler/tf2xla/BUILD | 4 +- tensorflow/compiler/tf2xla/cc/BUILD | 38 +---- .../tf2xla/functionalize_control_flow_test.cc | 2 +- tensorflow/compiler/tf2xla/kernels/BUILD | 8 +- .../tf2xla/kernels/dynamic_slice_ops.cc | 69 +++++++++ .../tf2xla/kernels/reduce_window_op.cc | 135 ++++++++++++++++++ .../compiler/tf2xla/kernels/sendrecv_ops.cc | 6 +- tensorflow/compiler/tf2xla/ops/BUILD | 30 ++-- .../compiler/tf2xla/ops/dynamic_slice_ops.cc | 49 +++++++ .../compiler/tf2xla/ops/reduce_window_op.cc | 45 ++++++ .../compiler/tf2xla/ops/sendrecv_ops.cc | 23 +-- tensorflow/compiler/tf2xla/python/BUILD | 8 ++ tensorflow/compiler/tf2xla/python/xla.py | 80 +++++++++++ 17 files changed, 795 insertions(+), 70 deletions(-) create mode 100644 tensorflow/compiler/tests/dynamic_slice_ops_test.py create mode 100644 tensorflow/compiler/tests/reduce_window_test.py create mode 100644 tensorflow/compiler/tests/while_test.py create mode 100644 tensorflow/compiler/tf2xla/kernels/dynamic_slice_ops.cc create mode 100644 tensorflow/compiler/tf2xla/kernels/reduce_window_op.cc create mode 100644 tensorflow/compiler/tf2xla/ops/dynamic_slice_ops.cc create mode 100644 tensorflow/compiler/tf2xla/ops/reduce_window_op.cc create mode 100644 tensorflow/compiler/tf2xla/python/xla.py diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD index 47c6ab58c0..b9e42ca677 100644 --- a/tensorflow/compiler/tests/BUILD +++ b/tensorflow/compiler/tests/BUILD @@ -271,6 +271,18 @@ tf_xla_py_test( ], ) +tf_xla_py_test( + name = "dynamic_slice_ops_test", + size = "small", + srcs = ["dynamic_slice_ops_test.py"], + deps = [ + "//tensorflow/compiler/tests:xla_test", + "//tensorflow/compiler/tf2xla/python:xla", + "//tensorflow/python:array_ops", + "//tensorflow/python:dtypes", + ], +) + tf_xla_py_test( name = "dynamic_stitch_test", size = "small", @@ -497,6 +509,22 @@ tf_xla_py_test( ], ) +tf_xla_py_test( + name = "reduce_window_test", + size = "small", + srcs = ["reduce_window_test.py"], + disabled_backends = ["cpu_ondemand"], + deps = [ + ":xla_test", + "//tensorflow/compiler/tf2xla/python:xla", + "//tensorflow/python:array_ops", + "//tensorflow/python:errors", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:math_ops", + "//tensorflow/python:platform_test", + ], +) + tf_xla_py_test( name = "reverse_ops_test", size = "medium", @@ -689,6 +717,21 @@ tf_xla_py_test( ], ) +tf_xla_py_test( + name = "while_test", + size = "small", + srcs = ["while_test.py"], + disabled_backends = ["cpu_ondemand"], + deps = [ + ":xla_test", + "//tensorflow/compiler/tf2xla/python:xla", + "//tensorflow/python:array_ops", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:platform_test", + "//tensorflow/python:training", + ], +) + tf_xla_py_test( name = "gather_test", size = "medium", diff --git a/tensorflow/compiler/tests/dynamic_slice_ops_test.py b/tensorflow/compiler/tests/dynamic_slice_ops_test.py new file mode 100644 index 0000000000..6a46d2ec3e --- /dev/null +++ b/tensorflow/compiler/tests/dynamic_slice_ops_test.py @@ -0,0 +1,93 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for XLA dynamic slicing ops.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.compiler.tests.xla_test import XLATestCase +from tensorflow.compiler.tf2xla.python import xla +from tensorflow.python.framework import dtypes +from tensorflow.python.ops import array_ops +from tensorflow.python.platform import test + + +class DynamicUpdateSliceOpsTest(XLATestCase): + + def _assertOpOutputMatchesExpected(self, op, args, expected): + with self.test_session() as session: + with self.test_scope(): + placeholders = [ + array_ops.placeholder(dtypes.as_dtype(arg.dtype), arg.shape) + for arg in args + ] + feeds = {placeholders[i]: args[i] for i in range(0, len(args))} + output = op(*placeholders) + result = session.run(output, feeds) + self.assertAllClose(result, expected, rtol=1e-3) + + def testUpdateSlice(self): + for dtype in self.numeric_types: + self._assertOpOutputMatchesExpected( + xla.dynamic_update_slice, [ + np.array([], dtype=dtype), + np.array([], dtype=dtype), + np.array([0], dtype=np.int32) + ], + expected=np.array([], dtype=dtype)) + + self._assertOpOutputMatchesExpected( + xla.dynamic_update_slice, [ + np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], dtype=dtype), + np.array([-1, -2, -3], dtype=dtype), + np.array([6], dtype=np.int32) + ], + expected=np.array([1, 2, 3, 4, 5, 6, -1, -2, -3, 10], dtype=dtype)) + + self._assertOpOutputMatchesExpected( + xla.dynamic_update_slice, [ + np.array( + [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]], dtype=dtype), + np.array([[42, 43], [44, 45]], dtype=dtype), + np.array([1, 2], dtype=np.int32) + ], + expected=np.array( + [[1, 2, 3, 4], [5, 6, 42, 43], [9, 10, 44, 45]], dtype=dtype)) + + self._assertOpOutputMatchesExpected( + xla.dynamic_update_slice, [ + np.array( + [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]], dtype=dtype), + np.array([[], []], dtype=dtype), + np.array([1, 2], dtype=np.int32) + ], + expected=np.array( + [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]], dtype=dtype)) + + self._assertOpOutputMatchesExpected( + xla.dynamic_update_slice, [ + np.array( + [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]], dtype=dtype), + np.ones([3, 4], dtype=dtype), + np.array([0, 0], dtype=np.int32) + ], + expected=np.ones([3, 4], dtype=dtype)) + + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/compiler/tests/reduce_window_test.py b/tensorflow/compiler/tests/reduce_window_test.py new file mode 100644 index 0000000000..e78a63465b --- /dev/null +++ b/tensorflow/compiler/tests/reduce_window_test.py @@ -0,0 +1,102 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for xla.reduce_window.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.compiler.tests.xla_test import XLATestCase +from tensorflow.compiler.tf2xla.python import xla +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import function +from tensorflow.python.ops import array_ops +from tensorflow.python.platform import googletest + + +class ReduceWindowTest(XLATestCase): + """Test cases for xla.reduce_window.""" + + def _reduce_window(self, operand, init, reducer, **kwargs): + with self.test_session(): + placeholder = array_ops.placeholder(operand.dtype) + with self.test_scope(): + output = xla.reduce_window(placeholder, init, reducer, **kwargs) + return output.eval(feed_dict={placeholder: operand}) + + def testReduceWindow(self): + + # TODO(b/77644762): float16 and float64 ReduceWindow are unimplemented. + for dtype in set(self.numeric_types).intersection( + set([dtypes.bfloat16.as_numpy_dtype, np.float32])): + + @function.Defun(dtype, dtype) + def sum_reducer(x, y): + return x + y + + @function.Defun(dtype, dtype) + def mul_reducer(x, y): + return x * y + + self.assertAllClose( + np.array([3, 5, 7, 9, 11, 13], dtype=dtype), + self._reduce_window( + np.array([1, 2, 3, 4, 5, 6, 7], dtype=dtype), + 0.0, + sum_reducer, + window_dimensions=[2])) + + self.assertAllClose( + np.array([3, 7, 11], dtype=dtype), + self._reduce_window( + np.array([1, 2, 3, 4, 5, 6, 7], dtype=dtype), + 0.0, + sum_reducer, + window_dimensions=[2], + window_strides=[2])) + + self.assertAllClose( + np.array([1, 4, 7], dtype=dtype), + self._reduce_window( + np.array([1, 2, 3, 4, 5, 6, 7], dtype=dtype), + 0.0, + sum_reducer, + window_dimensions=[1], + window_strides=[3])) + + self.assertAllClose( + np.array([[24, 36, 24], [96, 0, 0]], dtype=dtype), + self._reduce_window( + np.array([[1, 2, 3, 4], [4, 3, 2, 1], [2, 4, 0, 1]], dtype=dtype), + 1.0, + mul_reducer, + window_dimensions=[2, 2], + window_strides=[1, 1])) + + self.assertAllClose( + np.array([[0, 0, 0], [5, 10, 5], [2, 4, 1], [0, 0, 0]], dtype=dtype), + self._reduce_window( + np.array([[1, 2, 3, 4], [4, 3, 2, 1], [2, 4, 0, 1]], dtype=dtype), + 0.0, + sum_reducer, + window_dimensions=[2, 2], + window_strides=[2, 2], + padding=[[2, 3], [1, 2]])) + + +if __name__ == '__main__': + googletest.main() diff --git a/tensorflow/compiler/tests/while_test.py b/tensorflow/compiler/tests/while_test.py new file mode 100644 index 0000000000..f79eb27435 --- /dev/null +++ b/tensorflow/compiler/tests/while_test.py @@ -0,0 +1,130 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for while loops in XLA.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.compiler.tests.xla_test import XLATestCase +from tensorflow.compiler.tf2xla.python import xla +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import function +from tensorflow.python.ops import array_ops +from tensorflow.python.platform import test + + +class WhileTest(XLATestCase): + + def testSingletonLoopHandrolled(self): + # Define a function for the loop body + @function.Defun(dtypes.int32) + def loop_body(step): + step_out = step + constant_op.constant(1, dtype=dtypes.int32) + return step_out + + # Define a function for the loop condition + @function.Defun(dtypes.int32) + def loop_cond(step): + return step < 10 + + with self.test_session() as sess: + init_index = array_ops.placeholder(dtypes.int32, []) + with self.test_scope(): + loop_outputs = xla.while_loop([init_index], loop_cond, loop_body) + + result = sess.run(loop_outputs, {init_index: 0}) + self.assertAllClose(result, [10], rtol=1e-3) + + def testCountingLoopHandrolled(self): + # Define a function for the loop body + @function.Defun(dtypes.int32, dtypes.float32) + def loop_body(step, rsum): + step_out = step + constant_op.constant(1, dtype=dtypes.int32) + sum_out = rsum + constant_op.constant(1.5, dtype=dtypes.float32) + return step_out, sum_out + + # Define a function for the loop condition + @function.Defun(dtypes.int32, dtypes.float32) + def loop_cond(step, rsum): + del rsum + return step < 10 + + with self.test_session() as sess: + init_index = array_ops.placeholder(dtypes.int32, []) + init_sum = array_ops.placeholder(dtypes.float32, []) + with self.test_scope(): + loop_outputs = xla.while_loop([init_index, init_sum], loop_cond, + loop_body) + + result = sess.run(loop_outputs, {init_index: 0, init_sum: 0.0}) + self.assertAllClose(result, [10, 15.0], rtol=1e-3) + no_iters_result = sess.run(loop_outputs, {init_index: 10, init_sum: 0.0}) + self.assertAllClose(no_iters_result, [10, 0.0], rtol=1e-3) + + def testCountingLoopHandrolledC64(self): + # Define a function for the loop body + @function.Defun(dtypes.int32, dtypes.complex64) + def loop_body(step, rsum): + step_out = step + constant_op.constant(1, dtype=dtypes.int32) + sum_out = rsum + constant_op.constant(1.5 + 2j, dtype=dtypes.complex64) + return step_out, sum_out + + # Define a function for the loop condition + @function.Defun(dtypes.int32, dtypes.complex64) + def loop_cond(step, rsum): + del rsum + return step < 10 + + with self.test_session() as sess: + init_index = array_ops.placeholder(dtypes.int32, []) + init_sum = array_ops.placeholder(dtypes.complex64, []) + with self.test_scope(): + loop_outputs = xla.while_loop([init_index, init_sum], loop_cond, + loop_body) + + result = sess.run(loop_outputs, {init_index: 0, init_sum: 0.0}) + self.assertAllClose(result[1], np.complex64(15 + 20j), rtol=1e-3) + no_iters_result = sess.run(loop_outputs, {init_index: 10, init_sum: 0.0}) + self.assertAllClose(no_iters_result[1], np.complex64(0), rtol=1e-3) + + def testLoopWithConstantOutput(self): + # Define a function for the loop body + @function.Defun(dtypes.int32, dtypes.int32) + def loop_body(step, x): + del x + step_out = step + constant_op.constant(1, dtype=dtypes.int32) + return (step_out, 7) + + # Define a function for the loop condition + @function.Defun(dtypes.int32, dtypes.int32) + def loop_cond(step, x): + del x + return step < 10 + + with self.test_session() as sess: + init_index = array_ops.placeholder(dtypes.int32, []) + with self.test_scope(): + loop_outputs = xla.while_loop([init_index, 42], loop_cond, loop_body) + + result = sess.run(loop_outputs, {init_index: 0}) + self.assertAllClose(result, [10, 7], rtol=1e-3) + + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD index e7daf4e01c..ba5c3a1484 100644 --- a/tensorflow/compiler/tf2xla/BUILD +++ b/tensorflow/compiler/tf2xla/BUILD @@ -415,7 +415,7 @@ cc_library( "//tensorflow/compiler/jit:graph_to_functiondef", "//tensorflow/compiler/jit:union_find", "//tensorflow/compiler/tf2xla:dump_graph", - "//tensorflow/compiler/tf2xla/ops:functional_ops", + "//tensorflow/compiler/tf2xla/ops:xla_ops", "//tensorflow/compiler/xla:status_macros", "//tensorflow/compiler/xla:util", "//tensorflow/core:core_cpu", @@ -437,7 +437,7 @@ tf_cc_test( "//tensorflow/cc:function_ops", "//tensorflow/cc:ops", "//tensorflow/cc:resource_variable_ops", - "//tensorflow/compiler/tf2xla/cc:functional_ops", + "//tensorflow/compiler/tf2xla/cc:xla_ops", "//tensorflow/compiler/xla:status_macros", "//tensorflow/core:core_cpu", "//tensorflow/core:core_cpu_internal", diff --git a/tensorflow/compiler/tf2xla/cc/BUILD b/tensorflow/compiler/tf2xla/cc/BUILD index c30bb9cacd..4f8bb8ad74 100644 --- a/tensorflow/compiler/tf2xla/cc/BUILD +++ b/tensorflow/compiler/tf2xla/cc/BUILD @@ -7,44 +7,20 @@ licenses(["notice"]) # Apache 2.0 load("//tensorflow:tensorflow.bzl", "tf_gen_op_wrapper_cc") tf_gen_op_wrapper_cc( - name = "functional_ops_gen", - include_internal_ops = 1, - out_ops_file = "ops/functional_ops", - deps = ["//tensorflow/compiler/tf2xla/ops:functional_ops"], + name = "xla_ops_gen", + out_ops_file = "ops/xla_ops", + deps = ["//tensorflow/compiler/tf2xla/ops:xla_ops"], ) cc_library( - name = "functional_ops", - srcs = ["ops/functional_ops.cc"], - hdrs = ["ops/functional_ops.h"], + name = "xla_ops", + srcs = ["ops/xla_ops.cc"], + hdrs = ["ops/xla_ops.h"], deps = [ "//tensorflow/cc:const_op", "//tensorflow/cc:ops", "//tensorflow/cc:scope", - "//tensorflow/compiler/tf2xla/ops:functional_ops", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:protos_all_cc", - ], -) - -tf_gen_op_wrapper_cc( - name = "sendrecv_ops_gen", - include_internal_ops = 1, - out_ops_file = "ops/sendrecv_ops", - deps = ["//tensorflow/compiler/tf2xla/ops:sendrecv_ops"], -) - -cc_library( - name = "sendrecv_ops", - srcs = ["ops/sendrecv_ops.cc"], - hdrs = ["ops/sendrecv_ops.h"], - deps = [ - "//tensorflow/cc:const_op", - "//tensorflow/cc:ops", - "//tensorflow/cc:scope", - "//tensorflow/compiler/tf2xla/ops:sendrecv_ops", + "//tensorflow/compiler/tf2xla/ops:xla_ops", "//tensorflow/core:core_cpu", "//tensorflow/core:framework", "//tensorflow/core:lib", diff --git a/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc b/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc index bc7276c3af..e494f42e8e 100644 --- a/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc +++ b/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc @@ -20,7 +20,7 @@ limitations under the License. #include "tensorflow/cc/ops/function_ops.h" #include "tensorflow/cc/ops/resource_variable_ops.h" #include "tensorflow/cc/ops/standard_ops.h" -#include "tensorflow/compiler/tf2xla/cc/ops/functional_ops.h" +#include "tensorflow/compiler/tf2xla/cc/ops/xla_ops.h" #include "tensorflow/compiler/tf2xla/test_util.h" #include "tensorflow/compiler/xla/status_macros.h" #include "tensorflow/core/common_runtime/function.h" diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD index 3ba37b0383..579b669699 100644 --- a/tensorflow/compiler/tf2xla/kernels/BUILD +++ b/tensorflow/compiler/tf2xla/kernels/BUILD @@ -29,6 +29,7 @@ tf_kernel_library( "cwise_ops.h", "depthtospace_op.cc", "diag_op.cc", + "dynamic_slice_ops.cc", "dynamic_stitch_op.cc", "elu_op.cc", "extract_image_patches_op.cc", @@ -56,6 +57,7 @@ tf_kernel_library( "pooling_ops.cc", "quantize_and_dequantize_op.cc", "random_ops.cc", + "reduce_window_op.cc", "reduction_ops.cc", "reduction_ops.h", "reduction_ops_common.cc", @@ -103,7 +105,7 @@ tf_kernel_library( "//tensorflow/compiler/tf2xla/lib:triangular_solve", "//tensorflow/compiler/tf2xla/lib:util", "//tensorflow/compiler/tf2xla/lib:while_loop", - "//tensorflow/compiler/tf2xla/ops:sendrecv_ops", + "//tensorflow/compiler/tf2xla/ops:xla_ops", "//tensorflow/compiler/xla:array4d", "//tensorflow/compiler/xla:literal_util", "//tensorflow/compiler/xla:shape_util", @@ -146,7 +148,7 @@ tf_kernel_library( deps = [ "//tensorflow/compiler/tf2xla:common", "//tensorflow/compiler/tf2xla:xla_compiler", - "//tensorflow/compiler/tf2xla/ops:functional_ops", + "//tensorflow/compiler/tf2xla/ops:xla_ops", "//tensorflow/compiler/xla:literal_util", "//tensorflow/compiler/xla/client:computation_builder", "//tensorflow/core:framework", @@ -162,7 +164,7 @@ tf_kernel_library( deps = [ "//tensorflow/compiler/tf2xla:common", "//tensorflow/compiler/tf2xla:xla_compiler", - "//tensorflow/compiler/tf2xla/ops:functional_ops", + "//tensorflow/compiler/tf2xla/ops:xla_ops", "//tensorflow/compiler/xla:literal_util", "//tensorflow/compiler/xla/client:computation_builder", "//tensorflow/core:framework", diff --git a/tensorflow/compiler/tf2xla/kernels/dynamic_slice_ops.cc b/tensorflow/compiler/tf2xla/kernels/dynamic_slice_ops.cc new file mode 100644 index 0000000000..800ef5ab98 --- /dev/null +++ b/tensorflow/compiler/tf2xla/kernels/dynamic_slice_ops.cc @@ -0,0 +1,69 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "tensorflow/compiler/tf2xla/shape_util.h" +#include "tensorflow/compiler/tf2xla/xla_op_kernel.h" +#include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "tensorflow/compiler/xla/client/computation_builder.h" +#include "tensorflow/core/framework/op_kernel.h" + +#include "tensorflow/compiler/tf2xla/type_util.h" +#include "tensorflow/compiler/tf2xla/xla_helpers.h" +#include "tensorflow/core/framework/kernel_def_builder.h" + +namespace tensorflow { +namespace { + +class DynamicUpdateSliceOp : public XlaOpKernel { + public: + explicit DynamicUpdateSliceOp(OpKernelConstruction* context) + : XlaOpKernel(context) {} + + void Compile(XlaOpKernelContext* ctx) override { + VLOG(3) << "DynamicUpdateSliceOp::Compile"; + + DataType index_type = input_type(2); + OP_REQUIRES(ctx, index_type == DT_INT32 || index_type == DT_INT64, + errors::InvalidArgument("index must be int32 or int64")); + + const TensorShape input_shape = ctx->InputShape(0); + const TensorShape update_shape = ctx->InputShape(1); + const TensorShape index_shape = ctx->InputShape(2); + + OP_REQUIRES( + ctx, + TensorShapeUtils::IsVector(index_shape) && + index_shape.num_elements() == input_shape.dims(), + errors::InvalidArgument("index must be a vector with length equal to " + "the number of input dimensions")); + OP_REQUIRES( + ctx, input_shape.dims() == update_shape.dims(), + errors::InvalidArgument("input and update must have the same rank," + " input shape is ", + input_shape.DebugString(), "; update shape is ", + update_shape.DebugString())); + + xla::ComputationDataHandle result = ctx->builder()->DynamicUpdateSlice( + ctx->Input(0), ctx->Input(1), ctx->Input(2)); + ctx->SetOutput(0, result); + } +}; + +REGISTER_XLA_OP(Name("XlaDynamicUpdateSlice"), DynamicUpdateSliceOp); + +} // namespace +} // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/kernels/reduce_window_op.cc b/tensorflow/compiler/tf2xla/kernels/reduce_window_op.cc new file mode 100644 index 0000000000..cb144bea9e --- /dev/null +++ b/tensorflow/compiler/tf2xla/kernels/reduce_window_op.cc @@ -0,0 +1,135 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/tf2xla/kernels/while_op.h" + +#include "tensorflow/compiler/tf2xla/shape_util.h" +#include "tensorflow/compiler/tf2xla/xla_compiler.h" +#include "tensorflow/compiler/tf2xla/xla_op_kernel.h" +#include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "tensorflow/compiler/xla/client/computation_builder.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/op_kernel.h" + +namespace tensorflow { +namespace { + +class ReduceWindowOp : public XlaOpKernel { + public: + explicit ReduceWindowOp(OpKernelConstruction* context) + : XlaOpKernel(context) { + OP_REQUIRES_OK(context, context->GetAttr("computation", &computation_)); + OP_REQUIRES_OK(context, + context->GetAttr("window_dimensions", &window_dimensions_)); + OP_REQUIRES_OK(context, + context->GetAttr("window_strides", &window_strides_)); + OP_REQUIRES_OK(context, context->GetAttr("padding_low", &padding_low_)); + OP_REQUIRES_OK(context, context->GetAttr("padding_high", &padding_high_)); + } + + void Compile(XlaOpKernelContext* context) override { + const TensorShape input_shape = context->InputShape(0); + const DataType dtype = context->input_type(0); + + const int rank = input_shape.dims(); + OP_REQUIRES(context, rank == window_dimensions_.size(), + errors::InvalidArgument( + "The size of window_dimensions must be equal to the input " + "rank (", + window_dimensions_.size(), " vs. ", rank, ")")); + OP_REQUIRES(context, rank == window_strides_.size(), + errors::InvalidArgument( + "The size of window_strides must be equal to the input " + "rank (", + window_strides_.size(), " vs. ", rank, ")")); + OP_REQUIRES(context, rank == padding_low_.size(), + errors::InvalidArgument( + "The size of padding_low must be equal to the input " + "rank (", + padding_low_.size(), " vs. ", rank, ")")); + OP_REQUIRES(context, rank == padding_high_.size(), + errors::InvalidArgument( + "The size of padding_high must be equal to the input " + "rank (", + padding_high_.size(), " vs. ", rank, ")")); + + xla::ComputationBuilder* builder = context->builder(); + + // Build the reducer function. + XlaCompiler::Argument reducer_arg; + reducer_arg.kind = XlaCompiler::Argument::kParameter; + reducer_arg.type = dtype; + reducer_arg.shape = TensorShape(); + + XlaCompiler::CompileOptions compile_options; + compile_options.use_tuple_arg = false; + compile_options.resolve_compile_time_constants = false; + compile_options.is_entry_computation = false; + XlaCompiler::CompilationResult reducer; + OP_REQUIRES_OK(context, context->compiler()->CompileFunction( + compile_options, *computation_, + {reducer_arg, reducer_arg}, &reducer)); + + xla::Shape scalar_shape; + OP_REQUIRES_OK(context, + TensorShapeToXLAShape(dtype, TensorShape(), &scalar_shape)); + OP_REQUIRES(context, + xla::ShapeUtil::Compatible( + reducer.xla_output_shape, + xla::ShapeUtil::MakeTupleShape({scalar_shape})), + errors::InvalidArgument( + "Invalid output shape of ReduceWindow reducer. Expected ", + xla::ShapeUtil::HumanString(scalar_shape), " got ", + xla::ShapeUtil::HumanString(reducer.xla_output_shape))); + + // Wraps the reducer in a computation that unpacks the output tuple. + xla::Computation wrapper; + { + std::unique_ptr cb = + builder->CreateSubBuilder("wrapper"); + auto x = cb->Parameter(0, scalar_shape, "x"); + auto y = cb->Parameter(1, scalar_shape, "y"); + auto outputs = cb->Call(*reducer.computation, {x, y}); + cb->GetTupleElement(outputs, 0); + xla::StatusOr result = cb->Build(); + OP_REQUIRES_OK(context, result.status()); + wrapper = std::move(result.ValueOrDie()); + } + + std::vector> padding(rank); + for (int i = 0; i < rank; ++i) { + padding[i] = {padding_low_[i], padding_high_[i]}; + } + + xla::ComputationDataHandle output = builder->ReduceWindowWithGeneralPadding( + context->Input(0), context->Input(1), wrapper, window_dimensions_, + window_strides_, padding); + context->SetOutput(0, output); + } + + private: + const NameAttrList* computation_; + std::vector window_dimensions_; + std::vector window_strides_; + std::vector padding_low_; + std::vector padding_high_; + + TF_DISALLOW_COPY_AND_ASSIGN(ReduceWindowOp); +}; + +REGISTER_XLA_OP(Name("XlaReduceWindow"), ReduceWindowOp); + +} // namespace +} // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/kernels/sendrecv_ops.cc b/tensorflow/compiler/tf2xla/kernels/sendrecv_ops.cc index 5172781c0d..d079b89861 100644 --- a/tensorflow/compiler/tf2xla/kernels/sendrecv_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/sendrecv_ops.cc @@ -48,7 +48,7 @@ void SendOp::Compile(XlaOpKernelContext* ctx) { ctx->builder()->Send(ctx->Input(0), channel); } -REGISTER_XLA_OP(Name("_XLASend"), SendOp); +REGISTER_XLA_OP(Name("XlaSend"), SendOp); class RecvOp : public XlaOpKernel { public: @@ -68,7 +68,7 @@ RecvOp::RecvOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) { TensorShape tensor_shape; DataType dtype; OP_REQUIRES_OK(ctx, ctx->GetAttr("shape", &tensor_shape)); - OP_REQUIRES_OK(ctx, ctx->GetAttr("T", &dtype)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("dtype", &dtype)); OP_REQUIRES_OK(ctx, TensorShapeToXLAShape(dtype, tensor_shape, &shape_)); } @@ -79,7 +79,7 @@ void RecvOp::Compile(XlaOpKernelContext* ctx) { ctx->SetOutput(0, ctx->builder()->Recv(shape_, channel)); } -REGISTER_XLA_OP(Name("_XLARecv"), RecvOp); +REGISTER_XLA_OP(Name("XlaRecv"), RecvOp); } // namespace } // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/ops/BUILD b/tensorflow/compiler/tf2xla/ops/BUILD index aeb743a663..bb9168fa35 100644 --- a/tensorflow/compiler/tf2xla/ops/BUILD +++ b/tensorflow/compiler/tf2xla/ops/BUILD @@ -7,17 +7,13 @@ licenses(["notice"]) # Apache 2.0 load("//tensorflow:tensorflow.bzl", "tf_gen_op_wrapper_py") cc_library( - name = "functional_ops", - srcs = ["functional_ops.cc"], - deps = [ - "//tensorflow/core:framework", + name = "xla_ops", + srcs = [ + "dynamic_slice_ops.cc", + "functional_ops.cc", + "reduce_window_op.cc", + "sendrecv_ops.cc", ], - alwayslink = 1, -) - -cc_library( - name = "sendrecv_ops", - srcs = ["sendrecv_ops.cc"], deps = [ "//tensorflow/core:framework", ], @@ -25,17 +21,9 @@ cc_library( ) tf_gen_op_wrapper_py( - name = "gen_functional_ops", - out = "gen_functional_ops.py", - deps = [ - ":functional_ops", - ], -) - -tf_gen_op_wrapper_py( - name = "gen_sendrecv_ops", - out = "gen_sendrecv_ops.py", + name = "gen_xla_ops", + out = "gen_xla_ops.py", deps = [ - ":sendrecv_ops", + ":xla_ops", ], ) diff --git a/tensorflow/compiler/tf2xla/ops/dynamic_slice_ops.cc b/tensorflow/compiler/tf2xla/ops/dynamic_slice_ops.cc new file mode 100644 index 0000000000..d6c0edbb88 --- /dev/null +++ b/tensorflow/compiler/tf2xla/ops/dynamic_slice_ops.cc @@ -0,0 +1,49 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/framework/common_shape_fns.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/shape_inference.h" + +namespace tensorflow { + +REGISTER_OP("XlaDynamicUpdateSlice") + .Input("input: T") + .Input("update: T") + .Input("indices: Tindices") + .Output("output: T") + .Attr("T: type") + .Attr("Tindices: {int32, int64}") + .SetShapeFn(shape_inference::UnchangedShape) + .Doc(R"doc( +Wraps the XLA DynamicUpdateSlice operator, documented at + https://www.tensorflow.org/performance/xla/operation_semantics#dynamicupdateslice +. + +XlaDynamicUpdateSlice generates a result which is the value of the `input` +operand, with a slice update overwritten at `indices`. The shape of `update` +determines the shape of the sub-array of the result which is updated. The shape +of indices must be rank == 1, with dimension size equal to the rank of `input`. + +Handling of out-of-bounds slice indices is implementation-defined. + +input: A `Tensor` of type T. +indices: A vector of indices into `input`. Must have length equal to the rank of + `input`. +update: A `Tensor` of type T. Same rank as `input`. +output: A `Tensor` of type T. +)doc"); + +} // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/ops/reduce_window_op.cc b/tensorflow/compiler/tf2xla/ops/reduce_window_op.cc new file mode 100644 index 0000000000..d9af982adc --- /dev/null +++ b/tensorflow/compiler/tf2xla/ops/reduce_window_op.cc @@ -0,0 +1,45 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/framework/common_shape_fns.h" +#include "tensorflow/core/framework/op.h" + +namespace tensorflow { + +REGISTER_OP("XlaReduceWindow") + .Input("input: T") + .Input("init_value: T") + .Attr("T: numbertype") + .Attr("computation: func") + .Attr("window_dimensions: list(int)") + .Attr("window_strides: list(int)") + .Attr("padding_low: list(int)") + .Attr("padding_high: list(int)") + .Output("output: T") + .SetShapeFn(shape_inference::UnknownShape) + .Doc(R"doc( +Wraps the XLA ReduceWindow operator, documented at + https://www.tensorflow.org/performance/xla/operation_semantics#reducewindow . + +input: the input tensor +init_value: a scalar representing the initial value for the reduction +computation: a reducer function to apply +window_dimensions: the shape of the window +window_strides: the inter-window strides +padding_low: the padding to apply at the start of each input dimensions +padding_high: the padding to apply at the end of each input dimension. +)doc"); + +} // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/ops/sendrecv_ops.cc b/tensorflow/compiler/tf2xla/ops/sendrecv_ops.cc index 4b41c16a8b..7ec7b50e90 100644 --- a/tensorflow/compiler/tf2xla/ops/sendrecv_ops.cc +++ b/tensorflow/compiler/tf2xla/ops/sendrecv_ops.cc @@ -18,22 +18,24 @@ limitations under the License. namespace tensorflow { -REGISTER_OP("_XLASend") +REGISTER_OP("XlaSend") .Input("tensor: T") .Attr("T: type") .Attr("tensor_name: string") .SetIsStateful() .SetShapeFn(shape_inference::UnknownShape) .Doc(R"doc( -Sends the named tensor to another XLA computation. +Sends the named tensor to another XLA computation. Wraps the XLA Send operator +documented at + https://www.tensorflow.org/performance/xla/operation_semantics#send . tensor: The tensor to send. -tensor_name: The name of the tensor to send. +tensor_name: A string key that identifies the channel. )doc"); -REGISTER_OP("_XLARecv") - .Output("tensor: T") - .Attr("T: type") +REGISTER_OP("XlaRecv") + .Output("tensor: dtype") + .Attr("dtype: type") .Attr("tensor_name: string") .Attr("shape: shape") .SetIsStateful() @@ -46,11 +48,14 @@ REGISTER_OP("_XLARecv") return Status::OK(); }) .Doc(R"doc( -Receives the named tensor from another XLA computation. +Receives the named tensor from another XLA computation. Wraps the XLA Recv +operator documented at + https://www.tensorflow.org/performance/xla/operation_semantics#recv . tensor: The tensor to receive. -tensor_name: The name of the tensor to receive. -shape: The shape of the input tensor. +dtype: The type of the tensor. +tensor_name: A string key that identifies the channel. +shape: The shape of the tensor. )doc"); } // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/python/BUILD b/tensorflow/compiler/tf2xla/python/BUILD index f0a2ef0651..42b6292f79 100644 --- a/tensorflow/compiler/tf2xla/python/BUILD +++ b/tensorflow/compiler/tf2xla/python/BUILD @@ -22,3 +22,11 @@ tf_py_clif_cc( "//tensorflow/compiler/tf2xla:xla_compiler", ], ) + +py_library( + name = "xla", + srcs = ["xla.py"], + deps = [ + "//tensorflow/compiler/tf2xla/ops:gen_xla_ops", + ], +) diff --git a/tensorflow/compiler/tf2xla/python/xla.py b/tensorflow/compiler/tf2xla/python/xla.py new file mode 100644 index 0000000000..e5ce65bec9 --- /dev/null +++ b/tensorflow/compiler/tf2xla/python/xla.py @@ -0,0 +1,80 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Experimental library that exposes XLA operations directly in TensorFlow. + +It is sometimes useful to be able to build HLO programs directly from +TensorFlow. This file provides Tensorflow operators that map as closely as +possible to HLO operators. + +There is no promise of backward or forward compatibility for operators defined +in this module. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.compiler.tf2xla.ops import gen_xla_ops + +# TODO(phawkins): provide wrappers for all XLA operators. + +dynamic_update_slice = gen_xla_ops.xla_dynamic_update_slice + + +def reduce_window(operand, + init, + reducer, + window_dimensions, + window_strides=None, + padding=None, + name=None): + """Wraps the XLA ReduceWindow operator. + + ReduceWindow is documented at + https://www.tensorflow.org/performance/xla/operation_semantics#reducewindow . + + Args: + operand: the input tensor + init: a scalar tensor representing the initial value for the reduction + reducer: a reduction function that combines a pair of scalars. + window_dimensions: shape of the window, as a list of integers + window_strides: inter-window strides, as a list of integers. Optional; + if omitted, defaults to strides of 1. + padding: padding to apply to 'operand'. List of (low, high) pairs of + integers that specify the padding to apply before and after each + dimension. Optional; if omitted, defaults to no padding. + name: the operator name, or None. + Returns: + A tensor that represents the output of the reduce_window operator. + """ + window_strides = window_strides or [1] * len(window_dimensions) + padding = padding or [(0, 0)] * len(window_dimensions) + padding_low = [x for (x, _) in padding] + padding_high = [y for (_, y) in padding] + return gen_xla_ops.xla_reduce_window( + operand, + init, + reducer, + window_dimensions, + window_strides, + padding_low, + padding_high, + name=name) + + +recv = gen_xla_ops.xla_recv +send = gen_xla_ops.xla_send + +while_loop = gen_xla_ops.xla_while -- GitLab From 2d07eb5109ff3987681f6bac07d1b322dab5950b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 11:16:36 -0700 Subject: [PATCH 331/791] Fixing output alternatives PiperOrigin-RevId: 192795596 --- .../boosted_trees/estimator_batch/estimator_utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator_utils.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator_utils.py index c9cf4ae25a..48a7f85ead 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/estimator_utils.py +++ b/tensorflow/contrib/boosted_trees/estimator_batch/estimator_utils.py @@ -58,9 +58,12 @@ def _export_outputs_to_output_alternatives(export_outputs): return None -def estimator_spec_to_model_fn_ops(estimator_spec): - alternatives = _export_outputs_to_output_alternatives( - estimator_spec.export_outputs) +def estimator_spec_to_model_fn_ops(estimator_spec, export_alternatives=False): + if export_alternatives: + alternatives = _export_outputs_to_output_alternatives( + estimator_spec.export_outputs) + else: + alternatives = [] return model_fn.ModelFnOps( mode=_core_mode_to_contrib_mode(estimator_spec.mode), -- GitLab From 6942b87c255e9bce9289f87ff6894d198fcab6f4 Mon Sep 17 00:00:00 2001 From: Akshay Modi Date: Fri, 13 Apr 2018 11:09:58 -0700 Subject: [PATCH 332/791] Use eager compatible wrappers in load_library for custom ops --- tensorflow/python/BUILD | 1 + tensorflow/python/framework/load_library.py | 2 +- tensorflow/python/framework/python_op_gen.i | 8 ++-- .../tools/ci_build/builds/test_user_ops.sh | 41 +++++++++++-------- 4 files changed, 30 insertions(+), 22 deletions(-) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index a683c8cfa6..579a8faaad 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -3482,6 +3482,7 @@ tf_py_wrap_cc( "//tensorflow/core/profiler/internal:print_model_analysis", "//tensorflow/tools/graph_transforms:transform_graph_lib", "//tensorflow/python/eager:pywrap_tfe_lib", + "//tensorflow/python/eager:python_eager_op_gen", "//util/python:python_headers", ] + (tf_additional_lib_deps() + tf_additional_plugin_deps() + diff --git a/tensorflow/python/framework/load_library.py b/tensorflow/python/framework/load_library.py index 535c6017f5..9a8477debb 100644 --- a/tensorflow/python/framework/load_library.py +++ b/tensorflow/python/framework/load_library.py @@ -58,7 +58,7 @@ def load_op_library(library_filename): op_list_str = py_tf.TF_GetOpList(lib_handle) op_list = op_def_pb2.OpList() op_list.ParseFromString(compat.as_bytes(op_list_str)) - wrappers = py_tf.GetPythonWrappers(op_list_str) + wrappers = py_tf.GetEagerPythonWrappers(op_list_str) # Delete the library handle to release any memory held in C # that are no longer needed. diff --git a/tensorflow/python/framework/python_op_gen.i b/tensorflow/python/framework/python_op_gen.i index 26ec4e8e66..e39c425b05 100644 --- a/tensorflow/python/framework/python_op_gen.i +++ b/tensorflow/python/framework/python_op_gen.i @@ -16,10 +16,10 @@ limitations under the License. %include "tensorflow/python/platform/base.i" %{ -#include "tensorflow/python/framework/python_op_gen.h" +#include "tensorflow/python/eager/python_eager_op_gen.h" %} -// Input typemap for GetPythonWrappers. +// Input typemap for GetEagerPythonWrappers. // Accepts a python object of 'bytes' type, and converts it to // a const char* pointer and size_t length. The default typemap // going from python bytes to const char* tries to decode the @@ -37,5 +37,5 @@ limitations under the License. %ignoreall; -%unignore tensorflow::GetPythonWrappers; -%include "tensorflow/python/framework/python_op_gen.h" +%unignore tensorflow::GetEagerPythonWrappers; +%include "third_party/tensorflow/python/eager/python_eager_op_gen.h" diff --git a/tensorflow/tools/ci_build/builds/test_user_ops.sh b/tensorflow/tools/ci_build/builds/test_user_ops.sh index caa3a40817..c342367bac 100755 --- a/tensorflow/tools/ci_build/builds/test_user_ops.sh +++ b/tensorflow/tools/ci_build/builds/test_user_ops.sh @@ -213,27 +213,34 @@ USER_OP=$(echo "${USER_OP_SO}" | sed -e 's/\.so//') echo "Invoking user op ${USER_OP} defined in file ${USER_OP_SO} "\ "via pip installation" -ORIG_OUTPUT=$("${PYTHON_BIN_PATH}" -c "import tensorflow as tf; print(tf.Session('').run(tf.load_op_library('./${USER_OP_SO}').${USER_OP}(${OP_INPUT})))") - -# Format OUTPUT for analysis -if [[ -z $(echo "${ORIG_OUTPUT}" | grep -o ',') ]]; then - if [[ ${IS_MAC} == "1" ]]; then - OUTPUT=$(echo "${ORIG_OUTPUT}" | sed -E -e 's/[ \t]+/,/g') +function run_op() { + local ORIG_OUTPUT=$1 + local ADDITIONAL_LOG=$2 + + # Format OUTPUT for analysis + if [[ -z $(echo "${ORIG_OUTPUT}" | grep -o ',') ]]; then + if [[ ${IS_MAC} == "1" ]]; then + local OUTPUT=$(echo "${ORIG_OUTPUT}" | sed -E -e 's/[ \t]+/,/g') + else + local OUTPUT=$(echo "${ORIG_OUTPUT}" | sed -r -e 's/[ \t]+/,/g') + fi else - OUTPUT=$(echo "${ORIG_OUTPUT}" | sed -r -e 's/[ \t]+/,/g') + local OUTPUT="${ORIG_OUTPUT}" fi -else - OUTPUT="${ORIG_OUTPUT}" -fi -EQUALS_EXPECTED=$("${PYTHON_BIN_PATH}" -c "print(${OUTPUT} == ${EXPECTED_OUTPUT})") + local EQUALS_EXPECTED=$("${PYTHON_BIN_PATH}" -c "print(${OUTPUT} == ${EXPECTED_OUTPUT})") -if [[ "${EQUALS_EXPECTED}" != "True" ]]; then - die "FAILED: Output from user op (${OUTPUT}) does not match expected "\ -"output ${EXPECTED_OUTPUT}" -else - echo "Output from user op (${OUTPUT}) matches expected output" -fi + if [[ "${EQUALS_EXPECTED}" != "True" ]]; then + local ERROR="FAILED: Output from user op (${OUTPUT}) does not match expected "\ + "output ${EXPECTED_OUTPUT}"${ADDITIONAL_LOG} + die ${ERROR} + else + echo "Output from user op (${OUTPUT}) matches expected output" + fi +} + +run_op $("${PYTHON_BIN_PATH}" -c "import tensorflow as tf; print(tf.Session('').run(tf.load_op_library('./${USER_OP_SO}').${USER_OP}(${OP_INPUT})))") +run_op $("${PYTHON_BIN_PATH}" -c "import tensorflow as tf; tf.enable_eager_execution(); print(tf.load_op_library('./${USER_OP_SO}').${USER_OP}(${OP_INPUT}))") " in eager mode" popd -- GitLab From 889a63b641f3b6204c8a772fb42c3e256166cac9 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 13 Apr 2018 11:26:03 -0700 Subject: [PATCH 333/791] Add deprecation args decoration for tf.squeeze (#18495) * Add deprecation args decoration with tf.squeeze This fix adds deprecation args decoration with tf.squeeze, with deprecates `squeeze_dims` with `axis`. Signed-off-by: Yong Tang * Enhancement with deprecated_argument_lookup Signed-off-by: Yong Tang --- tensorflow/python/ops/array_ops.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index 9e136937f6..ceeabe090d 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -2578,6 +2578,8 @@ def sequence_mask(lengths, maxlen=None, dtype=dtypes.bool, name=None): @tf_export("squeeze") +@deprecation.deprecated_args(None, "Use the `axis` argument instead", + "squeeze_dims") def squeeze(input, axis=None, name=None, squeeze_dims=None): # pylint: disable=redefined-builtin """Removes dimensions of size 1 from the shape of a tensor. @@ -2618,10 +2620,8 @@ def squeeze(input, axis=None, name=None, squeeze_dims=None): Raises: ValueError: When both `squeeze_dims` and `axis` are specified. """ - if squeeze_dims is not None: - if axis is not None: - raise ValueError("Cannot specify both 'squeeze_dims' and 'axis'") - axis = squeeze_dims + axis = deprecation.deprecated_argument_lookup( + "axis", axis, "squeeze_dims", squeeze_dims) if np.isscalar(axis): axis = [axis] return gen_array_ops.squeeze(input, axis, name) -- GitLab From 584d072537ff350f21ed973e64ed67a3d0d943e3 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 13 Apr 2018 11:26:39 -0700 Subject: [PATCH 334/791] Fix warnings in `nn.sampled_softmax_loss` (#18494) * Fix warnings in `nn.sampled_softmax_loss` The softmax_cross_entropy_with_logits has been deprecated and replaced with softmax_cross_entropy_with_logits_v2. This causes nn.sampled_softmax_loss to always generate a WANRING whenever called. This fix replaces `softmax_cross_entropy_with_logits` with `softmax_cross_entropy_with_logits_v2` and maintains the existing behavior to fix the warning. Signed-off-by: Yong Tang * Pylint fix for line too long Signed-off-by: Yong Tang --- tensorflow/python/ops/nn_impl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py index 47cc4da7f2..1715e5b36a 100644 --- a/tensorflow/python/ops/nn_impl.py +++ b/tensorflow/python/ops/nn_impl.py @@ -1340,7 +1340,8 @@ def sampled_softmax_loss(weights, partition_strategy=partition_strategy, name=name, seed=seed) - sampled_losses = nn_ops.softmax_cross_entropy_with_logits( + labels = array_ops.stop_gradient(labels, name="labels_stop_gradient") + sampled_losses = nn_ops.softmax_cross_entropy_with_logits_v2( labels=labels, logits=logits) # sampled_losses is a [batch_size] tensor. return sampled_losses -- GitLab From 988ad74476250eee70227349b5f1eabc86d22833 Mon Sep 17 00:00:00 2001 From: Akshay Modi Date: Fri, 13 Apr 2018 11:29:31 -0700 Subject: [PATCH 335/791] Not in third_party --- tensorflow/python/framework/python_op_gen.i | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/framework/python_op_gen.i b/tensorflow/python/framework/python_op_gen.i index e39c425b05..efcce2f209 100644 --- a/tensorflow/python/framework/python_op_gen.i +++ b/tensorflow/python/framework/python_op_gen.i @@ -38,4 +38,4 @@ limitations under the License. %ignoreall; %unignore tensorflow::GetEagerPythonWrappers; -%include "third_party/tensorflow/python/eager/python_eager_op_gen.h" +%include "tensorflow/python/eager/python_eager_op_gen.h" -- GitLab From 692a71da6aad55dcaa597633aaf88de8322ca8ab Mon Sep 17 00:00:00 2001 From: Yu-Cheng Ling Date: Fri, 13 Apr 2018 11:33:07 -0700 Subject: [PATCH 336/791] Fix the broken TFLite iOS example. (#18483) The demo app is only relying on CocoaPod now, but it's incorrectly configured to use the headers on Github. It crashes the app when the header is different between Github and CocoaPod. --- .../tflite_camera_example.xcodeproj/project.pbxproj | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tensorflow/contrib/lite/examples/ios/camera/tflite_camera_example.xcodeproj/project.pbxproj b/tensorflow/contrib/lite/examples/ios/camera/tflite_camera_example.xcodeproj/project.pbxproj index b0236e9c60..98d3b5bb8a 100644 --- a/tensorflow/contrib/lite/examples/ios/camera/tflite_camera_example.xcodeproj/project.pbxproj +++ b/tensorflow/contrib/lite/examples/ios/camera/tflite_camera_example.xcodeproj/project.pbxproj @@ -326,10 +326,6 @@ GCC_WARN_UNUSED_VARIABLE = YES; HEADER_SEARCH_PATHS = ( "$(inherited)", - ../../../../../../, - ../../../downloads/flatbuffers/include/, - ../../../downloads/eigen/, - ../../../downloads/, ); IPHONEOS_DEPLOYMENT_TARGET = 8.0; MTL_ENABLE_DEBUG_INFO = YES; @@ -373,10 +369,6 @@ GCC_WARN_UNUSED_VARIABLE = YES; HEADER_SEARCH_PATHS = ( "$(inherited)", - ../../../../../../, - ../../../downloads/flatbuffers/include/, - ../../../downloads/eigen/, - ../../../downloads/, ); IPHONEOS_DEPLOYMENT_TARGET = 8.0; MTL_ENABLE_DEBUG_INFO = NO; -- GitLab From 0c2ca00e1082ab2692af68af183083e41393f6c4 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 13 Apr 2018 11:38:43 -0700 Subject: [PATCH 337/791] Fix crash when invalid dtype was passed (#18481) * Fix crash when invalid dtype was passed This fix tries to address the issue raised in 18474 where crash may happen if invalid dtype (e.g., `"[,]"`) is passed to `tf.constant(tf.string, "[,]")`. The crash happens during the comparision of `"[,]"` and numpy dtype candidate (e.g., `np.dtype([("qint8", np.int8, 1)])`: ``` >>> import numpy as np >>> np.dtype([("qint8", np.int8, 1)]) == "[,]" Segmentation fault: 11 ``` This fix adds a type check to make sure the type of the passed dtype is either numpy.dtype or type. This fix fixes 18474. Signed-off-by: Yong Tang * Add test case for invalid type to tf.constant Signed-off-by: Yong Tang --- tensorflow/python/framework/dtypes.py | 14 ++++++++------ tensorflow/python/kernel_tests/constant_op_test.py | 5 +++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/framework/dtypes.py b/tensorflow/python/framework/dtypes.py index 5efda44a5f..eda713641d 100644 --- a/tensorflow/python/framework/dtypes.py +++ b/tensorflow/python/framework/dtypes.py @@ -699,11 +699,13 @@ def as_dtype(type_value): if type_value.type == np.string_ or type_value.type == np.unicode_: return string - for key, val in _NP_TO_TF: - try: - if key == type_value: - return val - except TypeError as e: - raise TypeError("Cannot convert {} to a dtype. {}".format(type_value, e)) + if isinstance(type_value, (type, np.dtype)): + for key, val in _NP_TO_TF: + try: + if key == type_value: + return val + except TypeError as e: + raise TypeError("Cannot convert {} to a dtype. {}".format( + type_value, e)) raise TypeError("Cannot convert value %r to a TensorFlow DType." % type_value) diff --git a/tensorflow/python/kernel_tests/constant_op_test.py b/tensorflow/python/kernel_tests/constant_op_test.py index 749313b00d..107ee37fab 100644 --- a/tensorflow/python/kernel_tests/constant_op_test.py +++ b/tensorflow/python/kernel_tests/constant_op_test.py @@ -65,6 +65,11 @@ class ConstantTest(test.TestCase): self._testCpu(x) self._testGpu(x) + def testInvalidDType(self): + # Test case for GitHub issue 18474 + with self.assertRaises(TypeError): + constant_op.constant(dtypes_lib.string, "[,]") + def testBFloat16(self): bfloat16 = dtypes_lib.bfloat16.as_numpy_dtype self._testAll(np.arange(-15, 15).reshape([2, 3, 5]).astype(bfloat16)) -- GitLab From 6c22bbdda41d839cb9e1f7803533c571596ea4ee Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 13 Apr 2018 11:47:02 -0700 Subject: [PATCH 338/791] Fix warnings in tf.distributions.Categorical (#18468) In tf.distributions.Categorical dimension was used with argmax. As dimension has been deprecated this generates a warning. This fix fixes the warning by changing to axis. Signed-off-by: Yong Tang --- tensorflow/python/ops/distributions/categorical.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/ops/distributions/categorical.py b/tensorflow/python/ops/distributions/categorical.py index 9161e3fa9f..995dd9ca2a 100644 --- a/tensorflow/python/ops/distributions/categorical.py +++ b/tensorflow/python/ops/distributions/categorical.py @@ -311,7 +311,7 @@ class Categorical(distribution.Distribution): nn_ops.log_softmax(self.logits) * self.probs, axis=-1) def _mode(self): - ret = math_ops.argmax(self.logits, dimension=self._batch_rank) + ret = math_ops.argmax(self.logits, axis=self._batch_rank) ret = math_ops.cast(ret, self.dtype) ret.set_shape(self.batch_shape) return ret -- GitLab From 7e0db0fe4992c466f758338183dfa0636c61a36b Mon Sep 17 00:00:00 2001 From: James Wexler Date: Fri, 13 Apr 2018 15:18:17 -0400 Subject: [PATCH 339/791] fix build file format --- tensorflow/core/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index ab25283cc4..46da23f6f9 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -256,8 +256,8 @@ proto_library( closure_proto_library( name = "example_protos_closure", - deps = [":example_protos"], visibility = ["//visibility:public"], + deps = [":example_protos"], ) exports_files([ -- GitLab From be328931086e212a87bac26ccff021b51863d875 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 12:18:53 -0700 Subject: [PATCH 340/791] Expose tf.decode_compressed to the public API. PiperOrigin-RevId: 192805605 --- .../core/api_def/python_api/api_def_DecodeCompressed.pbtxt | 4 ---- tensorflow/tools/api/golden/tensorflow.pbtxt | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) delete mode 100644 tensorflow/core/api_def/python_api/api_def_DecodeCompressed.pbtxt diff --git a/tensorflow/core/api_def/python_api/api_def_DecodeCompressed.pbtxt b/tensorflow/core/api_def/python_api/api_def_DecodeCompressed.pbtxt deleted file mode 100644 index f0b7539918..0000000000 --- a/tensorflow/core/api_def/python_api/api_def_DecodeCompressed.pbtxt +++ /dev/null @@ -1,4 +0,0 @@ -op { - graph_op_name: "DecodeCompressed" - visibility: HIDDEN -} diff --git a/tensorflow/tools/api/golden/tensorflow.pbtxt b/tensorflow/tools/api/golden/tensorflow.pbtxt index be64fd19d8..c66249999f 100644 --- a/tensorflow/tools/api/golden/tensorflow.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.pbtxt @@ -912,6 +912,10 @@ tf_module { name: "decode_base64" argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "decode_compressed" + argspec: "args=[\'bytes\', \'compression_type\', \'name\'], varargs=None, keywords=None, defaults=[\'\', \'None\'], " + } member_method { name: "decode_csv" argspec: "args=[\'records\', \'record_defaults\', \'field_delim\', \'use_quote_delim\', \'name\', \'na_value\', \'select_cols\'], varargs=None, keywords=None, defaults=[\',\', \'True\', \'None\', \'\', \'None\'], " -- GitLab From cb3cd61be2301202731e1157c3ee957d26f9695e Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 12:35:32 -0700 Subject: [PATCH 341/791] [XLA] Redesign: add a method that creates fake data for XlaComputation. PiperOrigin-RevId: 192807851 --- tensorflow/compiler/xla/client/lib/BUILD | 1 + tensorflow/compiler/xla/client/lib/testing.cc | 16 ++++++++++++++++ tensorflow/compiler/xla/client/lib/testing.h | 7 +++++++ 3 files changed, 24 insertions(+) diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD index f4673a8204..59c4a53c05 100644 --- a/tensorflow/compiler/xla/client/lib/BUILD +++ b/tensorflow/compiler/xla/client/lib/BUILD @@ -46,6 +46,7 @@ cc_library( "//tensorflow/compiler/xla/client:computation", "//tensorflow/compiler/xla/client:computation_builder", "//tensorflow/compiler/xla/client:global_data", + "//tensorflow/compiler/xla/client/xla_client:xla_computation", "//tensorflow/compiler/xla/tests:test_utils", "//tensorflow/core:lib", ], diff --git a/tensorflow/compiler/xla/client/lib/testing.cc b/tensorflow/compiler/xla/client/lib/testing.cc index b63a1465ea..311dc4bdd7 100644 --- a/tensorflow/compiler/xla/client/lib/testing.cc +++ b/tensorflow/compiler/xla/client/lib/testing.cc @@ -111,4 +111,20 @@ std::vector> MakeFakeArgumentsOrDie( return fake_arguments; } +std::vector> MakeFakeArgumentsOrDie( + const XlaComputation& computation, Client* client) { + CHECK(computation.proto().has_program_shape()) + << "Computation should have progran shape."; + auto program_shape = computation.proto().program_shape(); + + // For every (unbound) parameter that the computation wants, we manufacture + // some arbitrary data so that we can invoke the computation. + std::vector> fake_arguments; + for (const Shape& parameter : program_shape.parameters()) { + fake_arguments.push_back(MakeFakeDataOrDie(parameter, client)); + } + + return fake_arguments; +} + } // namespace xla diff --git a/tensorflow/compiler/xla/client/lib/testing.h b/tensorflow/compiler/xla/client/lib/testing.h index 7e640d1307..1dc2622972 100644 --- a/tensorflow/compiler/xla/client/lib/testing.h +++ b/tensorflow/compiler/xla/client/lib/testing.h @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow/compiler/xla/client/client.h" #include "tensorflow/compiler/xla/client/computation.h" #include "tensorflow/compiler/xla/client/global_data.h" +#include "tensorflow/compiler/xla/client/xla_client/xla_computation.h" #include "tensorflow/compiler/xla/xla_data.pb.h" namespace xla { @@ -38,6 +39,12 @@ std::unique_ptr MakeFakeDataOrDie(const Shape& shape, std::vector> MakeFakeArgumentsOrDie( const Computation& computation, Client* client); +// Returns vector of GlobalData handles of fake data (created using +// MakeFakeDataOrDie) that are correctly shaped arguments for the given +// xla computation. +std::vector> MakeFakeArgumentsOrDie( + const XlaComputation& computation, Client* client); + } // namespace xla #endif // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_TESTING_H_ -- GitLab From ec6003aee63a8eabace3c211e15d9587a405c1f0 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 12:37:04 -0700 Subject: [PATCH 342/791] [XLA] Redesign: add a constructor: XlaComputation(HloModuleProto). PiperOrigin-RevId: 192808038 --- tensorflow/compiler/xla/client/xla_client/xla_computation.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/compiler/xla/client/xla_client/xla_computation.h b/tensorflow/compiler/xla/client/xla_client/xla_computation.h index 7182908666..085fabd56d 100644 --- a/tensorflow/compiler/xla/client/xla_client/xla_computation.h +++ b/tensorflow/compiler/xla/client/xla_client/xla_computation.h @@ -30,6 +30,8 @@ namespace xla { class XlaComputation { public: XlaComputation() : unique_id_(-1) {} + XlaComputation(const HloModuleProto& proto) + : unique_id_(proto.id()), proto_(proto) {} XlaComputation(const XlaComputation&) = delete; XlaComputation& operator=(const XlaComputation&) = delete; -- GitLab From f6c5cd435df9c64e79ad0f6434b619d4517e740a Mon Sep 17 00:00:00 2001 From: James Wexler Date: Fri, 13 Apr 2018 12:44:41 -0700 Subject: [PATCH 343/791] Remove closure_js_proto_library rule for tf.example protos. PiperOrigin-RevId: 192809073 --- tensorflow/core/BUILD | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index c461f9ed2f..7ea8a38834 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -70,10 +70,6 @@ package(default_visibility = [ licenses(["notice"]) # Apache 2.0 -load( - "@io_bazel_rules_closure//closure:defs.bzl", - "closure_js_proto_library", -) load( "//tensorflow:tensorflow.bzl", "full_path", @@ -249,15 +245,6 @@ tf_nano_proto_library( deps = [":protos_all_cc"], ) -closure_js_proto_library( - name = "example_js_protos", - srcs = [ - "example/example.proto", - "example/feature.proto", - ], - visibility = ["//visibility:public"], -) - exports_files([ "framework/types.proto", ]) -- GitLab From 544ae7128d5684644319d529de35a3f761ba5385 Mon Sep 17 00:00:00 2001 From: Peng Yu Date: Fri, 13 Apr 2018 16:07:36 -0400 Subject: [PATCH 344/791] Add myself into code ownder for tensor_forest --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index 007a304c3e..b9f0313cc6 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -45,7 +45,7 @@ # /tensorflow/contrib/session_bundle/ @nfiedel @sukritiramesh # /tensorflow/contrib/slim/ @sguada @thenbasilmanran # /tensorflow/contrib/stateless/ @girving -# /tensorflow/contrib/tensor_forest/ @gilberthendry @thomascolthurst +# /tensorflow/contrib/tensor_forest/ @gilberthendry @thomascolthurst @yupbank # /tensorflow/contrib/testing/ @dandelionmane # /tensorflow/contrib/timeseries/ @allenlavoie # /tensorflow/contrib/tpu/ @frankchn @saeta @jhseu -- GitLab From 460a8b6a5df176412c0d261d91eccdc32e9d39f1 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 13:40:28 -0700 Subject: [PATCH 345/791] Support scalar mean in resolve_batch_normalization PiperOrigin-RevId: 192816848 --- .../resolve_batch_normalization.cc | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/lite/toco/graph_transformations/resolve_batch_normalization.cc b/tensorflow/contrib/lite/toco/graph_transformations/resolve_batch_normalization.cc index fb109eb91b..2b3ee36ad1 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/resolve_batch_normalization.cc +++ b/tensorflow/contrib/lite/toco/graph_transformations/resolve_batch_normalization.cc @@ -33,7 +33,7 @@ bool ResolveBatchNormalization::Run(Model* model, std::size_t op_index) { const auto* bn_op = static_cast(bn_it->get()); - const auto& mean_array = model->GetArray(bn_op->inputs[1]); + auto& mean_array = model->GetArray(bn_op->inputs[1]); const auto& multiplier_array = model->GetArray(bn_op->inputs[2]); const auto& offset_array = model->GetArray(bn_op->inputs[3]); @@ -49,6 +49,13 @@ bool ResolveBatchNormalization::Run(Model* model, std::size_t op_index) { CHECK(multiplier_array.data_type == ArrayDataType::kFloat); CHECK(offset_array.data_type == ArrayDataType::kFloat); + // This graph transformations will need to address constant buffers below, + // so we need to exit early if these buffers don't exist (i.e. if the params + // haven't yet been resolved as constants). + if (!mean_array.buffer || !multiplier_array.buffer || !offset_array.buffer) { + return false; + } + // Create the new Mul, Add operators auto* mul_op = new MulOperator; auto* add_op = new AddOperator; @@ -80,9 +87,15 @@ bool ResolveBatchNormalization::Run(Model* model, std::size_t op_index) { DCHECK_EQ(bn_it->get(), bn_op); // Create the new param arrays - const auto& mean_shape = mean_array.shape(); + auto& mean_shape = *mean_array.mutable_shape(); const auto& multiplier_shape = multiplier_array.shape(); const auto& offset_shape = offset_array.shape(); + if (mean_shape.dims().empty()) { + *mean_shape.mutable_dims() = multiplier_shape.dims(); + auto& data = mean_array.GetMutableBuffer().data; + CHECK_EQ(data.size(), 1); + data.resize(RequiredBufferSizeForShape(mean_shape), data[0]); + } CHECK(mean_shape.dims() == multiplier_shape.dims()); CHECK(mean_shape.dims() == offset_shape.dims()); const auto& param_shape = mean_shape; -- GitLab From 6a2d781e2c529511442e1818d23334d89b171cf2 Mon Sep 17 00:00:00 2001 From: Anna R Date: Fri, 13 Apr 2018 14:09:58 -0700 Subject: [PATCH 346/791] Internal change. PiperOrigin-RevId: 192821482 --- tensorflow/workspace.bzl | 8 ++++---- third_party/llvm/llvm.BUILD | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 72f446d359..85bd1ea28b 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -462,11 +462,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""): tf_http_archive( name = "llvm", urls = [ - "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/7e78daafdd22f3f17720a103d29d89590534004e.tar.gz", - "https://github.com/llvm-mirror/llvm/archive/7e78daafdd22f3f17720a103d29d89590534004e.tar.gz", + "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/15535accd9e1e9d7772202ce51c8428c1994a04b.tar.gz", + "https://github.com/llvm-mirror/llvm/archive/15535accd9e1e9d7772202ce51c8428c1994a04b.tar.gz", ], - sha256 = "a6d94bd9de23515a1e3792a830421e3885977ea43d03427cdbe68f98cb7e0045", - strip_prefix = "llvm-7e78daafdd22f3f17720a103d29d89590534004e", + sha256 = "3470c2dde055dc974e859e707aa6cd1d22eadd4f3a1f282e74c3cf1f7dc9510a", + strip_prefix = "llvm-15535accd9e1e9d7772202ce51c8428c1994a04b", build_file = clean_dep("//third_party/llvm:llvm.BUILD"), ) diff --git a/third_party/llvm/llvm.BUILD b/third_party/llvm/llvm.BUILD index 075b46896e..097bbf5d42 100644 --- a/third_party/llvm/llvm.BUILD +++ b/third_party/llvm/llvm.BUILD @@ -2053,6 +2053,7 @@ cc_library( "include/llvm/Target/*.def", "include/llvm/Target/*.inc", "include/llvm/CodeGen/*.def", + "include/llvm/CodeGen/*.inc", ]), deps = [ ":analysis", -- GitLab From 92f870d1a95cb598c0fec9ff1f5c0cf95fa42eae Mon Sep 17 00:00:00 2001 From: Francois Chollet Date: Fri, 13 Apr 2018 14:12:16 -0700 Subject: [PATCH 347/791] Extend Keras symbol-feeding to dynamic-length tensors and tensors of different dtypes from the target placeholders. PiperOrigin-RevId: 192821770 --- .../python/keras/_impl/keras/backend.py | 2 ++ .../python/keras/_impl/keras/backend_test.py | 5 ++++ .../keras/_impl/keras/engine/training_test.py | 17 +++++++++++ .../_impl/keras/engine/training_utils.py | 29 +++++++++---------- 4 files changed, 38 insertions(+), 15 deletions(-) diff --git a/tensorflow/python/keras/_impl/keras/backend.py b/tensorflow/python/keras/_impl/keras/backend.py index 6647cc5b79..81a4d2f820 100644 --- a/tensorflow/python/keras/_impl/keras/backend.py +++ b/tensorflow/python/keras/_impl/keras/backend.py @@ -2833,6 +2833,8 @@ class Function(object): # Handle symbolic feed. for x, y in zip(feed_symbols, symbol_vals): connection = callable_opts.tensor_connection.add() + if x.dtype != y.dtype: + y = math_ops.cast(y, dtype=x.dtype) from_tensor = ops._as_graph_element(y) if from_tensor is None: from_tensor = y diff --git a/tensorflow/python/keras/_impl/keras/backend_test.py b/tensorflow/python/keras/_impl/keras/backend_test.py index 0193fc6976..de1ed467a2 100644 --- a/tensorflow/python/keras/_impl/keras/backend_test.py +++ b/tensorflow/python/keras/_impl/keras/backend_test.py @@ -217,6 +217,11 @@ class BackendUtilsTest(test.TestCase): outs = f([y4, y2, None]) self.assertEqual(outs, [5., 2.]) + # Test with a different dtype + y5 = keras.backend.constant(10., dtype='float64') + outs = f([y5, y2, None]) + self.assertEqual(outs, [11., 2.]) + def test_function_tf_fetches(self): # Additional operations can be passed to tf.Session().run() via its # `fetches` arguments. In contrast to `updates` argument of diff --git a/tensorflow/python/keras/_impl/keras/engine/training_test.py b/tensorflow/python/keras/_impl/keras/engine/training_test.py index 08fd26dd18..6699fd5212 100644 --- a/tensorflow/python/keras/_impl/keras/engine/training_test.py +++ b/tensorflow/python/keras/_impl/keras/engine/training_test.py @@ -23,10 +23,12 @@ import unittest import numpy as np +from tensorflow.python.framework import tensor_shape from tensorflow.python.keras._impl import keras from tensorflow.python.keras._impl.keras import testing_utils from tensorflow.python.keras._impl.keras.engine.training_utils import weighted_masked_objective from tensorflow.python.keras._impl.keras.utils.generic_utils import slice_arrays +from tensorflow.python.ops import array_ops from tensorflow.python.platform import test try: @@ -1140,6 +1142,21 @@ class TestTrainingWithDataTensors(test.TestCase): epochs=1, steps_per_epoch=2, verbose=0, validation_data=(inputs, targets), validation_steps=2) + # Test with dynamic shape + inputs = array_ops.placeholder_with_default( + np.zeros((2, 3)), shape=tensor_shape.TensorShape([None, 3])) + targets = array_ops.placeholder_with_default( + np.zeros((2, 4)), shape=tensor_shape.TensorShape([None, 4])) + self.assertEqual(inputs.shape[0].value, None) + model.fit(inputs, targets, epochs=1, steps_per_epoch=2, verbose=0) + model.evaluate(inputs, targets, steps=2, verbose=0) + model.predict(inputs, steps=2) + model.train_on_batch(inputs, targets) + model.test_on_batch(inputs, targets) + model.fit(inputs, targets, + epochs=1, steps_per_epoch=2, verbose=0, + validation_data=(inputs, targets), validation_steps=2) + def test_training_and_eval_methods_on_symbolic_tensors_multi_io(self): with self.test_session(): a = keras.layers.Input(shape=(3,), name='input_a') diff --git a/tensorflow/python/keras/_impl/keras/engine/training_utils.py b/tensorflow/python/keras/_impl/keras/engine/training_utils.py index a3fc8ef2a0..48afe48e6c 100644 --- a/tensorflow/python/keras/_impl/keras/engine/training_utils.py +++ b/tensorflow/python/keras/_impl/keras/engine/training_utils.py @@ -61,22 +61,21 @@ def check_num_samples(ins, Raises: ValueError: In case of invalid arguments. """ - if steps is not None: - num_samples = None - if batch_size is not None: - raise ValueError( - 'If ' + steps_name + ' is set, the `batch_size` must be None.') - if has_symbolic_tensors(ins) and steps is None: - raise ValueError('If your data is in the form of symbolic tensors, ' - 'you should specify the `' + steps_name + '` argument ' - '(instead of the `batch_size` argument).') - if ins and hasattr(ins[0], 'shape'): - num_samples = int(ins[0].shape[0]) - elif steps is None: + if steps is not None and batch_size is not None: raise ValueError( - 'Either the input data should have ' - 'a defined shape, or ' + steps_name + ' should be specified.') - return num_samples + 'If ' + steps_name + ' is set, the `batch_size` must be None.') + + if not ins or has_symbolic_tensors(ins): + if steps is None: + raise ValueError('If your data is in the form of symbolic tensors, ' + 'you should specify the `' + steps_name + '` argument ' + '(instead of the `batch_size` argument, ' + 'because symbolic tensors are expected to produce ' + 'batches of input data).') + return None + if hasattr(ins[0], 'shape'): + return int(ins[0].shape[0]) + return None # Edge case where ins == [static_learning_phase] def standardize_single_array(x): -- GitLab From 638fd98e844a9ba8857b9b6fa194f555f53c033d Mon Sep 17 00:00:00 2001 From: Austin Anderson Date: Fri, 13 Apr 2018 14:13:12 -0700 Subject: [PATCH 348/791] Small tag change PiperOrigin-RevId: 192821895 --- tensorflow/contrib/lite/kernels/BUILD | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tensorflow/contrib/lite/kernels/BUILD b/tensorflow/contrib/lite/kernels/BUILD index 800e2a9558..ac7c3f071f 100644 --- a/tensorflow/contrib/lite/kernels/BUILD +++ b/tensorflow/contrib/lite/kernels/BUILD @@ -265,8 +265,7 @@ tf_cc_test( size = "small", srcs = ["arg_max_test.cc"], tags = [ - "tflite_not_portable_ios_arm64", - "tflite_not_portable_ios_x86_64", + "tflite_not_portable_ios", ], deps = [ ":builtin_ops", -- GitLab From bf724a8ced3710ed2234f25748ed7719e319d78c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 14:17:31 -0700 Subject: [PATCH 349/791] [XLA] Redesign: add ~XlaOp() and ~XlaComputation(). PiperOrigin-RevId: 192822559 --- tensorflow/compiler/xla/client/xla_client/xla_builder.h | 1 + tensorflow/compiler/xla/client/xla_client/xla_computation.h | 2 ++ 2 files changed, 3 insertions(+) diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.h b/tensorflow/compiler/xla/client/xla_client/xla_builder.h index e583b4fe48..1f7c731064 100644 --- a/tensorflow/compiler/xla/client/xla_client/xla_builder.h +++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.h @@ -53,6 +53,7 @@ class XlaBuilder; class XlaOp { public: XlaOp() : handle_(0), builder_(nullptr) {} + ~XlaOp() {} StatusOr GetShape() const; diff --git a/tensorflow/compiler/xla/client/xla_client/xla_computation.h b/tensorflow/compiler/xla/client/xla_client/xla_computation.h index 085fabd56d..7ad212aa24 100644 --- a/tensorflow/compiler/xla/client/xla_client/xla_computation.h +++ b/tensorflow/compiler/xla/client/xla_client/xla_computation.h @@ -33,6 +33,8 @@ class XlaComputation { XlaComputation(const HloModuleProto& proto) : unique_id_(proto.id()), proto_(proto) {} + ~XlaComputation() {} + XlaComputation(const XlaComputation&) = delete; XlaComputation& operator=(const XlaComputation&) = delete; -- GitLab From 8600d918a63c658b9b79ba96ee821c903ba3ee94 Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Fri, 13 Apr 2018 14:32:45 -0700 Subject: [PATCH 350/791] Allow tf.train.Saver to load object-based checkpoints (using names) This is the second part of the compatibility story. Object-based checkpointing APIs can already read name-based checkpoints, and now the name-based APIs can read object-based checkpoints by looking up the modified keys in the object graph proto. PiperOrigin-RevId: 192824907 --- tensorflow/python/training/checkpointable.py | 5 + .../python/training/checkpointable_utils.py | 14 +- .../training/checkpointable_utils_test.py | 3 - tensorflow/python/training/saver.py | 70 +++++++- tensorflow/python/training/saver_test.py | 150 ++++++++++++++++++ 5 files changed, 227 insertions(+), 15 deletions(-) diff --git a/tensorflow/python/training/checkpointable.py b/tensorflow/python/training/checkpointable.py index 9bf48df22e..0b8473742c 100644 --- a/tensorflow/python/training/checkpointable.py +++ b/tensorflow/python/training/checkpointable.py @@ -26,6 +26,11 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_io_ops as io_ops from tensorflow.python.util import nest + +# Key where the object graph proto is saved in a TensorBundle +OBJECT_GRAPH_PROTO_KEY = "_CHECKPOINTABLE_OBJECT_GRAPH" + + # A key indicating a variable's value in an object's checkpointed Tensors # (Checkpointable._gather_saveables_for_checkpoint). If this is the only key and # the object has no dependencies, then its value may be restored on object diff --git a/tensorflow/python/training/checkpointable_utils.py b/tensorflow/python/training/checkpointable_utils.py index da99d2ec31..2c4677a278 100644 --- a/tensorflow/python/training/checkpointable_utils.py +++ b/tensorflow/python/training/checkpointable_utils.py @@ -54,8 +54,6 @@ _OPTIMIZER_SLOTS_NAME = _ESCAPE_CHAR + "OPTIMIZER_SLOT" # attribute in checkpoint names. Used like: # /<_OBJECT_ATTRIBUTES_NAME>/ _OBJECT_ATTRIBUTES_NAME = _ESCAPE_CHAR + "ATTRIBUTES" -# Key where the object graph proto is saved in a TensorBundle -_OBJECT_GRAPH_PROTO_KEY = "_CHECKPOINTABLE_OBJECT_GRAPH" class _CheckpointRestoreCoordinator(object): @@ -680,10 +678,11 @@ class CheckpointableSaver(object): object_graph_tensor = constant_op.constant( graph_proto.SerializeToString(), dtype=dtypes.string) feed_additions = None - assert _OBJECT_GRAPH_PROTO_KEY not in named_variables - named_variables[_OBJECT_GRAPH_PROTO_KEY] = _NoRestoreSaveable( - tensor=object_graph_tensor, - name=_OBJECT_GRAPH_PROTO_KEY) + assert checkpointable_lib.OBJECT_GRAPH_PROTO_KEY not in named_variables + named_variables[checkpointable_lib.OBJECT_GRAPH_PROTO_KEY] = ( + _NoRestoreSaveable( + tensor=object_graph_tensor, + name=checkpointable_lib.OBJECT_GRAPH_PROTO_KEY)) if (self._last_save_object_graph != graph_proto # When executing eagerly, we need to re-create SaveableObjects each time # save() is called so they pick up new Tensors passed to their @@ -786,7 +785,8 @@ class CheckpointableSaver(object): file_prefix_feed_dict = None reader = pywrap_tensorflow.NewCheckpointReader(save_path) try: - object_graph_string = reader.get_tensor(_OBJECT_GRAPH_PROTO_KEY) + object_graph_string = reader.get_tensor( + checkpointable_lib.OBJECT_GRAPH_PROTO_KEY) except errors_impl.NotFoundError: # The object graph proto does not exist in this checkpoint. Try again with # name-based saving. diff --git a/tensorflow/python/training/checkpointable_utils_test.py b/tensorflow/python/training/checkpointable_utils_test.py index ddf9820616..29fcdb70b4 100644 --- a/tensorflow/python/training/checkpointable_utils_test.py +++ b/tensorflow/python/training/checkpointable_utils_test.py @@ -1268,9 +1268,6 @@ class CheckpointCompatibilityTests(test.TestCase): status.initialize_or_restore() self._check_sentinels(root) - # TODO(allenl): Test for the core name-based saver loading object-based - # checkpoints once object-based checkpointing is in core. - def testSaveGraphLoadEager(self): checkpoint_directory = self.get_temp_dir() checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py index e40b8d22ed..79d278cf90 100644 --- a/tensorflow/python/training/saver.py +++ b/tensorflow/python/training/saver.py @@ -22,6 +22,7 @@ from __future__ import print_function import collections import os.path import re +import sys import time import uuid @@ -30,8 +31,10 @@ import six from google.protobuf import text_format +from tensorflow.core.protobuf import checkpointable_object_graph_pb2 from tensorflow.core.protobuf import meta_graph_pb2 from tensorflow.core.protobuf import saver_pb2 +from tensorflow.python import pywrap_tensorflow from tensorflow.python.client import session from tensorflow.python.eager import context from tensorflow.python.framework import constant_op @@ -1340,6 +1343,9 @@ class Saver(object): self._check_saver_def() self._write_version = self.saver_def.version self._save_relative_paths = save_relative_paths + # For compatibility with object-based checkpoints, we may build a second + # Saver to read the renamed keys. + self._object_restore_saver = None def build(self): if context.executing_eagerly(): @@ -1795,11 +1801,65 @@ class Saver(object): if save_path is None: raise ValueError("Can't load save_path when it is None.") logging.info("Restoring parameters from %s", save_path) - if context.executing_eagerly(): - self._build_eager(save_path, build_save=False, build_restore=True) - else: - sess.run(self.saver_def.restore_op_name, - {self.saver_def.filename_tensor_name: save_path}) + try: + if context.executing_eagerly(): + self._build_eager(save_path, build_save=False, build_restore=True) + else: + sess.run(self.saver_def.restore_op_name, + {self.saver_def.filename_tensor_name: save_path}) + except errors.NotFoundError: + exception_type, exception_value, exception_traceback = sys.exc_info() + # The checkpoint would not be loaded successfully as is. Try to parse it + # as an object-based checkpoint. + try: + reader = pywrap_tensorflow.NewCheckpointReader(save_path) + object_graph_string = reader.get_tensor( + checkpointable.OBJECT_GRAPH_PROTO_KEY) + except errors.NotFoundError: + # This is not an object-based checkpoint, or the checkpoint doesn't + # exist. Re-raise the original exception. + six.reraise(exception_type, exception_value, exception_traceback) + del exception_traceback # avoid reference cycles + + # This is an object-based checkpoint. We'll print a warning and then do + # the restore. + logging.warning( + # TODO(allenl): Modify instructions for using the object-based saver + # once that's in core. + "Restoring an object-based checkpoint using a name-based saver. This " + "may be somewhat fragile, and will re-build the Saver. Instead, " + "consider loading object-based checkpoints using " + "tf.contrib.eager.Checkpoint().") + self._restore_from_object_based_checkpoint( + sess=sess, save_path=save_path, + object_graph_string=object_graph_string) + + def _restore_from_object_based_checkpoint(self, sess, save_path, + object_graph_string): + """A compatibility mode for reading object-based checkpoints.""" + object_graph_proto = ( + checkpointable_object_graph_pb2.CheckpointableObjectGraph()) + object_graph_proto.ParseFromString(object_graph_string) + names_to_keys = {} + for node in object_graph_proto.nodes: + for attribute in node.attributes: + names_to_keys[attribute.full_name] = attribute.checkpoint_key + saveables = self._builder._ValidateAndSliceInputs(self._var_list) # pylint: disable=protected-access + for saveable in saveables: + for spec in saveable.specs: + if spec.name not in names_to_keys: + raise errors.NotFoundError( + None, None, + message=("Attempting to load an object-based checkpoint using " + "variable names, but could not find %s in the " + "checkpoint.") % spec.name) + spec.name = names_to_keys[spec.name] + if self._object_restore_saver is None: + # Cache the Saver so multiple restore() calls don't pollute the graph when + # graph building. This assumes keys are consistent (i.e. this is the same + # type of object-based checkpoint we saw previously). + self._object_restore_saver = Saver(saveables) + self._object_restore_saver.restore(sess=sess, save_path=save_path) @staticmethod def _add_collection_def(meta_graph_def, key, export_scope=None): diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py index 14dda79979..3867c0d8da 100644 --- a/tensorflow/python/training/saver_test.py +++ b/tensorflow/python/training/saver_test.py @@ -19,6 +19,7 @@ from __future__ import division from __future__ import print_function import contextlib +import functools import math import os import random @@ -50,6 +51,8 @@ from tensorflow.python.framework import graph_io from tensorflow.python.framework import meta_graph from tensorflow.python.framework import ops as ops_lib from tensorflow.python.framework import test_util +from tensorflow.python.keras._impl.keras.engine import training +from tensorflow.python.keras._impl.keras.layers import core from tensorflow.python.lib.io import file_io from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops @@ -69,10 +72,12 @@ from tensorflow.python.platform import test from tensorflow.python.summary import summary from tensorflow.python.training import adam from tensorflow.python.training import checkpointable +from tensorflow.python.training import checkpointable_utils from tensorflow.python.training import gradient_descent from tensorflow.python.training import queue_runner_impl from tensorflow.python.training import saver as saver_module from tensorflow.python.training import saver_test_utils +from tensorflow.python.training import training_util from tensorflow.python.training.checkpoint_state_pb2 import CheckpointState from tensorflow.python.util import compat @@ -2948,6 +2953,29 @@ class _OwnsMirroredVariables(checkpointable.CheckpointableBase): return self.non_dep_variable.name +class NonLayerCheckpointable(checkpointable.Checkpointable): + + def __init__(self): + super(NonLayerCheckpointable, self).__init__() + self.a_variable = checkpointable_utils.add_variable( + self, name="a_variable", shape=[]) + + +class MyModel(training.Model): + """A concrete Model for testing.""" + + def __init__(self): + super(MyModel, self).__init__() + self._named_dense = core.Dense(1, use_bias=True) + self._second = core.Dense(1, use_bias=False) + # We can still track Checkpointables which aren't Layers. + self._non_layer = NonLayerCheckpointable() + + def call(self, values): + ret = self._second(self._named_dense(values)) + return ret + + @test_util.with_c_api class CheckpointableCompatibilityTests(test.TestCase): @@ -3011,6 +3039,128 @@ class CheckpointableCompatibilityTests(test.TestCase): saver.restore(sess, save_path) self.assertEqual(1, v.eval_count) + def _initialized_model(self): + input_value = constant_op.constant([[3.]]) + model = MyModel() + optimizer = adam.AdamOptimizer(0.001) + optimizer_step = training_util.get_or_create_global_step() + root_checkpointable = checkpointable_utils.Checkpoint( + optimizer=optimizer, model=model, optimizer_step=optimizer_step) + train_op = optimizer.minimize( + functools.partial(model, input_value), + global_step=optimizer_step) + self.evaluate(checkpointable_utils.gather_initializers( + root_checkpointable)) + self.evaluate(train_op) + # A regular variable, a slot variable, and a non-slot Optimizer variable + # with known values to check when loading. + self.evaluate(model._named_dense.bias.assign([1.])) + self.evaluate(optimizer.get_slot( + var=model._named_dense.bias, name="m").assign([2.])) + beta1_power, _ = optimizer._get_beta_accumulators() + self.evaluate(beta1_power.assign(3.)) + return root_checkpointable + + def _set_sentinels(self, root_checkpointable): + self.evaluate(root_checkpointable.model._named_dense.bias.assign([101.])) + self.evaluate( + root_checkpointable.optimizer.get_slot( + var=root_checkpointable.model._named_dense.bias, name="m") + .assign([102.])) + beta1_power, _ = root_checkpointable.optimizer._get_beta_accumulators() + self.evaluate(beta1_power.assign(103.)) + + def _check_sentinels(self, root_checkpointable): + self.assertAllEqual( + [1.], self.evaluate(root_checkpointable.model._named_dense.bias)) + self.assertAllEqual([2.], self.evaluate( + root_checkpointable.optimizer.get_slot( + var=root_checkpointable.model._named_dense.bias, name="m"))) + beta1_power, _ = root_checkpointable.optimizer._get_beta_accumulators() + self.assertAllEqual(3., self.evaluate(beta1_power)) + + def testVariableNotFoundErrorRaised(self): + # Restore does some tricky exception handling to figure out if it should + # load an object-based checkpoint. Tests that the exception handling isn't + # too broad. + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + + a = resource_variable_ops.ResourceVariable(1., name="a") + b = resource_variable_ops.ResourceVariable(1., name="b") + a_saver = saver_module.Saver([a]) + b_saver = saver_module.Saver([b]) + with self.test_session() as sess: + sess.run(a.initializer) + save_path = a_saver.save(sess=sess, save_path=checkpoint_prefix) + with self.assertRaisesRegexp( + errors.NotFoundError, "Key b not found in checkpoint"): + b_saver.restore(sess=sess, save_path=save_path) + + def testCheckpointNotFoundErrorRaised(self): + # Restore does some tricky exception handling to figure out if it should + # load an object-based checkpoint. Tests that the exception handling isn't + # too broad. + a = resource_variable_ops.ResourceVariable(1., name="a") + saver = saver_module.Saver([a]) + with self.test_session() as sess: + with self.assertRaisesRegexp( + errors.NotFoundError, + "Failed to find any matching files for path_which_does_not_exist"): + saver.restore(sess=sess, save_path="path_which_does_not_exist") + + def testLoadFromObjectBasedGraph(self): + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + + save_graph = ops_lib.Graph() + with save_graph.as_default(), self.test_session(graph=save_graph) as sess: + root = self._initialized_model() + object_saver = checkpointable_utils.CheckpointableSaver(root) + save_path = object_saver.save(file_prefix=checkpoint_prefix) + + # An incompatible object-based checkpoint to check error messages + var = resource_variable_ops.ResourceVariable(1., name="a") + self.evaluate(var.initializer) + second_saver = checkpointable_utils.CheckpointableSaver(var) + second_path = second_saver.save(file_prefix=os.path.join( + checkpoint_directory, "second")) + + restore_graph = ops_lib.Graph() + with restore_graph.as_default(), self.test_session( + graph=restore_graph) as sess: + root = self._initialized_model() + self._set_sentinels(root) + saver = saver_module.Saver() + saver.restore(sess=sess, save_path=save_path) + self._check_sentinels(root) + before_second_restore_ops = restore_graph.get_operations() + # Test that multiple restores do not pollute the graph + saver.restore(sess=sess, save_path=save_path) + self.assertEqual(before_second_restore_ops, + restore_graph.get_operations()) + with self.assertRaisesRegexp(errors.NotFoundError, + "could not find a_variable"): + saver.restore(sess=sess, save_path=second_path) + + def testLoadFromObjectBasedEager(self): + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + + save_graph = ops_lib.Graph() + with save_graph.as_default(), self.test_session(graph=save_graph): + root = self._initialized_model() + object_saver = checkpointable_utils.CheckpointableSaver(root) + save_path = object_saver.save(file_prefix=checkpoint_prefix) + + with context.eager_mode(): + root = self._initialized_model() + self._set_sentinels(root) + saver = saver_module.Saver( + root.model.variables + root.optimizer.variables()) + saver.restore(sess=None, save_path=save_path) + self._check_sentinels(root) + if __name__ == "__main__": test.main() -- GitLab From aedc409605be54f9c7cb67f7b49bdc123d65a8fb Mon Sep 17 00:00:00 2001 From: Sung Jin Hwang Date: Fri, 13 Apr 2018 14:51:16 -0700 Subject: [PATCH 351/791] Added PmfToQuantizedCdf op to contrib/coder in TensorFlow. The added op transforms probability mass functions (PMF) to quantized cumulative distribution function (CDF), which can be used by range coder ops in contrib/coder. The op takes greedy approach to ensure that the post-quantization probability masses do not sum over the maximum quantized value. The op does not make any adjustment when the post-quantization probability masses already sum less than the maximum value. PiperOrigin-RevId: 192827779 --- tensorflow/contrib/coder/BUILD | 34 +++- .../contrib/coder/kernels/pmf_to_cdf_op.cc | 150 ++++++++++++++++++ .../coder/kernels/pmf_to_cdf_op_test.cc | 140 ++++++++++++++++ tensorflow/contrib/coder/ops/coder_ops.cc | 32 ++++ 4 files changed, 355 insertions(+), 1 deletion(-) create mode 100644 tensorflow/contrib/coder/kernels/pmf_to_cdf_op.cc create mode 100644 tensorflow/contrib/coder/kernels/pmf_to_cdf_op_test.cc diff --git a/tensorflow/contrib/coder/BUILD b/tensorflow/contrib/coder/BUILD index ce12e38248..9ca4ce8a9c 100644 --- a/tensorflow/contrib/coder/BUILD +++ b/tensorflow/contrib/coder/BUILD @@ -92,6 +92,34 @@ tf_cc_test( ], ) +tf_kernel_library( + name = "pmf_to_cdf_op", + srcs = ["kernels/pmf_to_cdf_op.cc"], + visibility = ["//visibility:public"], + deps = [ + ":coder_ops_op_lib", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + ], +) + +tf_cc_test( + name = "pmf_to_cdf_op_test", + size = "small", + srcs = ["kernels/pmf_to_cdf_op_test.cc"], + deps = [ + ":pmf_to_cdf_op", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + "//tensorflow/core/kernels:ops_testutil", + ], +) + cc_library( name = "all_ops", deps = [":coder_ops_op_lib"], @@ -99,12 +127,16 @@ cc_library( cc_library( name = "all_kernels", - deps = [":range_coder_ops"], + deps = [ + ":pmf_to_cdf_op", + ":range_coder_ops", + ], ) tf_custom_op_library( name = "python/ops/_coder_ops.so", srcs = [ + "kernels/pmf_to_cdf_op.cc", "kernels/range_coder.cc", "kernels/range_coder.h", "kernels/range_coder_ops.cc", diff --git a/tensorflow/contrib/coder/kernels/pmf_to_cdf_op.cc b/tensorflow/contrib/coder/kernels/pmf_to_cdf_op.cc new file mode 100644 index 0000000000..c787e8eded --- /dev/null +++ b/tensorflow/contrib/coder/kernels/pmf_to_cdf_op.cc @@ -0,0 +1,150 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#define EIGEN_USE_THREADS + +#include +#include +#include +#include + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/tensor_types.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/threadpool.h" +#include "tensorflow/core/lib/gtl/array_slice.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/macros.h" +#include "tensorflow/core/platform/types.h" + +namespace tensorflow { +namespace { +using errors::InvalidArgument; + +class PmfToCdfOp : public OpKernel { + public: + explicit PmfToCdfOp(OpKernelConstruction* context) : OpKernel(context) { + OP_REQUIRES_OK(context, context->GetAttr("precision", &precision_)); + OP_REQUIRES( + context, 0 < precision_ && precision_ <= 16, + InvalidArgument("`precision` must be in [1, 16]: ", precision_)); + } + + void Compute(OpKernelContext* context) override { + const Tensor& pmf_tensor = context->input(0); + + TensorShape shape = pmf_tensor.shape(); + OP_REQUIRES(context, TensorShapeUtils::IsVectorOrHigher(shape), + InvalidArgument("`pmf` should be at least 1-D.")); + OP_REQUIRES( + context, shape.dim_size(shape.dims() - 1) > 1, + InvalidArgument("`pmf` size should be at least 2 in the last axis.")); + shape.set_dim(shape.dims() - 1, shape.dim_size(shape.dims() - 1) + 1); + + Tensor* cdf_tensor; + OP_REQUIRES_OK(context, context->allocate_output(0, shape, &cdf_tensor)); + + auto pmf = pmf_tensor.flat_inner_dims(); + auto cdf = cdf_tensor->flat_inner_dims(); + CHECK_EQ(pmf.dimension(0), cdf.dimension(0)); + CHECK_EQ(pmf.dimension(1) + 1, cdf.dimension(1)); + + const double n = pmf.dimension(1); + const int64 cost_per_unit = static_cast(50.0 * n * std::log2(n)); + thread::ThreadPool* thread_pool = + context->device()->tensorflow_cpu_worker_threads()->workers; + thread_pool->ParallelFor( + pmf.dimension(0), cost_per_unit, + [this, pmf, &cdf](int64 start, int64 limit) { + const gtl::ArraySlice::size_type pmf_size = pmf.dimension(1); + for (int64 i = start; i < limit; ++i) { + cdf(i, 0) = 0; + PerShard({&pmf(i, 0), pmf_size}, {&cdf(i, 1), pmf_size}); + } + }); + } + + private: + struct Item { + Item(int32* p, double mass) : pointer(p), mass(mass) { + penalty = ComputeNextPenalty(); + } + + void Decrease() { + CHECK_GT(*pointer, 1); + --*pointer; + penalty = ComputeNextPenalty(); + } + + friend bool operator<(const Item& lhs, const Item& rhs) { + return lhs.penalty < rhs.penalty; + } + + double ComputeNextPenalty() { + if (*pointer <= 1) { + return std::numeric_limits::infinity(); + } + return mass * (std::log2(*pointer) - std::log2(*pointer - 1)); + } + + int32* pointer; + double mass; + double penalty; + }; + + void PerShard(gtl::ArraySlice pmf, + gtl::MutableArraySlice cdf) const { + CHECK_EQ(pmf.size(), cdf.size()); + + const int32 normalizer = 1 << precision_; + std::transform(pmf.begin(), pmf.end(), cdf.begin(), + [normalizer](float mass) { + int32 value = std::rint(mass * normalizer); + // NOTE: Consider checking if mass > 0. + value = std::max(value, 1); + return value; + }); + + int32 sum = std::accumulate(cdf.begin(), cdf.end(), 0); + if (sum > normalizer) { + std::vector queue; + queue.reserve(cdf.size()); + for (int i = 0; i < cdf.size(); ++i) { + queue.emplace_back(&cdf[i], pmf[i]); + } + + std::sort(queue.begin(), queue.end()); + while (sum-- > normalizer) { + queue[0].Decrease(); + // Performs a linear search because this find_if is likely to return + // iterator very close to the begin. + auto iter = + std::find_if(std::next(queue.begin()), queue.end(), + [&queue](const Item& rhs) { return queue[0] < rhs; }); + std::rotate(queue.begin(), std::next(queue.begin()), iter); + } + } + std::partial_sum(cdf.begin(), cdf.end(), cdf.begin()); + } + + int precision_; +}; + +REGISTER_KERNEL_BUILDER(Name("PmfToQuantizedCdf").Device(DEVICE_CPU), + PmfToCdfOp); +} // namespace +} // namespace tensorflow diff --git a/tensorflow/contrib/coder/kernels/pmf_to_cdf_op_test.cc b/tensorflow/contrib/coder/kernels/pmf_to_cdf_op_test.cc new file mode 100644 index 0000000000..c70e38faab --- /dev/null +++ b/tensorflow/contrib/coder/kernels/pmf_to_cdf_op_test.cc @@ -0,0 +1,140 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "tensorflow/core/framework/fake_input.h" +#include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/framework/node_def_builder.h" +#include "tensorflow/core/framework/shape_inference_testutil.h" +#include "tensorflow/core/framework/tensor_types.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/kernels/ops_testutil.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/lib/gtl/array_slice.h" +#include "tensorflow/core/lib/random/philox_random.h" +#include "tensorflow/core/lib/random/random.h" +#include "tensorflow/core/lib/random/simple_philox.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace { +class PmfToQuantizedCdfOpTest : public OpsTestBase { + protected: + void SetupOp(int precision, Tensor* input) { + TF_ASSERT_OK(NodeDefBuilder("pmf_to_cdf", "PmfToQuantizedCdf") + .Input(FakeInput(DT_FLOAT)) + .Attr("precision", precision) + .Finalize(node_def())); + TF_ASSERT_OK(InitOp()); + + inputs_.clear(); + inputs_.emplace_back(input); + } + + void GenerateData(random::SimplePhilox* rand, + gtl::MutableArraySlice slice) { + constexpr float minimum = std::numeric_limits::epsilon(); + float sum = 0; + for (float& value : slice) { + value = std::max(rand->RandFloat(), minimum); + sum += value; + } + for (float& value : slice) { + value /= sum; + } + } + + void Verify(int precision, const Tensor& pmf_tensor, + const Tensor& cdf_tensor) { + ASSERT_EQ(pmf_tensor.dims(), cdf_tensor.dims()); + const int n = pmf_tensor.dims(); + + for (int i = 0; i < n - 1; ++i) { + EXPECT_EQ(pmf_tensor.dim_size(i), cdf_tensor.dim_size(i)); + } + + auto pmf = pmf_tensor.flat_inner_dims(); + auto cdf = cdf_tensor.flat_inner_dims(); + EXPECT_EQ(pmf.dimension(1) + 1, cdf.dimension(1)); + + const int normalizer = 1 << precision; + for (int i = 0; i < pmf.dimension(0); ++i) { + EXPECT_EQ(0, cdf(i, 0)); + + TTypes::UnalignedConstVec cdf_slice(&cdf(i, 0), cdf.dimension(1)); + + for (int j = 1; j < cdf_slice.size(); ++j) { + const int32 diff = cdf_slice(j) - cdf_slice(j - 1); + EXPECT_GT(diff, 0); + } + + EXPECT_LE(cdf_slice(cdf_slice.size() - 1), normalizer); + } + } +}; + +TEST_F(PmfToQuantizedCdfOpTest, UnderSum) { + Tensor pmf(DT_FLOAT, {1, 10, 1, 32}); + auto matrix = pmf.flat_inner_dims(); + const std::size_t n = matrix.dimension(1); + + random::PhiloxRandom gen(random::New64(), random::New64()); + random::SimplePhilox rand(&gen); + for (int64 i = 0; i < matrix.dimension(0); ++i) { + GenerateData(&rand, {&matrix(i, 0), n}); + } + + constexpr int kPrecision = 10; + SetupOp(kPrecision, &pmf); + TF_ASSERT_OK(RunOpKernel()); + + Verify(kPrecision, pmf, *GetOutput(0)); +} + +TEST_F(PmfToQuantizedCdfOpTest, OverSum) { + Tensor pmf(DT_FLOAT, {10, 1, 1, 100}); + auto matrix = pmf.flat_inner_dims(); + + // Half of each PMF is filled with zeros. The op will round up zeros to ones, + // post quantization. These round ups are likely to make the sum over + // normalizer value. + matrix.setZero(); + const std::size_t n = matrix.dimension(1) / 2; + + random::PhiloxRandom gen; + random::SimplePhilox rand(&gen); + for (int64 i = 0; i < matrix.dimension(0); ++i) { + GenerateData(&rand, {&matrix(i, 0), n}); + } + + constexpr int kPrecision = 7; + SetupOp(kPrecision, &pmf); + TF_ASSERT_OK(RunOpKernel()); + + Verify(kPrecision, pmf, *GetOutput(0)); +} + +TEST_F(PmfToQuantizedCdfOpTest, ShapeFn) { + ShapeInferenceTestOp op("PmfToQuantizedCdf"); + + INFER_OK(op, "?", "?"); + INFER_OK(op, "[3]", "[4]"); + INFER_OK(op, "[3,4]", "[d0_0,5]"); + INFER_OK(op, "[3,4,5]", "[d0_0,d0_1,6]"); +} +} // namespace +} // namespace tensorflow diff --git a/tensorflow/contrib/coder/ops/coder_ops.cc b/tensorflow/contrib/coder/ops/coder_ops.cc index 9056d1a696..9bb171298f 100644 --- a/tensorflow/contrib/coder/ops/coder_ops.cc +++ b/tensorflow/contrib/coder/ops/coder_ops.cc @@ -19,6 +19,7 @@ limitations under the License. #include "tensorflow/core/lib/core/status.h" namespace tensorflow { +using shape_inference::DimensionHandle; using shape_inference::InferenceContext; using shape_inference::ShapeHandle; @@ -115,5 +116,36 @@ decoded: An int32 tensor with shape equal to `shape`. precision: The number of bits for probability quantization. Must be <= 16, and must match the precision used by RangeEncode that produced `encoded`. )doc"); + +REGISTER_OP("PmfToQuantizedCdf") + .Input("pmf: float") + .Output("cdf: int32") + .Attr("precision: int >= 1") + .SetShapeFn([] (InferenceContext* c) { + ShapeHandle in; + TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &in)); + DimensionHandle last; + TF_RETURN_IF_ERROR(c->Add(c->Dim(in, -1), 1, &last)); + ShapeHandle out; + TF_RETURN_IF_ERROR(c->ReplaceDim(in, -1, last, &out)); + c->set_output(0, out); + return Status::OK(); + }) + .Doc(R"doc( +Converts PMF to quantized CDF. This op uses floating-point operations +internally. Therefore the quantized output may not be consistent across multiple +platforms. For entropy encoders and decoders to have the same quantized CDF on +different platforms, the quantized CDF should be produced once and saved, then +the saved quantized CDF should be used everywhere. + +After quantization, if PMF sums to less than or equal to 2^precision, then this +is equivalent to cumsum over the last dimension. This op makes no effort to make +the sum close to 2^precision when the sum is already <= 2^precision. + +After quantization, if PMF sums to greater than 2^precision, then some values of +PMF is decreased to keep the sum no more than 2^precision. + +Note that the input PMF is pre-quantization. +)doc"); // clang-format on } // namespace tensorflow -- GitLab From fa6150d369ea40b795a17221e6f5a0bf054a8cc8 Mon Sep 17 00:00:00 2001 From: Sami Kama Date: Fri, 13 Apr 2018 15:01:07 -0700 Subject: [PATCH 352/791] Adding py_test for TF-TRT integration --- tensorflow/contrib/tensorrt/BUILD | 9 + .../contrib/tensorrt/test/test_integration.py | 178 ++++++++++++++++++ 2 files changed, 187 insertions(+) create mode 100644 tensorflow/contrib/tensorrt/test/test_integration.py diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD index fd3582e175..d116114db0 100644 --- a/tensorflow/contrib/tensorrt/BUILD +++ b/tensorflow/contrib/tensorrt/BUILD @@ -272,3 +272,12 @@ tf_cc_test( "//tensorflow/core:test_main", ], ) + +py_test( + name = "tf_trt_integration_test", + srcs = ["test/test_integration.py"], + srcs_version = "PY2AND3", + deps = [ + ":init_py" + ] +) \ No newline at end of file diff --git a/tensorflow/contrib/tensorrt/test/test_integration.py b/tensorflow/contrib/tensorrt/test/test_integration.py new file mode 100644 index 0000000000..8ad26c3f69 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/test_integration.py @@ -0,0 +1,178 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.contrib import tensorrt as trt +from tensorflow.core.protobuf import config_pb2 as cpb2 +from tensorflow.python.client import session as csess +from tensorflow.python.framework import test_util +from tensorflow.python.framework import constant_op as cop +from tensorflow.python.framework import dtypes as dtypes +from tensorflow.python.framework import importer as importer +from tensorflow.python.framework import ops as ops +from tensorflow.python.ops import array_ops as aops +from tensorflow.python.ops import nn as nn +from tensorflow.python.ops import nn_ops as nn_ops +from tensorflow.python.platform import googletest +from tensorflow.python.platform import test + + +@test_util.with_c_api +class IntegrationTest(test_util.TensofFlowTestCase): + + def setUp(self): + """ Setup method """ + super(IntegrationTest, self).setUp() + warnings.simplefilter('always') + inp_dims = (100, 24, 24, 2) + self._input = np.random.random_sample(inp_dims) + self._original_graph = get_simple_graph_def() + self._gpu_options = cpb2.GPUOptions( + per_process_gpu_memory_fraction=0.50) + self._config = cpb2.ConfigProto(gpu_options=gpu_options) + self._reference = self.run_graph(self._original_graph, self._input) + + def get_simple_graph_def(self): + """Create a simple graph and return its graph_def.""" + g = ops.Graph() + with g.as_default(): + a = aops.placeholder( + dtype=dtypes.float32, shape=(None, 24, 24, 2), name="input") + e = cop.constant( + [[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]], + name="weights", + dtype=dtypes.float32) + conv = nn.conv2d( + input=a, + filter=e, + strides=[1, 2, 2, 1], + padding="SAME", + name="conv") + b = cop.constant( + [4., 1.5, 2., 3., 5., 7.], name="bias", dtype=dtypes.float32) + t = nn.bias_add(conv, b, name="biasAdd") + relu = nn.relu(t, "relu") + idty = aops.identity(relu, "ID") + v = nn_ops.max_pool( + idty, [1, 2, 2, 1], [1, 2, 2, 1], "VALID", name="max_pool") + aops.squeeze(v, name="output") + return g.as_graph_def() + + def run_graph(self, gdef, dumm_inp): + """Run given graphdef once.""" + ops.reset_default_graph() + g = ops.Graph() + with g.as_default(): + inp, out = importer.import_graph_def( + graph_def=gdef, return_elements=["input", "output"]) + inp = inp.outputs[0] + out = out.outputs[0] + with self.test_session( + grap=g, config=self._config, use_gpu=True, + force_gpu=True) as sess: + val = sess.run(out, {inp: dumm_inp}) + return val + + # Use real data that is representative of the inference dataset + # for calibration. For this test script it is random data. + def run_calibration(self, gdef, dumm_inp): + """Run given calibration graph multiple times.""" + ops.reset_default_graph() + g = ops.Graph() + with g.as_default(): + inp, out = importer.import_graph_def( + graph_def=gdef, return_elements=["input", "output"]) + inp = inp.outputs[0] + out = out.outputs[0] + # run over real calibration data here, we are mimicking a calibration set of + # 30 different batches. Use as much calibration data as you want + with self.test_session( + grap=g, config=self._config, use_gpu=True, + force_gpu=True) as sess: + for _ in range(30): + val = sess.run(out, {inp: dumm_inp}) + return val + + def get_trt_graph(self, mode): + """ return trt converted graph """ + if mode == "FP32": + return trt.create_inference_graph( + input_graph_def=self._orig_graph, + outputs=["output"], + max_batch_size=inp_dims[0], + max_workspace_size_bytes=1 << 25, + precision_mode= + "FP32", # TRT Engine precision "FP32","FP16" or "INT8" + minimum_segment_size=2 # minimum number of nodes in an engine + ) + elif mode == "FP16": + return trt.create_inference_graph( + input_graph_def=self._orig_graph, + outputs=["output"], + max_batch_size=inp_dims[0], + max_workspace_size_bytes=1 << 25, + precision_mode= + "FP16", # TRT Engine precision "FP32","FP16" or "INT8" + minimum_segment_size=2 # minimum number of nodes in an engine + ) + elif mode == "INT8": + return trt.create_inference_graph( + input_graph_def=self._orig_graph, + outputs=["output"], + max_batch_size=inp_dims[0], + max_workspace_size_bytes=1 << 25, + precision_mode= + "INT8", # TRT Engine precision "FP32","FP16" or "INT8" + minimum_segment_size=2 # minimum number of nodes in an engine + ) + + return None + + def testFP32(self): + """ Test FP32 conversion. Results should be identical to native case """ + trt_graph = self.get_trt_graph("FP32") + result = self.run_graph(trt_graph, self._input) + self.assertAllEqual(self._reference, result) + result = self.run_graph(trt_graph, self._input) + self.assertAllEqual(self._reference, result) + + def testFP16(self): + """ Test FP16 conversion. Results may be different from native case """ + trt_graph = self.get_trt_graph("FP16") + result = self.run_graph(trt_graph, self._input) + self.assertAllEqual(self._reference, result) + result = self.run_graph(trt_graph, self._input) + self.assertAllEqual(self._reference, result) + + def testINT8(self): + """ Test INT8 conversion. Results may be different from native case """ + calib_graph = self.get_trt_graph("INT8") + result = self.run_calibration(calib_graph, self._input) + self.assertAllEqual(self._reference, result) + int8_graph = trt.calib_graph_to_infer_graph(int8_calib_gdef) + result = self.run_graph(int8_graph, self._input) + self.assertAllEqual(self._reference, result) + result = self.run_graph(int8_graph, self._input) + self.assertAllEqual(self._reference, result) + + +if __name__ == '__main__': + googletest.main() -- GitLab From 1298c3240aa9f36b79ea7f0e772edfff87381771 Mon Sep 17 00:00:00 2001 From: Bixia Zheng Date: Fri, 13 Apr 2018 15:15:44 -0700 Subject: [PATCH 353/791] [TF] Enable half precision XLA compiler tests for the gpu backend. Modify some tests to allow larger error for half precision. Enable half precision SpaceToBatchNDTest for the cpu backend. PiperOrigin-RevId: 192831909 --- tensorflow/compiler/tests/build_defs.bzl | 2 +- tensorflow/compiler/tests/ftrl_test.py | 14 +++++++---- tensorflow/compiler/tests/image_ops_test.py | 3 ++- .../compiler/tests/spacetobatch_op_test.py | 23 +++++++++++++------ tensorflow/python/framework/test_util.py | 4 +++- 5 files changed, 32 insertions(+), 14 deletions(-) diff --git a/tensorflow/compiler/tests/build_defs.bzl b/tensorflow/compiler/tests/build_defs.bzl index 45b6a6eb86..7b114d4f85 100644 --- a/tensorflow/compiler/tests/build_defs.bzl +++ b/tensorflow/compiler/tests/build_defs.bzl @@ -56,7 +56,7 @@ def tf_xla_py_test(name, srcs=[], deps=[], tags=[], data=[], main=None, elif backend == "gpu": backend_args += [ "--test_device=XLA_GPU", - "--types=DT_FLOAT,DT_DOUBLE,DT_INT32,DT_INT64,DT_BOOL,DT_COMPLEX64,DT_BFLOAT16" + "--types=DT_HALF,DT_FLOAT,DT_DOUBLE,DT_INT32,DT_INT64,DT_BOOL,DT_COMPLEX64,DT_BFLOAT16" ] backend_tags += ["requires-gpu-sm35"] elif backend in plugins: diff --git a/tensorflow/compiler/tests/ftrl_test.py b/tensorflow/compiler/tests/ftrl_test.py index f9db4cf201..8e6407dffd 100644 --- a/tensorflow/compiler/tests/ftrl_test.py +++ b/tensorflow/compiler/tests/ftrl_test.py @@ -134,9 +134,15 @@ class FtrlOptimizerTest(XLATestCase): # Validate updated params self.assertAllCloseAccordingToType( - np.array([-2.60260963, -4.29698515]), var0.eval(), float_rtol=1e-5) + np.array([-2.60260963, -4.29698515]), + var0.eval(), + float_rtol=1e-5, + half_rtol=1e-2) self.assertAllCloseAccordingToType( - np.array([-0.28432083, -0.56694895]), var1.eval(), float_rtol=1e-5) + np.array([-0.28432083, -0.56694895]), + var1.eval(), + float_rtol=1e-5, + half_rtol=1e-2) def testFtrlwithoutRegularization2(self): for dtype in self.float_types: @@ -272,8 +278,8 @@ class FtrlOptimizerTest(XLATestCase): with self.test_session(), self.test_scope(): val2, val3 = self.equivAdagradTest_AdagradPart(steps, dtype) - self.assertAllCloseAccordingToType(val0, val2, rtol=1e-4) - self.assertAllCloseAccordingToType(val1, val3, rtol=1e-4) + self.assertAllCloseAccordingToType(val0, val2, rtol=1e-4, half_rtol=1e-2) + self.assertAllCloseAccordingToType(val1, val3, rtol=1e-4, half_rtol=1e-2) def testEquivGradientDescentwithoutRegularization(self): steps = 5 diff --git a/tensorflow/compiler/tests/image_ops_test.py b/tensorflow/compiler/tests/image_ops_test.py index 3bc41b7cfd..12791ef8ac 100644 --- a/tensorflow/compiler/tests/image_ops_test.py +++ b/tensorflow/compiler/tests/image_ops_test.py @@ -65,7 +65,8 @@ class RGBToHSVTest(XLATestCase): # Verify that processing batch elements together is the same as separate self.assertAllClose(batch1, join1) self.assertAllClose(batch2, join2) - self.assertAllCloseAccordingToType(batch2, inp, bfloat16_atol=0.03) + self.assertAllCloseAccordingToType( + batch2, inp, bfloat16_atol=0.03, half_rtol=0.02) def testRGBToHSVRoundTrip(self): data = [0, 5, 13, 54, 135, 226, 37, 8, 234, 90, 255, 1] diff --git a/tensorflow/compiler/tests/spacetobatch_op_test.py b/tensorflow/compiler/tests/spacetobatch_op_test.py index ef47187477..f37c34156f 100644 --- a/tensorflow/compiler/tests/spacetobatch_op_test.py +++ b/tensorflow/compiler/tests/spacetobatch_op_test.py @@ -163,17 +163,26 @@ class SpaceToBatchNDTest(XLATestCase): # error. if dtype == dtypes.bfloat16.as_numpy_dtype: continue - # TODO(b/77694432): Half test failed on CPU, last ran on 04-06-2018. - if dtype == np.float16 and self.device == "XLA_CPU": - continue + if dtype == np.float16: + actual_inputs = np.array(inputs).astype(dtype) + actual_paddings = np.array(paddings).astype(dtype) + expected_outputs = np.array(outputs).astype(dtype) + else: + actual_inputs = inputs + actual_paddings = paddings + expected_outputs = outputs placeholder = array_ops.placeholder(dtype) # outputs = space_to_batch(inputs) - x_tf = array_ops.space_to_batch_nd(placeholder, block_shape, paddings) - self.assertAllEqual(sess.run(x_tf, {placeholder: inputs}), outputs) + x_tf = array_ops.space_to_batch_nd(placeholder, block_shape, + actual_paddings) + self.assertAllEqual( + sess.run(x_tf, {placeholder: actual_inputs}), expected_outputs) # inputs = batch_to_space(outputs) placeholder = array_ops.placeholder(dtype) - x_tf = array_ops.batch_to_space_nd(placeholder, block_shape, paddings) - self.assertAllEqual(sess.run(x_tf, {placeholder: outputs}), inputs) + x_tf = array_ops.batch_to_space_nd(placeholder, block_shape, + actual_paddings) + self.assertAllEqual( + sess.run(x_tf, {placeholder: expected_outputs}), actual_inputs) def _testDirect(self, input_shape, block_shape, paddings): inputs = np.arange(np.prod(input_shape), dtype=np.float32) diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py index eea27d76c6..70e70abc06 100644 --- a/tensorflow/python/framework/test_util.py +++ b/tensorflow/python/framework/test_util.py @@ -1380,7 +1380,9 @@ class TensorFlowTestCase(googletest.TestCase): " %s" % (a.shape, b.shape, msg)) same = (a == b) - if a.dtype == np.float32 or a.dtype == np.float64: + if (a.dtype in [ + np.float16, np.float32, np.float64, dtypes.bfloat16.as_numpy_dtype + ]): same = np.logical_or(same, np.logical_and(np.isnan(a), np.isnan(b))) if not np.all(same): # Prints more details than np.testing.assert_array_equal. -- GitLab From 9fb54c30efdcf38ef83c2709a8619a5bf20f2434 Mon Sep 17 00:00:00 2001 From: Sami Kama Date: Fri, 13 Apr 2018 15:18:48 -0700 Subject: [PATCH 354/791] Fix testing --- .../contrib/tensorrt/test/test_integration.py | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/tensorflow/contrib/tensorrt/test/test_integration.py b/tensorflow/contrib/tensorrt/test/test_integration.py index 8ad26c3f69..97915c2659 100644 --- a/tensorflow/contrib/tensorrt/test/test_integration.py +++ b/tensorflow/contrib/tensorrt/test/test_integration.py @@ -19,6 +19,7 @@ from __future__ import division from __future__ import print_function import numpy as np +import warnings from tensorflow.contrib import tensorrt as trt from tensorflow.core.protobuf import config_pb2 as cpb2 @@ -36,7 +37,7 @@ from tensorflow.python.platform import test @test_util.with_c_api -class IntegrationTest(test_util.TensofFlowTestCase): +class IntegrationTest(test_util.TensorFlowTestCase): def setUp(self): """ Setup method """ @@ -44,10 +45,10 @@ class IntegrationTest(test_util.TensofFlowTestCase): warnings.simplefilter('always') inp_dims = (100, 24, 24, 2) self._input = np.random.random_sample(inp_dims) - self._original_graph = get_simple_graph_def() + self._original_graph = self.get_simple_graph_def() self._gpu_options = cpb2.GPUOptions( per_process_gpu_memory_fraction=0.50) - self._config = cpb2.ConfigProto(gpu_options=gpu_options) + self._config = cpb2.ConfigProto(gpu_options=self._gpu_options) self._reference = self.run_graph(self._original_graph, self._input) def get_simple_graph_def(self): @@ -86,7 +87,7 @@ class IntegrationTest(test_util.TensofFlowTestCase): inp = inp.outputs[0] out = out.outputs[0] with self.test_session( - grap=g, config=self._config, use_gpu=True, + graph=g, config=self._config, use_gpu=True, force_gpu=True) as sess: val = sess.run(out, {inp: dumm_inp}) return val @@ -105,7 +106,7 @@ class IntegrationTest(test_util.TensofFlowTestCase): # run over real calibration data here, we are mimicking a calibration set of # 30 different batches. Use as much calibration data as you want with self.test_session( - grap=g, config=self._config, use_gpu=True, + graph=g, config=self._config, use_gpu=True, force_gpu=True) as sess: for _ in range(30): val = sess.run(out, {inp: dumm_inp}) @@ -115,9 +116,9 @@ class IntegrationTest(test_util.TensofFlowTestCase): """ return trt converted graph """ if mode == "FP32": return trt.create_inference_graph( - input_graph_def=self._orig_graph, + input_graph_def=self._original_graph, outputs=["output"], - max_batch_size=inp_dims[0], + max_batch_size=self._input.shape[0], max_workspace_size_bytes=1 << 25, precision_mode= "FP32", # TRT Engine precision "FP32","FP16" or "INT8" @@ -125,9 +126,9 @@ class IntegrationTest(test_util.TensofFlowTestCase): ) elif mode == "FP16": return trt.create_inference_graph( - input_graph_def=self._orig_graph, + input_graph_def=self._original_graph, outputs=["output"], - max_batch_size=inp_dims[0], + max_batch_size=self._input.shape[0], max_workspace_size_bytes=1 << 25, precision_mode= "FP16", # TRT Engine precision "FP32","FP16" or "INT8" @@ -135,9 +136,9 @@ class IntegrationTest(test_util.TensofFlowTestCase): ) elif mode == "INT8": return trt.create_inference_graph( - input_graph_def=self._orig_graph, + input_graph_def=self._original_graph, outputs=["output"], - max_batch_size=inp_dims[0], + max_batch_size=self._input.shape[0], max_workspace_size_bytes=1 << 25, precision_mode= "INT8", # TRT Engine precision "FP32","FP16" or "INT8" @@ -151,27 +152,27 @@ class IntegrationTest(test_util.TensofFlowTestCase): trt_graph = self.get_trt_graph("FP32") result = self.run_graph(trt_graph, self._input) self.assertAllEqual(self._reference, result) - result = self.run_graph(trt_graph, self._input) - self.assertAllEqual(self._reference, result) + result1 = self.run_graph(trt_graph, self._input) + self.assertAllEqual(result1, result) def testFP16(self): """ Test FP16 conversion. Results may be different from native case """ trt_graph = self.get_trt_graph("FP16") result = self.run_graph(trt_graph, self._input) - self.assertAllEqual(self._reference, result) - result = self.run_graph(trt_graph, self._input) - self.assertAllEqual(self._reference, result) + self.assertAllClose(self._reference, result,rtol=1.e-03) + result1 = self.run_graph(trt_graph, self._input) + self.assertAllEqual(result1, result) def testINT8(self): """ Test INT8 conversion. Results may be different from native case """ calib_graph = self.get_trt_graph("INT8") result = self.run_calibration(calib_graph, self._input) self.assertAllEqual(self._reference, result) - int8_graph = trt.calib_graph_to_infer_graph(int8_calib_gdef) - result = self.run_graph(int8_graph, self._input) - self.assertAllEqual(self._reference, result) + int8_graph = trt.calib_graph_to_infer_graph(calib_graph) result = self.run_graph(int8_graph, self._input) - self.assertAllEqual(self._reference, result) + self.assertAllClose(self._reference, result,rtol=1.e-03) + result1 = self.run_graph(int8_graph, self._input) + self.assertAllEqual(result1, result) if __name__ == '__main__': -- GitLab From a77dcb5e56dbbbcc3383cb0b39cd79dd88135635 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 15:23:08 -0700 Subject: [PATCH 355/791] Add broadcasting to all LinearOperators. This will broadcast in cases where batch shapes are not equal (but tries to determine statically if this is the case). The broadcasting is not as efficient as doing the broadcast in C++, but makes for the API to at least be completely broadcastable. PiperOrigin-RevId: 192832919 --- tensorflow/contrib/linalg/BUILD | 2 +- .../linear_operator_block_diag_test.py | 67 +--------------- .../python/ops/linalg/linear_operator.py | 5 +- .../ops/linalg/linear_operator_full_matrix.py | 4 +- .../linalg/linear_operator_low_rank_update.py | 25 +++--- .../linear_operator_lower_triangular.py | 5 +- .../ops/linalg/linear_operator_test_util.py | 76 ++++++++++++++----- 7 files changed, 82 insertions(+), 102 deletions(-) diff --git a/tensorflow/contrib/linalg/BUILD b/tensorflow/contrib/linalg/BUILD index a7812f74d1..8b7ff75ba5 100644 --- a/tensorflow/contrib/linalg/BUILD +++ b/tensorflow/contrib/linalg/BUILD @@ -58,6 +58,6 @@ cuda_py_test( "//tensorflow/python:math_ops", "//tensorflow/python:platform_test", ], - shard_count = 4, + shard_count = 5, tags = ["noasan"], ) diff --git a/tensorflow/contrib/linalg/python/kernel_tests/linear_operator_block_diag_test.py b/tensorflow/contrib/linalg/python/kernel_tests/linear_operator_block_diag_test.py index cc1a047d6a..e7407ede11 100644 --- a/tensorflow/contrib/linalg/python/kernel_tests/linear_operator_block_diag_test.py +++ b/tensorflow/contrib/linalg/python/kernel_tests/linear_operator_block_diag_test.py @@ -76,6 +76,8 @@ class SquareLinearOperatorBlockDiagTest( build_info((1, 1)), build_info((1, 3, 3)), build_info((5, 5), blocks=[(2, 2), (3, 3)]), + build_info((3, 7, 7), blocks=[(1, 2, 2), (3, 2, 2), (1, 3, 3)]), + build_info((2, 1, 5, 5), blocks=[(2, 1, 2, 2), (1, 3, 3)]), ] def _operator_and_mat_and_feed_dict(self, build_info, dtype, use_placeholder): @@ -184,70 +186,5 @@ class SquareLinearOperatorBlockDiagTest( block_diag.LinearOperatorBlockDiag([]) -# This test is for blocks with different batch dimensions. -# LinearOperatorFullMatrix doesn't broadcast matmul/solve. -class SquareDiagLinearOperatorBlockDiagTest( - linear_operator_test_util.SquareLinearOperatorDerivedClassTest): - """Most tests done in the base class LinearOperatorDerivedClassTest.""" - - def setUp(self): - # Increase from 1e-6 to 1e-4 - self._atol[dtypes.float32] = 1e-4 - self._atol[dtypes.complex64] = 1e-4 - self._rtol[dtypes.float32] = 1e-4 - self._rtol[dtypes.complex64] = 1e-4 - - @property - def _operator_build_infos(self): - build_info = linear_operator_test_util.OperatorBuildInfo - return [ - build_info((3, 7, 7), blocks=[(1, 2, 2), (3, 2, 2), (1, 3, 3)]), - build_info((2, 1, 6, 6), blocks=[(2, 1, 2, 2), (1, 1, 4, 4)]), - ] - - def _operator_and_mat_and_feed_dict(self, build_info, dtype, use_placeholder): - shape = list(build_info.shape) - expected_blocks = ( - build_info.__dict__["blocks"] if "blocks" in build_info.__dict__ - else [shape]) - diag_matrices = [ - linear_operator_test_util.random_uniform( - shape=block_shape[:-1], minval=1., maxval=20., dtype=dtype) - for block_shape in expected_blocks - ] - - if use_placeholder: - diag_matrices_ph = [ - array_ops.placeholder(dtype=dtype) for _ in expected_blocks - ] - diag_matrices = self.evaluate(diag_matrices) - # Evaluate here because (i) you cannot feed a tensor, and (ii) - # values are random and we want the same value used for both mat and - # feed_dict. - operator = block_diag.LinearOperatorBlockDiag( - [linalg.LinearOperatorDiag(m_ph) for m_ph in diag_matrices_ph]) - feed_dict = {m_ph: m for (m_ph, m) in zip( - diag_matrices_ph, diag_matrices)} - else: - operator = block_diag.LinearOperatorBlockDiag( - [linalg.LinearOperatorDiag(m) for m in diag_matrices]) - feed_dict = None - # Should be auto-set. - self.assertTrue(operator.is_square) - - # Broadcast the shapes. - expected_shape = list(build_info.shape) - - matrices = linear_operator_util.broadcast_matrix_batch_dims( - [array_ops.matrix_diag(diag_block) for diag_block in diag_matrices]) - - block_diag_dense = _block_diag_dense(expected_shape, matrices) - if not use_placeholder: - block_diag_dense.set_shape( - expected_shape[:-2] + [expected_shape[-1], expected_shape[-1]]) - - return operator, block_diag_dense, feed_dict - - if __name__ == "__main__": test.main() diff --git a/tensorflow/python/ops/linalg/linear_operator.py b/tensorflow/python/ops/linalg/linear_operator.py index 193c787baa..8cfe964b1c 100644 --- a/tensorflow/python/ops/linalg/linear_operator.py +++ b/tensorflow/python/ops/linalg/linear_operator.py @@ -699,9 +699,10 @@ class LinearOperator(object): " Requires conversion to a dense matrix and O(N^3) operations.") rhs = linalg.adjoint(rhs) if adjoint_arg else rhs if self._can_use_cholesky(): - return linalg_ops.cholesky_solve( + return linear_operator_util.cholesky_solve_with_broadcast( linalg_ops.cholesky(self.to_dense()), rhs) - return linalg_ops.matrix_solve(self.to_dense(), rhs, adjoint=adjoint) + return linear_operator_util.matrix_solve_with_broadcast( + self.to_dense(), rhs, adjoint=adjoint) def solve(self, rhs, adjoint=False, adjoint_arg=False, name="solve"): """Solve (exact or approx) `R` (batch) systems of equations: `A X = rhs`. diff --git a/tensorflow/python/ops/linalg/linear_operator_full_matrix.py b/tensorflow/python/ops/linalg/linear_operator_full_matrix.py index 5ba3b090ae..746da8df1c 100644 --- a/tensorflow/python/ops/linalg/linear_operator_full_matrix.py +++ b/tensorflow/python/ops/linalg/linear_operator_full_matrix.py @@ -21,8 +21,8 @@ from __future__ import print_function from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops from tensorflow.python.ops.linalg import linear_operator +from tensorflow.python.ops.linalg import linear_operator_util from tensorflow.python.util.tf_export import tf_export __all__ = ["LinearOperatorFullMatrix"] @@ -176,7 +176,7 @@ class LinearOperatorFullMatrix(linear_operator.LinearOperator): return array_ops.shape(self._matrix) def _matmul(self, x, adjoint=False, adjoint_arg=False): - return math_ops.matmul( + return linear_operator_util.matmul_with_broadcast( self._matrix, x, adjoint_a=adjoint, adjoint_b=adjoint_arg) def _to_dense(self): diff --git a/tensorflow/python/ops/linalg/linear_operator_low_rank_update.py b/tensorflow/python/ops/linalg/linear_operator_low_rank_update.py index be91102909..08e5896e10 100644 --- a/tensorflow/python/ops/linalg/linear_operator_low_rank_update.py +++ b/tensorflow/python/ops/linalg/linear_operator_low_rank_update.py @@ -27,6 +27,7 @@ from tensorflow.python.ops import math_ops from tensorflow.python.ops.linalg import linear_operator from tensorflow.python.ops.linalg import linear_operator_diag from tensorflow.python.ops.linalg import linear_operator_identity +from tensorflow.python.ops.linalg import linear_operator_util from tensorflow.python.util.tf_export import tf_export __all__ = [ @@ -365,14 +366,17 @@ class LinearOperatorLowRankUpdate(linear_operator.LinearOperator): leading_term = l.matmul(x, adjoint=adjoint, adjoint_arg=adjoint_arg) if adjoint: - uh_x = math_ops.matmul(u, x, adjoint_a=True, adjoint_b=adjoint_arg) + uh_x = linear_operator_util.matmul_with_broadcast( + u, x, adjoint_a=True, adjoint_b=adjoint_arg) d_uh_x = d.matmul(uh_x, adjoint=adjoint) - v_d_uh_x = math_ops.matmul(v, d_uh_x) + v_d_uh_x = linear_operator_util.matmul_with_broadcast( + v, d_uh_x) return leading_term + v_d_uh_x else: - vh_x = math_ops.matmul(v, x, adjoint_a=True, adjoint_b=adjoint_arg) + vh_x = linear_operator_util.matmul_with_broadcast( + v, x, adjoint_a=True, adjoint_b=adjoint_arg) d_vh_x = d.matmul(vh_x, adjoint=adjoint) - u_d_vh_x = math_ops.matmul(u, d_vh_x) + u_d_vh_x = linear_operator_util.matmul_with_broadcast(u, d_vh_x) return leading_term + u_d_vh_x def _determinant(self): @@ -431,16 +435,18 @@ class LinearOperatorLowRankUpdate(linear_operator.LinearOperator): # L^{-1} rhs linv_rhs = l.solve(rhs, adjoint=adjoint, adjoint_arg=adjoint_arg) # V^H L^{-1} rhs - vh_linv_rhs = math_ops.matmul(v, linv_rhs, adjoint_a=True) + vh_linv_rhs = linear_operator_util.matmul_with_broadcast( + v, linv_rhs, adjoint_a=True) # C^{-1} V^H L^{-1} rhs if self._use_cholesky: - capinv_vh_linv_rhs = linalg_ops.cholesky_solve( + capinv_vh_linv_rhs = linear_operator_util.cholesky_solve_with_broadcast( self._chol_capacitance, vh_linv_rhs) else: - capinv_vh_linv_rhs = linalg_ops.matrix_solve( + capinv_vh_linv_rhs = linear_operator_util.matrix_solve_with_broadcast( self._capacitance, vh_linv_rhs, adjoint=adjoint) # U C^{-1} V^H M^{-1} rhs - u_capinv_vh_linv_rhs = math_ops.matmul(u, capinv_vh_linv_rhs) + u_capinv_vh_linv_rhs = linear_operator_util.matmul_with_broadcast( + u, capinv_vh_linv_rhs) # L^{-1} U C^{-1} V^H L^{-1} rhs linv_u_capinv_vh_linv_rhs = l.solve(u_capinv_vh_linv_rhs, adjoint=adjoint) @@ -454,7 +460,8 @@ class LinearOperatorLowRankUpdate(linear_operator.LinearOperator): # L^{-1} U linv_u = self.base_operator.solve(self.u) # V^H L^{-1} U - vh_linv_u = math_ops.matmul(self.v, linv_u, adjoint_a=True) + vh_linv_u = linear_operator_util.matmul_with_broadcast( + self.v, linv_u, adjoint_a=True) # D^{-1} + V^H L^{-1} V capacitance = self._diag_inv_operator.add_to_tensor(vh_linv_u) diff --git a/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py b/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py index c4d386ccb4..fb1eb2fedb 100644 --- a/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py +++ b/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py @@ -21,7 +21,6 @@ from __future__ import print_function from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops -from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops.linalg import linalg_impl as linalg from tensorflow.python.ops.linalg import linear_operator @@ -194,7 +193,7 @@ class LinearOperatorLowerTriangular(linear_operator.LinearOperator): message="Singular operator: Diagonal contained zero values.") def _matmul(self, x, adjoint=False, adjoint_arg=False): - return math_ops.matmul( + return linear_operator_util.matmul_with_broadcast( self._tril, x, adjoint_a=adjoint, adjoint_b=adjoint_arg) def _determinant(self): @@ -206,7 +205,7 @@ class LinearOperatorLowerTriangular(linear_operator.LinearOperator): def _solve(self, rhs, adjoint=False, adjoint_arg=False): rhs = linalg.adjoint(rhs) if adjoint_arg else rhs - return linalg_ops.matrix_triangular_solve( + return linear_operator_util.matrix_triangular_solve_with_broadcast( self._tril, rhs, lower=True, adjoint=adjoint) def _to_dense(self): diff --git a/tensorflow/python/ops/linalg/linear_operator_test_util.py b/tensorflow/python/ops/linalg/linear_operator_test_util.py index ce1a112ad5..9c8abb9740 100644 --- a/tensorflow/python/ops/linalg/linear_operator_test_util.py +++ b/tensorflow/python/ops/linalg/linear_operator_test_util.py @@ -32,6 +32,7 @@ from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import random_ops from tensorflow.python.ops.linalg import linalg_impl as linalg +from tensorflow.python.ops.linalg import linear_operator_util from tensorflow.python.platform import test @@ -126,13 +127,16 @@ class LinearOperatorDerivedClassTest(test.TestCase): raise NotImplementedError("Not implemented yet.") @abc.abstractmethod - def _make_rhs(self, operator, adjoint): + def _make_rhs(self, operator, adjoint, with_batch=True): """Make a rhs appropriate for calling operator.solve(rhs). Args: operator: A `LinearOperator` adjoint: Python `bool`. If `True`, we are making a 'rhs' value for the adjoint operator. + with_batch: Python `bool`. If `True`, create `rhs` with the same batch + shape as operator, and otherwise create a matrix without any batch + shape. Returns: A `Tensor` @@ -140,13 +144,15 @@ class LinearOperatorDerivedClassTest(test.TestCase): raise NotImplementedError("_make_rhs is not defined.") @abc.abstractmethod - def _make_x(self, operator, adjoint): + def _make_x(self, operator, adjoint, with_batch=True): """Make an 'x' appropriate for calling operator.matmul(x). Args: operator: A `LinearOperator` adjoint: Python `bool`. If `True`, we are making an 'x' value for the adjoint operator. + with_batch: Python `bool`. If `True`, create `x` with the same batch shape + as operator, and otherwise create a matrix without any batch shape. Returns: A `Tensor` @@ -224,8 +230,7 @@ class LinearOperatorDerivedClassTest(test.TestCase): [op_log_abs_det, mat_log_abs_det], feed_dict=feed_dict) self.assertAC(op_log_abs_det_v, mat_log_abs_det_v) - def test_matmul(self): - self._skip_if_tests_to_skip_contains("matmul") + def _test_matmul(self, with_batch): for use_placeholder in self._use_placeholder_options: for build_info in self._operator_build_infos: for dtype in self._dtypes_to_test: @@ -235,7 +240,8 @@ class LinearOperatorDerivedClassTest(test.TestCase): sess.graph.seed = random_seed.DEFAULT_GRAPH_SEED operator, mat, feed_dict = self._operator_and_mat_and_feed_dict( build_info, dtype, use_placeholder=use_placeholder) - x = self._make_x(operator, adjoint=adjoint) + x = self._make_x( + operator, adjoint=adjoint, with_batch=with_batch) # If adjoint_arg, compute A X^H^H = A X. if adjoint_arg: op_matmul = operator.matmul( @@ -244,7 +250,8 @@ class LinearOperatorDerivedClassTest(test.TestCase): adjoint_arg=adjoint_arg) else: op_matmul = operator.matmul(x, adjoint=adjoint) - mat_matmul = math_ops.matmul(mat, x, adjoint_a=adjoint) + mat_matmul = linear_operator_util.matmul_with_broadcast( + mat, x, adjoint_a=adjoint) if not use_placeholder: self.assertAllEqual(op_matmul.get_shape(), mat_matmul.get_shape()) @@ -252,8 +259,15 @@ class LinearOperatorDerivedClassTest(test.TestCase): [op_matmul, mat_matmul], feed_dict=feed_dict) self.assertAC(op_matmul_v, mat_matmul_v) - def test_solve(self): - self._skip_if_tests_to_skip_contains("solve") + def test_matmul(self): + self._skip_if_tests_to_skip_contains("matmul") + self._test_matmul(with_batch=True) + + def test_matmul_with_broadcast(self): + self._skip_if_tests_to_skip_contains("matmul_with_broadcast") + self._test_matmul(with_batch=False) + + def _test_solve(self, with_batch): for use_placeholder in self._use_placeholder_options: for build_info in self._operator_build_infos: for dtype in self._dtypes_to_test: @@ -263,7 +277,8 @@ class LinearOperatorDerivedClassTest(test.TestCase): sess.graph.seed = random_seed.DEFAULT_GRAPH_SEED operator, mat, feed_dict = self._operator_and_mat_and_feed_dict( build_info, dtype, use_placeholder=use_placeholder) - rhs = self._make_rhs(operator, adjoint=adjoint) + rhs = self._make_rhs( + operator, adjoint=adjoint, with_batch=with_batch) # If adjoint_arg, solve A X = (rhs^H)^H = rhs. if adjoint_arg: op_solve = operator.solve( @@ -273,7 +288,8 @@ class LinearOperatorDerivedClassTest(test.TestCase): else: op_solve = operator.solve( rhs, adjoint=adjoint, adjoint_arg=adjoint_arg) - mat_solve = linalg_ops.matrix_solve(mat, rhs, adjoint=adjoint) + mat_solve = linear_operator_util.matrix_solve_with_broadcast( + mat, rhs, adjoint=adjoint) if not use_placeholder: self.assertAllEqual(op_solve.get_shape(), mat_solve.get_shape()) @@ -281,6 +297,14 @@ class LinearOperatorDerivedClassTest(test.TestCase): [op_solve, mat_solve], feed_dict=feed_dict) self.assertAC(op_solve_v, mat_solve_v) + def test_solve(self): + self._skip_if_tests_to_skip_contains("solve") + self._test_solve(with_batch=True) + + def test_solve_with_broadcast(self): + self._skip_if_tests_to_skip_contains("solve_with_broadcast") + self._test_solve(with_batch=False) + def test_trace(self): self._skip_if_tests_to_skip_contains("trace") for use_placeholder in self._use_placeholder_options: @@ -358,13 +382,13 @@ class SquareLinearOperatorDerivedClassTest(LinearOperatorDerivedClassTest): build_info((3, 4, 4)), build_info((2, 1, 4, 4))] - def _make_rhs(self, operator, adjoint): + def _make_rhs(self, operator, adjoint, with_batch=True): # This operator is square, so rhs and x will have same shape. # adjoint value makes no difference because the operator shape doesn't # change since it is square, but be pedantic. - return self._make_x(operator, adjoint=not adjoint) + return self._make_x(operator, adjoint=not adjoint, with_batch=with_batch) - def _make_x(self, operator, adjoint): + def _make_x(self, operator, adjoint, with_batch=True): # Value of adjoint makes no difference because the operator is square. # Return the number of systems to solve, R, equal to 1 or 2. r = self._get_num_systems(operator) @@ -373,11 +397,17 @@ class SquareLinearOperatorDerivedClassTest(LinearOperatorDerivedClassTest): if operator.shape.is_fully_defined(): batch_shape = operator.batch_shape.as_list() n = operator.domain_dimension.value - x_shape = batch_shape + [n, r] + if with_batch: + x_shape = batch_shape + [n, r] + else: + x_shape = [n, r] else: batch_shape = operator.batch_shape_tensor() n = operator.domain_dimension_tensor() - x_shape = array_ops.concat((batch_shape, [n, r]), 0) + if with_batch: + x_shape = array_ops.concat((batch_shape, [n, r]), 0) + else: + x_shape = [n, r] return random_normal(x_shape, dtype=operator.dtype) @@ -404,7 +434,7 @@ class NonSquareLinearOperatorDerivedClassTest(LinearOperatorDerivedClassTest): @property def _tests_to_skip(self): """List of test names to skip.""" - return ["solve", "det", "log_abs_det"] + return ["solve", "solve_with_broadcast", "det", "log_abs_det"] @property def _operator_build_infos(self): @@ -417,12 +447,12 @@ class NonSquareLinearOperatorDerivedClassTest(LinearOperatorDerivedClassTest): build_info((3, 3, 4)), build_info((2, 1, 2, 4))] - def _make_rhs(self, operator, adjoint): + def _make_rhs(self, operator, adjoint, with_batch=True): # TODO(langmore) Add once we're testing solve_ls. raise NotImplementedError( "_make_rhs not implemented because we don't test solve") - def _make_x(self, operator, adjoint): + def _make_x(self, operator, adjoint, with_batch=True): # Return the number of systems for the argument 'x' for .matmul(x) r = self._get_num_systems(operator) # If operator.shape = [B1,...,Bb, M, N] this returns a random matrix of @@ -433,14 +463,20 @@ class NonSquareLinearOperatorDerivedClassTest(LinearOperatorDerivedClassTest): n = operator.range_dimension.value else: n = operator.domain_dimension.value - x_shape = batch_shape + [n, r] + if with_batch: + x_shape = batch_shape + [n, r] + else: + x_shape = [n, r] else: batch_shape = operator.batch_shape_tensor() if adjoint: n = operator.range_dimension_tensor() else: n = operator.domain_dimension_tensor() - x_shape = array_ops.concat((batch_shape, [n, r]), 0) + if with_batch: + x_shape = array_ops.concat((batch_shape, [n, r]), 0) + else: + x_shape = [n, r] return random_normal(x_shape, dtype=operator.dtype) -- GitLab From a22344f82ddd1e877f0b9f82584b9bb1d6c8dc16 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 15:32:11 -0700 Subject: [PATCH 356/791] [XLA] Pattern matcher for HLO, Shapes, Layouts PiperOrigin-RevId: 192834129 --- tensorflow/compiler/xla/service/BUILD | 23 + .../compiler/xla/service/pattern_matcher.h | 1014 +++++++++++++++++ .../xla/service/pattern_matcher_test.cc | 144 +++ tensorflow/compiler/xla/shape_util.cc | 12 + tensorflow/compiler/xla/shape_util.h | 3 + 5 files changed, 1196 insertions(+) create mode 100644 tensorflow/compiler/xla/service/pattern_matcher.h create mode 100644 tensorflow/compiler/xla/service/pattern_matcher_test.cc diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index 65203fa2a0..ddc099807d 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -302,6 +302,29 @@ tf_cc_test( ], ) +cc_library( + name = "pattern_matcher", + hdrs = ["pattern_matcher.h"], + deps = [ + ":hlo", + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/core:lib", + ], +) + +tf_cc_test( + name = "pattern_matcher_test", + srcs = ["pattern_matcher_test.cc"], + deps = [ + ":hlo", + ":pattern_matcher", + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla/tests:xla_internal_test_main", + "//tensorflow/compiler/xla/tools/parser:hlo_parser", + "//tensorflow/core:test", + ], +) + cc_library( name = "hlo_reachability", srcs = ["hlo_reachability.cc"], diff --git a/tensorflow/compiler/xla/service/pattern_matcher.h b/tensorflow/compiler/xla/service/pattern_matcher.h new file mode 100644 index 0000000000..5d49638077 --- /dev/null +++ b/tensorflow/compiler/xla/service/pattern_matcher.h @@ -0,0 +1,1014 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_PATTERN_MATCHER_H_ +#define TENSORFLOW_COMPILER_XLA_SERVICE_PATTERN_MATCHER_H_ + +#include "tensorflow/compiler/xla/layout_util.h" +#include "tensorflow/compiler/xla/service/hlo_instruction.h" +#include "tensorflow/compiler/xla/service/hlo_opcode.h" +#include "tensorflow/compiler/xla/shape_util.h" +#include "tensorflow/core/lib/core/stringpiece.h" + +namespace xla { + +// A pattern matcher for HloInstructions, Shapes, and Layouts. +// +// The Match function's first argument must be HloInstruction*, Shape*, or +// Layout*. The second argument is a pattern that will be matched against the +// first argument, as described below. +// +// Patterns are constructed using the match::Op, match::Shape, or match::Layout +// functions. By default, the returned patterns will match any HloInstruction, +// Shape, or Layout, respectively. However the match can be made more specific +// by using the pattern's modifier methods, for example: +// +// match::Op().WithOpcode(HloOpcode::kAdd).WithOperand( +// 0, match::Op().WithOpcode(HloOpcode::kConstant)) +// +// This pattern will match Add instructions whose first operand is a constant. +// +// Each pattern type has the following modifiers: +// +// Op(): +// - WithName: match operations with the given name +// - WithOpcode: match operations with the given opcode +// - WithShape: match operations whose shape matches the given pattern +// - WithOperand: match operations whose operand matches the given pattern +// +// Shape(): +// - EqualTo: matches shapes that are equal to the argument +// - CompatibleTo: matches shapes that are compatible to the argument +// - IsScalar/IsArray/IsTuple: matches scalar/array/tuple shapes +// - IsDenseArray/IsSparseArray: matches arrays with dense/sparse format +// - WithLayout: match shapes whose layout matches the given pattern +// - WithLayoutEqualTo: matches shapes whose layouts equal the argument +// - WithSubshape: matches tuple shapes whose subshape matches the given +// pattern +// - WithSubshapeEqualTo: matches shapes with a subshape equal the argument +// - WithElementType: matches array/scalar shapes with the given element +// type +// - WithRank: matches array/scalar types with the given rank +// +// Layout(): +// - EqualTo: matches layouts that are equal to the argument +// - WithDenseFormat/WithSparseFormat: matches layouts with dense/sparse +// format +// +// Op(), Shape(), and Layout() may be passed an argument of type +// HloInstruction**, Shape**, or Layout**, respectively, or const versions of +// these pointers. If the pattern is matched, the address of the matched value +// will be "captured" and stored at this location. +// +// For example: +// HloInstruction* foo = ...; +// HloInstruction* matched_operand; +// CHECK(Match(foo, +// match::Op().WithOperand(0, match::Op(&matched_operand)))); +// +// Helpers are provided for common nullary, unary, binary, and ternary +// instructions. These helpers can be called with no arguments, in which case +// they will match any instruction matching the opcode. They may also be called +// with matches for the operands and with an optional capture. (The capture must +// be the first argument.) Some examples of these helpers and their equivalents +// are provided below. +// +// Example nullary instruction: +// Recv() == Op().WithOpcode(HloOpcode::kRecv) +// Recv(&a) == Op(&a).WithOpcode(HloOpcode::kRecv) +// +// Example unary instruction: +// Abs() == Op().WithOpcode(HloOpcode::kAbs) +// Abs(Op(&a)) == Op().WithOpcode(HloOpcode::kAbs) +// .WithOperand(0, Op(&a))) +// Abs(&a, Op(&b)) == Op(&a).WithOpcode(HloOpcode::kAbs) +// .WithOperand(0, Op(&b)) +// +// Example binary instruction: +// Add() == Op().WithOpcode(HloOpcode::kAdd) +// Add(Op(&a), Op(&b)) == Op().WithOpcode(HloOpcode::kAdd) +// .WithOperand(0, Op(&a)) +// .WithOperand(1, Op(&b)) +// Add(&a, Op(&b), Op(&c)) == Op(&a).WithOpcode(HloOpcode::kAdd) +// .WithOperand(0, Op(&b)) +// .WithOperand(1, Op(&c)) +// +// Example ternary instruction: +// Clamp() == Op().WithOpcode(HloOpcode::kClamp) +// Clamp(Op(&a), Op(&b), Op(&c)) == Op().WithOpcode(HloOpcode::kClamp) +// .WithOperand(0, Op(&a)) +// .WithOperand(1, Op(&b)) +// .WithOperand(2, Op(&c)) +// Clamp(&a, Op(&b), Op(&c), Op(&d)) == Op(&a).WithOpcode(HloOpcode::kClamp) +// .WithOperand(0, Op(&b)) +// .WithOperand(1, Op(&c)) +// .WithOperand(2, Op(&d)) +// +template +bool Match(Value* value, const Pattern& pattern) { + return pattern.Match(value); +} + +namespace match { + +namespace detail { + +template +class LayoutPattern; + +// The base LayoutPattern implementation. Matches only if the layout is not +// nullptr. +class LayoutPatternBaseImpl { + public: + bool Match(const ::xla::Layout* layout) const { return layout != nullptr; } +}; + +// A LayoutPattern implementation that matches only if the layout equals a +// Layout proto. +template +class LayoutPatternEqualImpl { + public: + explicit constexpr LayoutPatternEqualImpl(const Previous& previous, + const ::xla::Layout* layout) + : previous_(previous), layout_(layout) {} + + bool Match(const ::xla::Layout* layout) const { + return previous_.Match(layout) && LayoutUtil::Equal(*layout_, *layout); + } + + private: + Previous previous_; + const ::xla::Layout* layout_; +}; + +// A LayoutPattern implementation that matches only if the layout has a given +// format. +template +class LayoutPatternFormatImpl { + public: + explicit constexpr LayoutPatternFormatImpl(const Previous& previous, + Format format) + : previous_(previous), format_(format) {} + + bool Match(const ::xla::Layout* layout) const { + return previous_.Match(layout) && layout->format() == format_; + } + + private: + Previous previous_; + Format format_; +}; + +// A pattern that matches Layouts. +template +class LayoutPattern { + public: + explicit constexpr LayoutPattern(const Impl& impl, + LayoutType** matched_layout) + : impl_(impl), matched_layout_(matched_layout) {} + + // Returns true and captures the layout iff it matches the pattern. + bool Match(const ::xla::Layout* layout) const { + if (impl_.Match(layout)) { + if (matched_layout_) { + *matched_layout_ = layout; + } + return true; + } + return false; + } + + // Returns true and captures the layout iff it matches the pattern. + bool Match(::xla::Layout* layout) const { + if (impl_.Match(layout)) { + if (matched_layout_) { + *matched_layout_ = layout; + } + return true; + } + return false; + } + + // Modifies the pattern to match only if the layout equals the given proto. + // The layout must outlive the returned pattern. + constexpr LayoutPattern> EqualTo( + const Layout* layout) const { + return LayoutPattern>( + LayoutPatternEqualImpl(impl_, layout), matched_layout_); + } + + // Modifies the pattern to match only if the layout has a dense format. + constexpr LayoutPattern> + WithDenseFormat() const { + return LayoutPattern>( + LayoutPatternFormatImpl(impl_, DENSE), matched_layout_); + } + + // Modifies the pattern to match only if the layout has a sparse format. + constexpr LayoutPattern> + WithSparseFormat() const { + return LayoutPattern>( + LayoutPatternFormatImpl(impl_, SPARSE), matched_layout_); + } + + private: + Impl impl_; + LayoutType** matched_layout_; +}; + +} // namespace detail + +// Creates a layout pattern that will capture the matched layout in the +// argument. +inline constexpr detail::LayoutPattern +Layout(const ::xla::Layout** matched_layout = nullptr) { + return detail::LayoutPattern( + detail::LayoutPatternBaseImpl(), matched_layout); +} + +// Creates a layout pattern that will capture the matched layout in the +// argument. +inline constexpr detail::LayoutPattern<::xla::Layout, + detail::LayoutPatternBaseImpl> +Layout(::xla::Layout** matched_layout) { + return detail::LayoutPattern<::xla::Layout, detail::LayoutPatternBaseImpl>( + detail::LayoutPatternBaseImpl(), matched_layout); +} + +namespace detail { + +template +class ShapePattern; + +// The base ShapePattern implementation. Matches only if the shape is not +// nullptr. +class ShapePatternBaseImpl { + public: + bool Match(const ::xla::Shape* shape) const { return shape != nullptr; } +}; + +// A ShapePattern implementation that matches only if the shape equals a Shape +// proto. +template +class ShapePatternEqualImpl { + public: + explicit constexpr ShapePatternEqualImpl(const Previous& previous, + const ::xla::Shape* shape) + : previous_(previous), shape_(shape) {} + + bool Match(const ::xla::Shape* shape) const { + return previous_.Match(shape) && ShapeUtil::Equal(*shape_, *shape); + } + + private: + Previous previous_; + const ::xla::Shape* shape_; +}; + +// A ShapePattern implementation that matches only if the shape is compatible to +// a Shape proto. +template +class ShapePatternCompatibleImpl { + public: + explicit constexpr ShapePatternCompatibleImpl(const Previous& previous, + const ::xla::Shape* shape) + : previous_(previous), shape_(shape) {} + + bool Match(const ::xla::Shape* shape) const { + return previous_.Match(shape) && ShapeUtil::Compatible(*shape_, *shape); + } + + private: + Previous previous_; + const ::xla::Shape* shape_; +}; + +// A ShapePattern implementation that matches only if the shape has a given +// element type. +template +class ShapePatternElementTypeImpl { + public: + explicit constexpr ShapePatternElementTypeImpl(const Previous& previous, + PrimitiveType element_type) + : previous_(previous), element_type_(element_type) {} + + bool Match(const ::xla::Shape* shape) const { + return previous_.Match(shape) && shape->element_type() == element_type_; + } + + private: + Previous previous_; + PrimitiveType element_type_; +}; + +// A ShapePattern implementation that matches only if the shape is scalar. +template +class ShapePatternIsScalarImpl { + public: + explicit constexpr ShapePatternIsScalarImpl(const Previous& previous) + : previous_(previous) {} + + bool Match(const ::xla::Shape* shape) const { + return previous_.Match(shape) && ShapeUtil::IsScalar(*shape); + } + + private: + Previous previous_; +}; + +// A ShapePattern implementation that matches only if the shape is an array +template +class ShapePatternIsArrayImpl { + public: + explicit constexpr ShapePatternIsArrayImpl(const Previous& previous) + : previous_(previous) {} + + bool Match(const ::xla::Shape* shape) const { + return previous_.Match(shape) && ShapeUtil::IsArray(*shape); + } + + private: + Previous previous_; +}; + +// A ShapePattern implementation that matches only if the shape is a tuple. +template +class ShapePatternIsTupleImpl { + public: + explicit constexpr ShapePatternIsTupleImpl(const Previous& previous) + : previous_(previous) {} + + bool Match(const ::xla::Shape* shape) const { + return previous_.Match(shape) && ShapeUtil::IsTuple(*shape); + } + + private: + Previous previous_; +}; + +// A ShapePattern implementation that matches only if the shape has a given +// rank. +template +class ShapePatternRankImpl { + public: + explicit constexpr ShapePatternRankImpl(const Previous& previous, int64 rank) + : previous_(previous), rank_(rank) {} + + bool Match(const ::xla::Shape* shape) const { + return previous_.Match(shape) && ShapeUtil::Rank(*shape) == rank_; + } + + private: + Previous previous_; + int64 rank_; +}; + +// A ShapePattern implementation that matches only if the shape has a layout +// that matches a given pattern. +template +class ShapePatternLayoutImpl { + public: + explicit constexpr ShapePatternLayoutImpl( + const Previous& previous, + const LayoutPattern& layout) + : previous_(previous), layout_(layout) {} + + bool Match(const ::xla::Shape* shape) const { + return previous_.Match(shape) && LayoutUtil::HasLayout(*shape) && + layout_.Match(&shape->layout()); + } + + bool Match(Shape* shape) const { + return previous_.Match(shape) && LayoutUtil::HasLayout(*shape) && + layout_.Match(shape->mutable_layout()); + } + + private: + Previous previous_; + LayoutPattern layout_; +}; + +// A ShapePattern implementation that matches only if the shape has a subshape +// that matches a given pattern. +template +class ShapePatternSubshapeImpl { + public: + explicit ShapePatternSubshapeImpl( + const Previous& previous, ShapeIndexView index, + const ShapePattern& subshape) + : previous_(previous), index_(index), subshape_(subshape) {} + + bool Match(const ::xla::Shape* shape) const { + return previous_.Match(shape) && ShapeUtil::IndexIsValid(*shape, index_) && + subshape_.Match(&ShapeUtil::GetSubshape(*shape, index_)); + } + + bool Match(::xla::Shape* shape) const { + return previous_.Match(shape) && ShapeUtil::IndexIsValid(*shape, index_) && + subshape_.Match(ShapeUtil::GetMutableSubshape(shape, index_)); + } + + private: + Previous previous_; + ShapeIndexView index_; + ShapePattern subshape_; +}; + +// A pattern that matches Shapes. +template +class ShapePattern { + public: + explicit constexpr ShapePattern(const Impl& impl, ShapeType** matched_shape) + : impl_(impl), matched_shape_(matched_shape) {} + + // Returns true and captures the shape iff it matches the pattern. + bool Match(const ::xla::Shape* shape) const { + if (impl_.Match(shape)) { + if (matched_shape_) { + *matched_shape_ = shape; + } + return true; + } + return false; + } + + // Returns true and captures the shape iff it matches the pattern. + bool Match(::xla::Shape* shape) const { + if (impl_.Match(shape)) { + if (matched_shape_) { + *matched_shape_ = shape; + } + return true; + } + return false; + } + + // Modifies the pattern to match only if the shape equals the given proto. + // The layout must outlive the returned pattern. + constexpr ShapePattern> EqualTo( + const ::xla::Shape* shape) const { + return ShapePattern>( + ShapePatternEqualImpl(impl_, shape), matched_shape_); + } + + // Modifies the pattern to match only if the shape is compatible to the given + // proto. The layout must outlive the returned pattern. + constexpr ShapePattern> + CompatibleTo(const ::xla::Shape* shape) const { + return ShapePattern>( + ShapePatternCompatibleImpl(impl_, shape), matched_shape_); + } + + // Modifies the pattern to match only if the shape has the given element type. + constexpr ShapePattern> + WithElementType(PrimitiveType element_type) const { + return ShapePattern>( + ShapePatternElementTypeImpl(impl_, element_type), matched_shape_); + } + + // Modifies the pattern to match only if the shape is scalar. + constexpr ShapePattern> IsScalar() + const { + return ShapePattern>( + ShapePatternIsScalarImpl(impl_), matched_shape_); + } + + // Modifies the pattern to match only if the shape is an array. + constexpr ShapePattern> IsArray() + const { + return ShapePattern>( + ShapePatternIsArrayImpl(impl_), matched_shape_); + } + + // Modifies the pattern to match only if the shape is a tuple. + constexpr ShapePattern> IsTuple() + const { + return ShapePattern>( + ShapePatternIsTupleImpl(impl_), matched_shape_); + } + + // Modifies the pattern to match only if the shape has the given rank. + constexpr ShapePattern> WithRank( + int64 rank) const { + return ShapePattern>( + ShapePatternRankImpl(impl_, rank), matched_shape_); + } + + // Modifies the pattern to match only if the shape has a layout that matches + // the given pattern. + template + constexpr ShapePattern> + WithLayout(const LayoutPattern& layout) const { + return ShapePattern>( + ShapePatternLayoutImpl(impl_, layout), + matched_shape_); + } + + constexpr ShapePattern< + ShapeType, + ShapePatternLayoutImpl>> + WithLayoutEqualTo(const ::xla::Layout* layout) const { + return WithLayout(Layout().EqualTo(layout)); + } + + constexpr ShapePattern< + ShapeType, + ShapePatternLayoutImpl>> + IsDenseArray(const ::xla::Layout* layout) const { + return WithLayout(Layout().WithDenseFormat()); + } + + constexpr ShapePattern< + ShapeType, + ShapePatternLayoutImpl>> + IsSparseArray(const ::xla::Layout* layout) const { + return WithLayout(Layout().WithSparseFormat()); + } + + // Modifies the pattern to match only if the shape has a subshape that matches + // the given pattern. + template + ShapePattern> + WithSubshape(ShapeIndexView index, + const ShapePattern& subshape) const { + return ShapePattern< + ShapeType, ShapePatternSubshapeImpl>( + ShapePatternSubshapeImpl(impl_, index, + subshape), + matched_shape_); + } + + ShapePattern>> + WithSubshapeEqualTo(ShapeIndexView index, const ::xla::Shape* shape) const { + return WithSubshape(index, + ShapePattern( + ShapePatternBaseImpl(), nullptr) + .EqualTo(shape)); + } + + ShapePattern>> + WithSubshapeCompatibleTo(ShapeIndexView index, + const ::xla::Shape* shape) const { + return WithSubshape(index, + ShapePattern( + ShapePatternBaseImpl(), nullptr) + .CompatibleTo(shape)); + } + + private: + Impl impl_; + ShapeType** matched_shape_; +}; + +} // namespace detail + +// Creates a shape pattern that will capture the matched layout in the argument. +inline constexpr detail::ShapePattern +Shape(const ::xla::Shape** matched_shape = nullptr) { + return detail::ShapePattern( + detail::ShapePatternBaseImpl(), matched_shape); +} + +// Creates a shape pattern that will capture the matched layout in the argument. +inline constexpr detail::ShapePattern<::xla::Shape, + detail::ShapePatternBaseImpl> +Shape(::xla::Shape** matched_shape) { + return detail::ShapePattern<::xla::Shape, detail::ShapePatternBaseImpl>( + detail::ShapePatternBaseImpl(), matched_shape); +} + +namespace detail { + +template +class HloInstructionPattern; + +// The base HloInstructionPattern implementation. Matches only if the +// instruction is not nullptr. +class HloInstructionPatternBaseImpl { + public: + bool Match(const ::xla::HloInstruction* inst) const { + return inst != nullptr; + } +}; + +// An HloInstructionPattern implementation that matches only if the instruction +// has a given name. +template +class HloInstructionPatternNameImpl { + public: + explicit HloInstructionPatternNameImpl(const Previous& previous, + tensorflow::StringPiece name) + : previous_(previous), name_(name) {} + + bool Match(const ::xla::HloInstruction* inst) const { + return previous_.Match(inst) && inst->name() == name_; + } + + private: + Previous previous_; + tensorflow::StringPiece name_; +}; + +// An HloInstructionPattern implementation that matches only if the instruction +// has a given opcode. +template +class HloInstructionPatternOpcodeImpl { + public: + explicit constexpr HloInstructionPatternOpcodeImpl(const Previous& previous, + HloOpcode opcode, + bool invert) + : previous_(previous), opcode_(opcode), invert_(invert) {} + + bool Match(const ::xla::HloInstruction* inst) const { + return previous_.Match(inst) && (invert_ ^ (inst->opcode() == opcode_)); + } + + private: + Previous previous_; + HloOpcode opcode_; + bool invert_; +}; + +// An HloInstructionPattern implementation that matches only if the instruction +// has a shape that matches a given pattern. +template +class HloInstructionPatternShapeImpl { + public: + explicit constexpr HloInstructionPatternShapeImpl( + const Previous& previous, const ShapePattern& shape) + : previous_(previous), shape_(shape) {} + + bool Match(const ::xla::HloInstruction* inst) const { + return previous_.Match(inst) && shape_.Match(&inst->shape()); + } + + bool Match(::xla::HloInstruction* inst) const { + return previous_.Match(inst) && shape_.Match(inst->mutable_shape()); + } + + private: + Previous previous_; + ShapePattern shape_; +}; + +// An HloInstructionPattern implementation that matches only if the instruction +// has an operand that matches a given pattern. +template +class HloInstructionPatternOperandImpl { + public: + explicit constexpr HloInstructionPatternOperandImpl( + const Previous& previous, int64 operand_index, + const HloInstructionPattern& operand) + : previous_(previous), operand_index_(operand_index), operand_(operand) {} + + bool Match(const ::xla::HloInstruction* inst) const { + return previous_.Match(inst) && operand_index_ < inst->operand_count() && + operand_.Match(inst->operand(operand_index_)); + } + + bool Match(::xla::HloInstruction* inst) const { + return previous_.Match(inst) && operand_index_ < inst->operand_count() && + operand_.Match(inst->mutable_operand(operand_index_)); + } + + private: + Previous previous_; + int64 operand_index_; + HloInstructionPattern operand_; +}; + +// A pattern that matches HloInstructions. +template +class HloInstructionPattern { + public: + explicit constexpr HloInstructionPattern(const Impl& impl, + HloInstructionType** matched_inst) + : impl_(impl), matched_inst_(matched_inst) {} + + // Returns true and captures the instruction iff it matches the pattern. + bool Match(const ::xla::HloInstruction* inst) const { + if (impl_.Match(inst)) { + if (matched_inst_) { + *matched_inst_ = inst; + } + return true; + } + return false; + } + + // Returns true and captures the instruction iff it matches the pattern. + bool Match(::xla::HloInstruction* inst) const { + if (impl_.Match(inst)) { + if (matched_inst_) { + *matched_inst_ = inst; + } + return true; + } + return false; + } + + // Modifies the pattern to match only if the instruction has the given name. + HloInstructionPattern> + WithName(tensorflow::StringPiece name) const { + return HloInstructionPattern>( + HloInstructionPatternNameImpl(impl_, name), matched_inst_); + } + + // Modifies the pattern to match only if the instruction has the given opcode. + constexpr HloInstructionPattern> + WithOpcode(HloOpcode opcode) const { + return HloInstructionPattern>( + HloInstructionPatternOpcodeImpl(impl_, opcode, false), + matched_inst_); + } + + // Modifies the pattern to match only if the instruction does not have the + // given opcode. + constexpr HloInstructionPattern> + WithoutOpcode(HloOpcode opcode) const { + return HloInstructionPattern>( + HloInstructionPatternOpcodeImpl(impl_, opcode, true), + matched_inst_); + } + + // Modifies the pattern to match only if the instruction is a constant. + constexpr HloInstructionPattern> + IsConstant() const { + return WithOpcode(HloOpcode::kConstant); + } + + // Modifies the pattern to match only if the instruction is not a constant. + constexpr HloInstructionPattern> + IsNonConstant() const { + return WithoutOpcode(HloOpcode::kConstant); + } + + // Modifies the pattern to match only if the instruction has a shape that + // matches the given pattern. + template + constexpr HloInstructionPattern< + HloInstructionType, + HloInstructionPatternShapeImpl> + WithShape(const ShapePattern& shape) const { + return HloInstructionPattern< + HloInstructionType, + HloInstructionPatternShapeImpl>( + HloInstructionPatternShapeImpl(impl_, + shape), + matched_inst_); + } + + // Modifies the pattern to match only if the instruction has an operand that + // matches the given pattern. + template + constexpr HloInstructionPattern< + HloInstructionType, + HloInstructionPatternOperandImpl> + WithOperand( + int64 operand_index, + const HloInstructionPattern& operand) const { + return HloInstructionPattern< + HloInstructionType, + HloInstructionPatternOperandImpl>( + HloInstructionPatternOperandImpl( + impl_, operand_index, operand), + matched_inst_); + } + + private: + Impl impl_; + HloInstructionType** matched_inst_; +}; + +} // namespace detail + +// Creates an instruction pattern that will capture the matched instruction in +// the argument. +inline constexpr detail::HloInstructionPattern< + const ::xla::HloInstruction, detail::HloInstructionPatternBaseImpl> +Op(const ::xla::HloInstruction** matched_inst = nullptr) { + return detail::HloInstructionPattern( + detail::HloInstructionPatternBaseImpl(), matched_inst); +} + +// Creates an instruction pattern that will capture the matched instruction in +// the argument. +inline constexpr detail::HloInstructionPattern< + ::xla::HloInstruction, detail::HloInstructionPatternBaseImpl> +Op(::xla::HloInstruction** matched_inst) { + return detail::HloInstructionPattern<::xla::HloInstruction, + detail::HloInstructionPatternBaseImpl>( + detail::HloInstructionPatternBaseImpl(), matched_inst); +} + +// Helpers for nullary instructions. +#define XLA_NULLOP_PATTERN(NAME) \ + inline auto NAME()->decltype(Op().WithOpcode(HloOpcode::k##NAME)) { \ + return Op().WithOpcode(HloOpcode::k##NAME); \ + } \ + \ + template \ + inline auto NAME(HloInstructionType** matched_inst) \ + ->decltype(Op(matched_inst).WithOpcode(HloOpcode::k##NAME)) { \ + return Op(matched_inst).WithOpcode(HloOpcode::k##NAME); \ + } +XLA_NULLOP_PATTERN(Constant) +XLA_NULLOP_PATTERN(Infeed) +XLA_NULLOP_PATTERN(Parameter) +XLA_NULLOP_PATTERN(Recv) +#undef XLA_NULLOP_PATTERN + +// Helpers for unary instructions. +#define XLA_UNOP_PATTERN(NAME) \ + inline auto NAME()->decltype(Op().WithOpcode(HloOpcode::k##NAME)) { \ + return Op().WithOpcode(HloOpcode::k##NAME); \ + } \ + \ + template \ + inline auto NAME(Arg&& arg)->decltype( \ + Op().WithOpcode(HloOpcode::k##NAME) \ + .WithOperand(0, std::forward(arg))) { \ + return Op() \ + .WithOpcode(HloOpcode::k##NAME) \ + .WithOperand(0, std::forward(arg)); \ + } \ + \ + template \ + inline auto NAME(HloInstructionType** matched_inst, Arg&& arg) \ + ->decltype(Op(matched_inst) \ + .WithOpcode(HloOpcode::k##NAME) \ + .WithOperand(0, std::forward(arg))) { \ + return Op(matched_inst) \ + .WithOpcode(HloOpcode::k##NAME) \ + .WithOperand(0, std::forward(arg)); \ + } +XLA_UNOP_PATTERN(Abs) +XLA_UNOP_PATTERN(RoundNearestAfz) +XLA_UNOP_PATTERN(Bitcast) +XLA_UNOP_PATTERN(Broadcast) +XLA_UNOP_PATTERN(BroadcastDimOne) +XLA_UNOP_PATTERN(Ceil) +XLA_UNOP_PATTERN(Copy) +XLA_UNOP_PATTERN(Cos) +XLA_UNOP_PATTERN(Exp) +XLA_UNOP_PATTERN(Fft) +XLA_UNOP_PATTERN(Floor) +XLA_UNOP_PATTERN(Imag) +XLA_UNOP_PATTERN(IsFinite) +XLA_UNOP_PATTERN(Log) +XLA_UNOP_PATTERN(Not) +XLA_UNOP_PATTERN(Negate) +XLA_UNOP_PATTERN(Outfeed) +XLA_UNOP_PATTERN(Real) +XLA_UNOP_PATTERN(Reduce) +XLA_UNOP_PATTERN(ReducePrecision) +XLA_UNOP_PATTERN(Reshape) +XLA_UNOP_PATTERN(Reverse) +XLA_UNOP_PATTERN(Send) +XLA_UNOP_PATTERN(Sign) +XLA_UNOP_PATTERN(Sin) +XLA_UNOP_PATTERN(Sort) +XLA_UNOP_PATTERN(Tanh) +XLA_UNOP_PATTERN(Transpose) +#undef XLA_UNOP_PATTERN + +// Helpers for binary instructions. +#define XLA_BINOP_PATTERN(NAME) \ + inline auto NAME()->decltype(Op().WithOpcode(HloOpcode::k##NAME)) { \ + return Op().WithOpcode(HloOpcode::k##NAME); \ + } \ + \ + template \ + inline auto NAME(Lhs&& lhs, Rhs&& rhs) \ + ->decltype(Op().WithOpcode(HloOpcode::k##NAME) \ + .WithOperand(0, std::forward(lhs)) \ + .WithOperand(1, std::forward(rhs))) { \ + return Op() \ + .WithOpcode(HloOpcode::k##NAME) \ + .WithOperand(0, std::forward(lhs)) \ + .WithOperand(1, std::forward(rhs)); \ + } \ + \ + template \ + inline auto NAME(HloInstructionType** matched_inst, Lhs&& lhs, Rhs&& rhs) \ + ->decltype(Op(matched_inst) \ + .WithOpcode(HloOpcode::k##NAME) \ + .WithOperand(0, std::forward(lhs)) \ + .WithOperand(1, std::forward(rhs))) { \ + return Op(matched_inst) \ + .WithOpcode(HloOpcode::k##NAME) \ + .WithOperand(0, std::forward(lhs)) \ + .WithOperand(1, std::forward(rhs)); \ + } +XLA_BINOP_PATTERN(Add) +XLA_BINOP_PATTERN(Atan2) +XLA_BINOP_PATTERN(Divide) +XLA_BINOP_PATTERN(Complex) +XLA_BINOP_PATTERN(Dot) +XLA_BINOP_PATTERN(Eq) +XLA_BINOP_PATTERN(Gather) +XLA_BINOP_PATTERN(Ge) +XLA_BINOP_PATTERN(Gt) +XLA_BINOP_PATTERN(Le) +XLA_BINOP_PATTERN(Lt) +XLA_BINOP_PATTERN(Maximum) +XLA_BINOP_PATTERN(Minimum) +XLA_BINOP_PATTERN(Multiply) +XLA_BINOP_PATTERN(Ne) +XLA_BINOP_PATTERN(Power) +XLA_BINOP_PATTERN(Remainder) +XLA_BINOP_PATTERN(Subtract) +XLA_BINOP_PATTERN(And) +XLA_BINOP_PATTERN(Or) +XLA_BINOP_PATTERN(ShiftLeft) +XLA_BINOP_PATTERN(ShiftRightArithmetic) +XLA_BINOP_PATTERN(ShiftRightLogical) +#undef XLA_BINOP_PATTERN + +// Helpers for ternary instructions. +#define XLA_TERNOP_PATTERN(NAME) \ + inline auto NAME()->decltype(Op().WithOpcode(HloOpcode::k##NAME)) { \ + return Op().WithOpcode(HloOpcode::k##NAME); \ + } \ + \ + template \ + inline auto NAME(Arg0&& arg0, Arg1&& arg1, Arg2&& arg2) \ + ->decltype(Op().WithOpcode(HloOpcode::k##NAME) \ + .WithOperand(0, std::forward(arg0)) \ + .WithOperand(1, std::forward(arg1)) \ + .WithOperand(2, std::forward(arg2))) { \ + return Op() \ + .WithOpcode(HloOpcode::k##NAME) \ + .WithOperand(0, std::forward(arg0)) \ + .WithOperand(1, std::forward(arg1)) \ + .WithOperand(2, std::forward(arg2)); \ + } \ + \ + template \ + inline auto NAME(HloInstructionType** matched_inst, Arg0&& arg0, \ + Arg1&& arg1, Arg2&& arg2) \ + ->decltype(Op(matched_inst) \ + .WithOpcode(HloOpcode::k##NAME) \ + .WithOperand(0, std::forward(arg0)) \ + .WithOperand(1, std::forward(arg1)) \ + .WithOperand(2, std::forward(arg2))) { \ + return Op(matched_inst) \ + .WithOpcode(HloOpcode::k##NAME) \ + .WithOperand(0, std::forward(arg0)) \ + .WithOperand(1, std::forward(arg1)) \ + .WithOperand(2, std::forward(arg2)); \ + } +XLA_TERNOP_PATTERN(Clamp); +XLA_TERNOP_PATTERN(Select); +#undef XLA_TERNOP_PATTERN + +// Helpers for matching non-constant instructions. +inline auto NonConstant() -> decltype(Op().IsNonConstant()) { + return Op().IsNonConstant(); +} + +template +inline auto NonConstant(HloInstructionType** matched_inst) + -> decltype(Op(matched_inst).IsNonConstant()) { + return Op(matched_inst).IsNonConstant(); +} + +} // namespace match + +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_SERVICE_PATTERN_MATCHER_H_ diff --git a/tensorflow/compiler/xla/service/pattern_matcher_test.cc b/tensorflow/compiler/xla/service/pattern_matcher_test.cc new file mode 100644 index 0000000000..5291b1437a --- /dev/null +++ b/tensorflow/compiler/xla/service/pattern_matcher_test.cc @@ -0,0 +1,144 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/pattern_matcher.h" +#include "tensorflow/compiler/xla/service/hlo_instruction.h" +#include "tensorflow/compiler/xla/service/hlo_opcode.h" +#include "tensorflow/compiler/xla/tools/parser/hlo_parser.h" +#include "tensorflow/core/platform/test.h" + +namespace xla { +namespace { + +TEST(PatternMatcherTest, AddOp) { + constexpr char kModuleStr[] = R"(HloModule two_plus_two_module + ENTRY %two_plus_two_computation () -> f32[] { + %two = f32[] constant(2) + ROOT %two_plus_two = f32[] add(f32[] %two, f32[] %two) + } + )"; + TF_ASSERT_OK_AND_ASSIGN(auto hlo_module, tools::Parse(kModuleStr)); + + const HloInstruction* matched_inst; + HloInstruction* matched_operand; + Shape* matched_shape; + Layout* matched_layout; + + ASSERT_TRUE(Match( + hlo_module->entry_computation()->root_instruction(), + match::Op(&matched_inst) + .WithName("two_plus_two") + .WithOpcode(HloOpcode::kAdd) + .WithShape( + match::Shape(&matched_shape) + .WithLayout(match::Layout(&matched_layout).WithDenseFormat())) + .WithOperand( + 0, + match::Op(&matched_operand).WithOpcode(HloOpcode::kConstant)))); + ASSERT_NE(matched_inst, nullptr); + EXPECT_EQ(matched_inst->name(), "two_plus_two"); + EXPECT_EQ(matched_inst->opcode(), HloOpcode::kAdd); + + EXPECT_TRUE(Match(hlo_module->entry_computation()->root_instruction(), + match::Add(match::Constant(), match::Constant()))); + + EXPECT_FALSE(Match(hlo_module->entry_computation()->root_instruction(), + match::Op().WithName("bad_name"))); + matched_inst = nullptr; + EXPECT_FALSE(Match(hlo_module->entry_computation()->root_instruction(), + match::Multiply(&matched_inst, match::Op(), match::Op()))); +} + +TEST(PatternMatcherTest, ScalarShape) { + auto scalar_shape = ShapeUtil::MakeShape(F32, {}); + Shape* matched_shape; + EXPECT_TRUE(Match(&scalar_shape, match::Shape(&matched_shape).IsScalar())); + EXPECT_EQ(matched_shape, &scalar_shape); + EXPECT_TRUE(Match(&scalar_shape, match::Shape().IsArray())); + EXPECT_FALSE(Match(&scalar_shape, match::Shape().IsTuple())); + EXPECT_TRUE(Match(&scalar_shape, match::Shape().WithElementType(F32))); + EXPECT_TRUE(Match(&scalar_shape, match::Shape().WithRank(0))); + EXPECT_FALSE(Match( + &scalar_shape, + match::Shape().WithSubshape({0}, match::Shape()).WithElementType(F32))); +} + +TEST(PatternMatcherTest, ArrayShape) { + auto array_shape = ShapeUtil::MakeShape(F32, {2, 3, 4}); + Shape* matched_shape; + EXPECT_TRUE(Match(&array_shape, match::Shape(&matched_shape).IsArray())); + EXPECT_EQ(matched_shape, &array_shape); + EXPECT_FALSE(Match(&array_shape, match::Shape().IsScalar())); + EXPECT_FALSE(Match(&array_shape, match::Shape().IsTuple())); + EXPECT_TRUE(Match(&array_shape, match::Shape().WithElementType(F32))); + EXPECT_TRUE(Match(&array_shape, match::Shape().WithRank(3))); + EXPECT_FALSE( + Match(&array_shape, match::Shape().WithSubshape({0}, match::Shape()))); + Layout* matched_layout; + EXPECT_FALSE(Match(&array_shape, + match::Shape().WithLayout( + match::Layout(&matched_layout).WithSparseFormat()))); +} + +TEST(PatternMatcherTest, TupleShape) { + auto tuple_shape = ShapeUtil::MakeTupleShape({ + ShapeUtil::MakeShape(F32, {1, 2, 3}), + ShapeUtil::MakeShape(S32, {4, 5}), + }); + EXPECT_TRUE(Match(&tuple_shape, match::Shape().IsTuple())); + EXPECT_FALSE(Match(&tuple_shape, match::Shape().IsArray())); + EXPECT_FALSE(Match(&tuple_shape, match::Shape().IsScalar())); + + Shape* subshape; + ASSERT_TRUE(Match( + &tuple_shape, + match::Shape().WithSubshape( + {0}, match::Shape(&subshape).WithElementType(F32).WithRank(3)))); + ASSERT_NE(subshape, nullptr); + EXPECT_TRUE( + ShapeUtil::Equal(*subshape, ShapeUtil::GetSubshape(tuple_shape, {0}))); + EXPECT_TRUE(Match(&tuple_shape, + match::Shape().WithSubshape( + {0}, match::Shape().EqualTo( + &ShapeUtil::GetSubshape(tuple_shape, {0}))))); + EXPECT_FALSE(Match(&tuple_shape, + match::Shape().WithSubshape( + {0}, match::Shape().EqualTo( + &ShapeUtil::GetSubshape(tuple_shape, {1}))))); + + ASSERT_TRUE(Match( + &tuple_shape, + match::Shape().WithSubshape( + {1}, match::Shape(&subshape).WithElementType(S32).WithRank(2)))); + ASSERT_NE(subshape, nullptr); + EXPECT_TRUE( + ShapeUtil::Equal(*subshape, ShapeUtil::GetSubshape(tuple_shape, {1}))); + EXPECT_TRUE(Match(&tuple_shape, + match::Shape().WithSubshape( + {1}, match::Shape().EqualTo( + &ShapeUtil::GetSubshape(tuple_shape, {1}))))); + EXPECT_FALSE(Match(&tuple_shape, + match::Shape().WithSubshape( + {1}, match::Shape().EqualTo( + &ShapeUtil::GetSubshape(tuple_shape, {0}))))); + + EXPECT_FALSE( + Match(&tuple_shape, match::Shape().WithSubshape({2}, match::Shape()))); + EXPECT_FALSE( + Match(&tuple_shape, match::Shape().WithSubshape({0, 0}, match::Shape()))); +} + +} // namespace +} // namespace xla diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc index 6825d24765..ac7e201bfd 100644 --- a/tensorflow/compiler/xla/shape_util.cc +++ b/tensorflow/compiler/xla/shape_util.cc @@ -824,6 +824,18 @@ StatusOr ParseShapeStringInternal(tensorflow::StringPiece* s) { return new_shape; } +/* static */ bool ShapeUtil::IndexIsValid(const Shape& shape, + ShapeIndexView index) { + const Shape* subshape = &shape; + for (auto i : index) { + if (!IsTuple(*subshape) || i >= subshape->tuple_shapes_size()) { + return false; + } + subshape = &subshape->tuple_shapes(i); + } + return true; +} + /* static */ const Shape& ShapeUtil::GetSubshape(const Shape& shape, ShapeIndexView index) { const Shape* return_shape = &shape; diff --git a/tensorflow/compiler/xla/shape_util.h b/tensorflow/compiler/xla/shape_util.h index 6d228eff46..63da9154cf 100644 --- a/tensorflow/compiler/xla/shape_util.h +++ b/tensorflow/compiler/xla/shape_util.h @@ -448,6 +448,9 @@ class ShapeUtil { static bool ShapeIs(const Shape& shape, PrimitiveType element_type, std::initializer_list dimensions); + // Returns true if the given shape has a subshape at the given index. + static bool IndexIsValid(const Shape& shape, ShapeIndexView index); + // GetSubshape and GetMutableSubshape return a particular nested Shape within // the given Shape argument. static const Shape& GetSubshape(const Shape& shape, ShapeIndexView index); -- GitLab From 026f052710475d1a5d08007e5ff7e105c653a965 Mon Sep 17 00:00:00 2001 From: Francois Chollet Date: Fri, 13 Apr 2018 15:33:07 -0700 Subject: [PATCH 357/791] Avoid mixing `Dimension` type and `int` when defining kernel shapes in conv layers. PiperOrigin-RevId: 192834255 --- .../keras/_impl/keras/layers/convolutional.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/keras/_impl/keras/layers/convolutional.py b/tensorflow/python/keras/_impl/keras/layers/convolutional.py index d202b6551d..12b965587f 100644 --- a/tensorflow/python/keras/_impl/keras/layers/convolutional.py +++ b/tensorflow/python/keras/_impl/keras/layers/convolutional.py @@ -148,7 +148,7 @@ class Conv(Layer): if input_shape[channel_axis].value is None: raise ValueError('The channel dimension of the inputs ' 'should be defined. Found `None`.') - input_dim = input_shape[channel_axis].value + input_dim = int(input_shape[channel_axis]) kernel_shape = self.kernel_size + (input_dim, self.filters) self.kernel = self.add_variable(name='kernel', @@ -705,6 +705,7 @@ class Conv2DTranspose(Conv2D): **kwargs) def build(self, input_shape): + input_shape = tensor_shape.TensorShape(input_shape) if len(input_shape) != 4: raise ValueError('Inputs should have rank 4. Received input shape: ' + str(input_shape)) @@ -712,10 +713,10 @@ class Conv2DTranspose(Conv2D): channel_axis = 1 else: channel_axis = -1 - if input_shape[channel_axis] is None: + if input_shape[channel_axis].value is None: raise ValueError('The channel dimension of the inputs ' 'should be defined. Found `None`.') - input_dim = input_shape[channel_axis] + input_dim = int(input_shape[channel_axis]) self.input_spec = InputSpec(ndim=4, axes={channel_axis: input_dim}) kernel_shape = self.kernel_size + (self.filters, input_dim) @@ -945,6 +946,7 @@ class Conv3DTranspose(Conv3D): **kwargs) def build(self, input_shape): + input_shape = tensor_shape.TensorShape(input_shape) if len(input_shape) != 5: raise ValueError('Inputs should have rank 5, received input shape:', str(input_shape)) @@ -952,10 +954,10 @@ class Conv3DTranspose(Conv3D): channel_axis = 1 else: channel_axis = -1 - if input_shape[channel_axis] is None: + if input_shape[channel_axis].value is None: raise ValueError('The channel dimension of the inputs ' 'should be defined, found None: ' + str(input_shape)) - input_dim = input_shape[channel_axis] + input_dim = int(input_shape[channel_axis]) kernel_shape = self.kernel_size + (self.filters, input_dim) self.input_spec = InputSpec(ndim=5, axes={channel_axis: input_dim}) @@ -1212,7 +1214,7 @@ class SeparableConv(Conv): if input_shape[channel_axis].value is None: raise ValueError('The channel dimension of the inputs ' 'should be defined. Found `None`.') - input_dim = input_shape[channel_axis].value + input_dim = int(input_shape[channel_axis]) self.input_spec = InputSpec(ndim=self.rank + 2, axes={channel_axis: input_dim}) depthwise_kernel_shape = self.kernel_size + (input_dim, -- GitLab From cfc59cb0e89077c5aa80f386602b0be6a357c7c1 Mon Sep 17 00:00:00 2001 From: Akshay Agrawal Date: Fri, 13 Apr 2018 15:47:37 -0700 Subject: [PATCH 358/791] Enable remote functions for TPU_SYSTEM. PiperOrigin-RevId: 192836098 --- .../core/common_runtime/process_function_library_runtime.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/common_runtime/process_function_library_runtime.cc b/tensorflow/core/common_runtime/process_function_library_runtime.cc index 92fdcb404e..d05f146f21 100644 --- a/tensorflow/core/common_runtime/process_function_library_runtime.cc +++ b/tensorflow/core/common_runtime/process_function_library_runtime.cc @@ -144,7 +144,10 @@ Status ProcessFunctionLibraryRuntime::GetDeviceContext( } Device* device = flr->device(); string device_type = device->parsed_name().type; - if (device_type == "CPU") return Status::OK(); + if (device_type == "CPU" || device_type == "TPU_SYSTEM") { + // "TPU_SYSTEM" indicates that `device` is a CPU. + return Status::OK(); + } if (device_type == "GPU") { auto* dev_info = flr->device()->tensorflow_gpu_device_info(); if (dev_info) { -- GitLab From aa65cee4bb9644ef4d3f8704161c70d61113cce3 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 15:53:05 -0700 Subject: [PATCH 359/791] Restore definitions of static members in MklCpuAllocator. These were removed in #17396 which made the static member variables of MklCpuAllocator into inline variables, which are a C++17 feature, and not properly restored in #18006 which reverted the inline declarations, leading to an ODR violation that is apparently ignored with some compilers. END_PUBLIC RELNOTES: n/a BEGIN_PUBLIC Automated g4 rollback of changelist 191305220 PiperOrigin-RevId: 192836808 --- tensorflow/core/common_runtime/mkl_cpu_allocator.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator.cc b/tensorflow/core/common_runtime/mkl_cpu_allocator.cc index 829c19204a..43a909466e 100644 --- a/tensorflow/core/common_runtime/mkl_cpu_allocator.cc +++ b/tensorflow/core/common_runtime/mkl_cpu_allocator.cc @@ -19,6 +19,9 @@ limitations under the License. namespace tensorflow { +constexpr const char* MklCPUAllocator::kMaxLimitStr; +constexpr const size_t MklCPUAllocator::kDefaultMaxLimit; + } // namespace tensorflow #endif // INTEL_MKL -- GitLab From 3bf8fe926b833aa5258d6a5ac58ed3aac2b4cda3 Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Fri, 13 Apr 2018 15:57:45 -0700 Subject: [PATCH 360/791] Upgrade SQLite PiperOrigin-RevId: 192837358 --- tensorflow/workspace.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 85bd1ea28b..aab0fb41fb 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -232,11 +232,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""): tf_http_archive( name = "org_sqlite", urls = [ - "https://mirror.bazel.build/www.sqlite.org/2017/sqlite-amalgamation-3200000.zip", - "http://www.sqlite.org/2017/sqlite-amalgamation-3200000.zip", + "https://mirror.bazel.build/www.sqlite.org/2018/sqlite-amalgamation-3230100.zip", + "https://www.sqlite.org/2018/sqlite-amalgamation-3230100.zip", ], - sha256 = "208780b3616f9de0aeb50822b7a8f5482f6515193859e91ed61637be6ad74fd4", - strip_prefix = "sqlite-amalgamation-3200000", + sha256 = "4239a1f69e5721d07d9a374eb84d594225229e54be4ee628da2995f4315d8dfc", + strip_prefix = "sqlite-amalgamation-3230100", build_file = clean_dep("//third_party:sqlite.BUILD"), ) -- GitLab From 0d3fda7691f21ff2cb84d391494697f37804bec6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Apr 2018 16:38:12 -0700 Subject: [PATCH 361/791] Improve layout optimizer tests -- Evaluate nodes before and after optimization, to confirm the graph's behavior is maintained after optimization. PiperOrigin-RevId: 192842623 --- tensorflow/core/grappler/optimizers/BUILD | 3 ++ .../optimizers/layout_optimizer_test.cc | 36 +++++++++++++++---- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD index a4545bb8f8..aa5102017c 100644 --- a/tensorflow/core/grappler/optimizers/BUILD +++ b/tensorflow/core/grappler/optimizers/BUILD @@ -479,10 +479,13 @@ tf_cuda_cc_test( "//tensorflow/core:test", "//tensorflow/core:test_main", "//tensorflow/core:testlib", + "//tensorflow/core/grappler:devices", "//tensorflow/core/grappler:grappler_item", "//tensorflow/core/grappler:utils", + "//tensorflow/core/grappler/clusters:single_machine", "//tensorflow/core/grappler/clusters:virtual_cluster", "//tensorflow/core/grappler/costs:virtual_placer", + "//tensorflow/core/grappler/utils:grappler_test", ], ) diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc b/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc index b913f2b004..e405c4c58c 100644 --- a/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc +++ b/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc @@ -17,11 +17,15 @@ limitations under the License. #include "tensorflow/cc/ops/standard_ops.h" #include "tensorflow/core/framework/node_def.pb.h" #include "tensorflow/core/framework/tensor_testutil.h" +#include "tensorflow/core/grappler/clusters/single_machine.h" #include "tensorflow/core/grappler/clusters/virtual_cluster.h" #include "tensorflow/core/grappler/costs/virtual_placer.h" +#include "tensorflow/core/grappler/devices.h" #include "tensorflow/core/grappler/grappler_item.h" #include "tensorflow/core/grappler/utils.h" +#include "tensorflow/core/grappler/utils/grappler_test.h" #include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/test.h" #include "tensorflow/core/protobuf/device_properties.pb.h" @@ -30,15 +34,25 @@ namespace tensorflow { namespace grappler { namespace { -class LayoutOptimizerTest : public ::testing::Test { +class LayoutOptimizerTest : public GrapplerTest { protected: void SetUp() override { - DeviceProperties device_properties; - device_properties.set_type("GPU"); - device_properties.mutable_environment()->insert({"architecture", "6"}); - virtual_cluster_.reset(new VirtualCluster({{"/GPU:1", device_properties}})); + gpu_available_ = GetNumAvailableGPUs() > 0; + + if (gpu_available_) { + virtual_cluster_.reset(new SingleMachine(/* timeout_s = */ 10, 1, 1)); + } else { + DeviceProperties device_properties; + device_properties.set_type("GPU"); + device_properties.mutable_environment()->insert({"architecture", "6"}); + virtual_cluster_.reset( + new VirtualCluster({{"/GPU:1", device_properties}})); + } + TF_CHECK_OK(virtual_cluster_->Provision()); } + void TearDown() override { TF_CHECK_OK(virtual_cluster_->Shutdown()); } + Output SimpleConv2D(tensorflow::Scope* s, int input_size, int filter_size, const string& padding) { return SimpleConv2D(s, input_size, filter_size, padding, ""); @@ -160,6 +174,7 @@ class LayoutOptimizerTest : public ::testing::Test { } std::unique_ptr virtual_cluster_; + bool gpu_available_; }; TEST_F(LayoutOptimizerTest, Conv2DBackpropInput) { @@ -183,6 +198,15 @@ TEST_F(LayoutOptimizerTest, Conv2DBackpropInput) { Tensor input_sizes_expected(DT_INT32, {4}); test::FillValues(&input_sizes_expected, {128, 3, 7, 7}); test::ExpectTensorEqual(input_sizes_expected, input_sizes); + + if (gpu_available_) { + std::vector fetch = {"Fetch"}; + auto tensors_expected = EvaluateNodes(item.graph, fetch); + auto tensors = EvaluateNodes(output, fetch); + EXPECT_EQ(1, tensors_expected.size()); + EXPECT_EQ(1, tensors.size()); + test::ExpectTensorNear(tensors_expected[0], tensors[0], 1e-6); + } } TEST_F(LayoutOptimizerTest, Conv2DBackpropInputNonConstInputSizes) { @@ -1150,7 +1174,7 @@ TEST_F(LayoutOptimizerTest, DevicePlacement) { NodeMap node_map(&output); auto vec_permute = node_map.GetNode("s-0-0-VecPermuteNCHWToNHWC-LayoutOptimizer"); - EXPECT_EQ(vec_permute->device(), "/device:CPU:0"); + EXPECT_TRUE(str_util::EndsWith(vec_permute->device(), "CPU:0")); } } // namespace } // namespace grappler -- GitLab From 3d66977d99c1d37cf318557ea613cd0dd6b001fd Mon Sep 17 00:00:00 2001 From: Yifei Feng Date: Fri, 13 Apr 2018 16:38:29 -0700 Subject: [PATCH 362/791] Automated g4 rollback of changelist 192784701 PiperOrigin-RevId: 192842670 --- .../ci_build/windows/bazel/bazel_test_lib.sh | 7 ------- .../windows/cpu/pip/build_tf_windows.sh | 17 +++++------------ 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh b/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh index b2e16902d6..d654b433e7 100644 --- a/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh +++ b/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh @@ -140,13 +140,6 @@ function run_configure_for_gpu_build { echo "" | ./configure } -function set_gcs_remote_cache_options { - echo "build --experimental_remote_spawn_cache" >> .bazelrc - echo "build --experimental_remote_platform_override='properties:{name:\"build\" value:\"windows-x64\"}'" >> .bazelrc - echo "build --remote_http_cache=https://storage.googleapis.com/$GCS_BUCKET_NAME" >> .bazelrc - echo "build --google_credentials=$GOOGLE_CLOUD_CREDENTIAL" >> .bazelrc -} - function create_python_test_dir() { rm -rf "$1" mkdir -p "$1" diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh index 4657ff196b..5e9ae497e1 100644 --- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh +++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh @@ -42,27 +42,20 @@ source "tensorflow/tools/ci_build/windows/bazel/common_env.sh" \ source "tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh" \ || { echo "Failed to source bazel_test_lib.sh" >&2; exit 1; } -# Recreate an empty bazelrc file under source root -rm -f .bazelrc -touch .bazelrc - skip_test=0 for ARG in "$@"; do if [[ "$ARG" == --skip_test ]]; then skip_test=1 - elif [[ "$ARG" == --enable_gcs_remote_cache ]]; then - set_gcs_remote_cache_options fi done -# --define=override_eigen_strong_inline=true speeds up the compiling of conv_grad_ops_3d.cc and conv_ops_3d.cc -# by 20 minutes. See https://github.com/tensorflow/tensorflow/issues/10521 -echo "build --define=override_eigen_strong_inline=true" >> .bazelrc - run_configure_for_cpu_build -bazel build -c opt tensorflow/tools/pip_package:build_pip_package || exit $? +# --define=override_eigen_strong_inline=true speeds up the compiling of conv_grad_ops_3d.cc and conv_ops_3d.cc +# by 20 minutes. See https://github.com/tensorflow/tensorflow/issues/10521 +BUILD_OPTS="--define=override_eigen_strong_inline=true" +bazel build -c opt $BUILD_OPTS tensorflow/tools/pip_package:build_pip_package || exit $? if [[ "$skip_test" == 1 ]]; then exit 0 @@ -80,7 +73,7 @@ reinstall_tensorflow_pip ${PIP_NAME} # Define no_tensorflow_py_deps=true so that every py_test has no deps anymore, # which will result testing system installed tensorflow -bazel test -c opt -k --test_output=errors \ +bazel test -c opt $BUILD_OPTS -k --test_output=errors \ --define=no_tensorflow_py_deps=true --test_lang_filters=py \ --test_tag_filters=-no_pip,-no_windows,-no_oss \ --build_tag_filters=-no_pip,-no_windows,-no_oss --build_tests_only \ -- GitLab From 6048b07adb364fcef086fb30ecdfb8a2881ba6ac Mon Sep 17 00:00:00 2001 From: Yu-Cheng Ling Date: Fri, 13 Apr 2018 17:13:45 -0700 Subject: [PATCH 363/791] TFLite: Copy output data from BufferHandle to CPU memory by default. PiperOrigin-RevId: 192846824 --- tensorflow/contrib/lite/interpreter.cc | 6 ++++++ tensorflow/contrib/lite/interpreter.h | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/tensorflow/contrib/lite/interpreter.cc b/tensorflow/contrib/lite/interpreter.cc index f258654608..31b874a6a6 100644 --- a/tensorflow/contrib/lite/interpreter.cc +++ b/tensorflow/contrib/lite/interpreter.cc @@ -570,6 +570,12 @@ TfLiteStatus Interpreter::Invoke() { } } + if (!allow_buffer_handle_output_) { + for (int tensor_index : outputs_) { + EnsureTensorDataIsReadable(tensor_index); + } + } + return status; } diff --git a/tensorflow/contrib/lite/interpreter.h b/tensorflow/contrib/lite/interpreter.h index df67cce9de..3c776aacb6 100644 --- a/tensorflow/contrib/lite/interpreter.h +++ b/tensorflow/contrib/lite/interpreter.h @@ -282,6 +282,7 @@ class Interpreter { // Ensure the data in `tensor.data` is readable. In case delegate is used, // it might require to copy the data from delegate buffer to raw memory. + // WARNING: This is an experimental API and subject to change. TfLiteStatus EnsureTensorDataIsReadable(int tensor_index) { TF_LITE_ENSURE(&context_, tensor_index < tensors_size()); TfLiteTensor* tensor = &tensors_[tensor_index]; @@ -328,6 +329,18 @@ class Interpreter { // pointers to existing tensors. static constexpr int kTensorsCapacityHeadroom = 16; + // Set if buffer handle output is allowed. + // + // When using hardware delegation, Interpreter will make the data of output + // tensors available in `tensor->data` by default. If the application can + // consume the buffer handle directly (e.g. reading output from OpenGL + // texture), it can set this flag to false, so Interpreter won't copy the data + // from buffer handle to CPU memory. + // WARNING: This is an experimental API and subject to change. + void SetAllowBufferHandleOutput(bool allow_buffer_handle_output) { + allow_buffer_handle_output_ = allow_buffer_handle_output; + } + private: // Give 'op_reg' a chance to initialize itself using the contents of // 'buffer'. @@ -518,6 +531,9 @@ class Interpreter { std::unique_ptr nnapi_delegate_; std::unique_ptr memory_planner_; + + // WARNING: This is an experimental interface that is subject to change. + bool allow_buffer_handle_output_ = false; }; } // namespace tflite -- GitLab From 360c5a37957311657d45c351248aaa8e8fcac3be Mon Sep 17 00:00:00 2001 From: James Qin Date: Fri, 13 Apr 2018 17:26:46 -0700 Subject: [PATCH 364/791] Revamp Cudnn RNN kernels for incoming autotune changes. * Create DoForward() and DoBackward() to be used by fwd/bak kernels and later autotune. * Simplify CudnnRnnForward Comupute() function. Offload the majority of its logic to other member functions. PiperOrigin-RevId: 192848100 --- tensorflow/core/kernels/cudnn_rnn_ops.cc | 689 ++++++++++++++--------- 1 file changed, 410 insertions(+), 279 deletions(-) diff --git a/tensorflow/core/kernels/cudnn_rnn_ops.cc b/tensorflow/core/kernels/cudnn_rnn_ops.cc index e4036ddaa9..a21f13a4dd 100644 --- a/tensorflow/core/kernels/cudnn_rnn_ops.cc +++ b/tensorflow/core/kernels/cudnn_rnn_ops.cc @@ -78,6 +78,7 @@ using CPUDevice = Eigen::ThreadPoolDevice; #if GOOGLE_CUDA using GPUDevice = Eigen::GpuDevice; +using ::perftools::gputools::StreamExecutor; template class CudnnRNNParamsSizeOp; @@ -101,15 +102,21 @@ enum class TFRNNInputMode { }; namespace { -using perftools::gputools::DeviceMemory; -using perftools::gputools::DeviceMemoryBase; -using perftools::gputools::ScratchAllocator; -using perftools::gputools::dnn::AlgorithmConfig; -using perftools::gputools::dnn::RnnDirectionMode; -using perftools::gputools::dnn::RnnInputMode; -using perftools::gputools::dnn::RnnMode; -using perftools::gputools::dnn::ToDataType; -using perftools::gputools::port::StatusOr; +using ::perftools::gputools::DeviceMemory; +using ::perftools::gputools::DeviceMemoryBase; +using ::perftools::gputools::ScratchAllocator; +using ::perftools::gputools::Stream; +using ::perftools::gputools::dnn::AlgorithmConfig; +using ::perftools::gputools::dnn::AlgorithmDesc; +using ::perftools::gputools::dnn::ProfileResult; +using ::perftools::gputools::dnn::RnnDescriptor; +using ::perftools::gputools::dnn::RnnDirectionMode; +using ::perftools::gputools::dnn::RnnInputMode; +using ::perftools::gputools::dnn::RnnMode; +using ::perftools::gputools::dnn::RnnSequenceTensorDescriptor; +using ::perftools::gputools::dnn::RnnStateTensorDescriptor; +using ::perftools::gputools::dnn::ToDataType; +using ::perftools::gputools::port::StatusOr; Status ParseRNNMode(const string& str, RnnMode* rnn_mode) { if (str == "rnn_relu") { @@ -252,12 +259,12 @@ class CudnnRnnAllocatorInTemp : public ScratchAllocator { explicit CudnnRnnAllocatorInTemp(OpKernelContext* context) : context_(context) {} - int64 GetMemoryLimitInBytes(perftools::gputools::Stream* stream) override { + int64 GetMemoryLimitInBytes(Stream* stream) override { return std::numeric_limits::max(); } - StatusOr> AllocateBytes( - perftools::gputools::Stream* stream, int64 byte_size) override { + StatusOr> AllocateBytes(Stream* stream, + int64 byte_size) override { Tensor temporary_memory; const DataType tf_data_type = ToTFDataType::value; int64 allocate_count = @@ -298,11 +305,11 @@ class CudnnRnnAllocatorInOutput : public ScratchAllocator { ~CudnnRnnAllocatorInOutput() override {} CudnnRnnAllocatorInOutput(OpKernelContext* context, int output_index) : context_(context), output_index_(output_index) {} - int64 GetMemoryLimitInBytes(perftools::gputools::Stream* stream) override { + int64 GetMemoryLimitInBytes(Stream* stream) override { return std::numeric_limits::max(); } - StatusOr> AllocateBytes( - perftools::gputools::Stream* stream, int64 byte_size) override { + StatusOr> AllocateBytes(Stream* stream, + int64 byte_size) override { CHECK(total_byte_size_ == 0) << "Reserve space allocator can only be called once"; int64 allocate_count = @@ -338,12 +345,12 @@ class CudnnRNNPersistentSpaceAllocator : public ScratchAllocator { ~CudnnRNNPersistentSpaceAllocator() override {} - int64 GetMemoryLimitInBytes(perftools::gputools::Stream* stream) override { + int64 GetMemoryLimitInBytes(Stream* stream) override { return std::numeric_limits::max(); } - StatusOr> AllocateBytes( - perftools::gputools::Stream* stream, int64 byte_size) override { + StatusOr> AllocateBytes(Stream* stream, + int64 byte_size) override { if (total_byte_size_ != 0) { return Status(error::FAILED_PRECONDITION, "Persistent space allocator can only be called once"); @@ -374,6 +381,13 @@ struct CudnnModelTypes { // input-h. return rnn_mode == RnnMode::kRnnLstm; } + + string DebugString() const { + return strings::Printf( + "[rnn_mode, rnn_input_mode, rnn_direction_mode]: %d, %d, %d ", + static_cast(rnn_mode), static_cast(rnn_input_mode), + static_cast(rnn_direction_mode)); + } }; // A helper class that collects the shapes to describe a RNN model. @@ -381,9 +395,9 @@ struct CudnnRnnModelShapes { int num_layers; int input_size; int num_units; + int dir_count; int seq_length; int batch_size; - int dir_count; TensorShape input_shape; TensorShape output_shape; TensorShape hidden_state_shape; @@ -392,10 +406,11 @@ struct CudnnRnnModelShapes { return num_layers == rhs.num_layers && input_size == rhs.input_size && num_units == rhs.num_units && dir_count == rhs.dir_count; } - string RnnDescDebugString() { + string DebugString() const { return strings::Printf( - "[num_layers, input_size, num_units, dir_count]: [%d, %d, %d, %d]", - num_layers, input_size, num_units, dir_count); + "[num_layers, input_size, num_units, dir_count, seq_length, " + "batch_size]: [%d, %d, %d, %d, %d, %d] ", + num_layers, input_size, num_units, dir_count, seq_length, batch_size); } }; @@ -420,8 +435,15 @@ struct CudnnRnnModelShapesComparator { } }; -// Extract and checks the forward input tensors, parameters, and shapes from -// the OpKernelContext. +// Pointers to RNN scratch space for a specific set of shape parameters (used as +// a hash table value in CudnnRNNForwardOp and CudnnRNNBackwardOp). +struct RnnScratchSpace { + std::unique_ptr rnn_desc; + std::unique_ptr dropout_state_allocator; +}; + +// Extract and checks the forward input tensors, parameters, and shapes from the +// OpKernelContext. Status ExtractForwardInput(OpKernelContext* context, const CudnnModelTypes& model_types, const Tensor** input, const Tensor** input_h, @@ -474,13 +496,171 @@ Status ExtractForwardInput(OpKernelContext* context, return Status::OK(); } -using perftools::gputools::dnn::RnnDescriptor; +template +Status CreateForwardAndBackwardIODescriptors( + OpKernelContext* context, const CudnnRnnModelShapes& model_shapes, + std::unique_ptr* input_desc, + std::unique_ptr* state_desc, + std::unique_ptr* output_desc) { + StreamExecutor* executor = context->op_device_context()->stream()->parent(); + ::perftools::gputools::dnn::DataType data_type = ToDataType::value; + + const TensorShape& input_shape = model_shapes.input_shape; + const TensorShape& hidden_state_shape = model_shapes.hidden_state_shape; + const TensorShape& output_shape = model_shapes.output_shape; + + DCHECK_EQ(input_shape.dims(), 3); + auto input_desc_s = executor->createRnnSequenceTensorDescriptor( + input_shape.dim_size(0), input_shape.dim_size(1), input_shape.dim_size(2), + data_type); + TF_RETURN_IF_ERROR(input_desc_s.status()); + *input_desc = input_desc_s.ConsumeValueOrDie(); + + DCHECK_EQ(hidden_state_shape.dims(), 3); + auto hidden_state_desc_s = executor->createRnnStateTensorDescriptor( + hidden_state_shape.dim_size(0), hidden_state_shape.dim_size(1), + hidden_state_shape.dim_size(2), data_type); + TF_RETURN_IF_ERROR(hidden_state_desc_s.status()); + *state_desc = hidden_state_desc_s.ConsumeValueOrDie(); + + DCHECK_EQ(output_shape.dims(), 3); + auto output_desc_s = executor->createRnnSequenceTensorDescriptor( + output_shape.dim_size(0), output_shape.dim_size(1), + output_shape.dim_size(2), data_type); + TF_RETURN_IF_ERROR(output_desc_s.status()); + *output_desc = output_desc_s.ConsumeValueOrDie(); + return Status::OK(); +} + +template +Status DoForward(OpKernelContext* context, const RnnDescriptor& rnn_desc, + const CudnnModelTypes& model_types, + const CudnnRnnModelShapes& model_shapes, + /* forward inputs */ + const Tensor* input, const Tensor* input_h, + const Tensor* input_c, const Tensor* params, + const bool is_training, + /* forward outputs, outputs of the function */ + Tensor* output, Tensor* output_h, Tensor* output_c, + ScratchAllocator* reserve_space_allocator, + ScratchAllocator* workspace_allocator, + ProfileResult* output_profile_result) { + std::unique_ptr input_desc; + std::unique_ptr state_desc; + std::unique_ptr output_desc; + + TF_RETURN_IF_ERROR(CreateForwardAndBackwardIODescriptors( + context, model_shapes, &input_desc, &state_desc, &output_desc)); + + auto input_data = AsDeviceMemory(input); + auto input_h_data = AsDeviceMemory(input_h); + DeviceMemory input_c_data; + if (model_types.HasInputC()) { + input_c_data = AsDeviceMemory(input_c); + } + auto params_data = AsDeviceMemory(params); + auto output_data = AsDeviceMemory(output); + auto output_h_data = AsDeviceMemory(output_h); + DeviceMemory output_c_data; + if (model_types.HasInputC()) { + output_c_data = AsDeviceMemory(output_c); + } + + Stream* stream = context->op_device_context()->stream(); + bool launch_success = + stream + ->ThenRnnForward(rnn_desc, *input_desc, input_data, *state_desc, + input_h_data, *state_desc, input_c_data, params_data, + *output_desc, &output_data, *state_desc, + &output_h_data, *state_desc, &output_c_data, + is_training, reserve_space_allocator, + workspace_allocator, output_profile_result) + .ok(); + return launch_success + ? Status::OK() + : errors::Internal( + "Failed to call ThenRnnForward with model config: ", + model_types.DebugString(), ", ", model_shapes.DebugString()); +} + +template +Status DoBackward( + OpKernelContext* context, const RnnDescriptor& rnn_desc, + const CudnnModelTypes& model_types, const CudnnRnnModelShapes& model_shapes, + /* forward inputs */ + const Tensor* input, const Tensor* input_h, const Tensor* input_c, + const Tensor* params, + /* forward outptus */ + const Tensor* output, const Tensor* output_h, const Tensor* output_c, + /* backprop inputs */ + const Tensor* output_backprop, const Tensor* output_h_backprop, + const Tensor* output_c_backprop, const Tensor* reserve_space, + /* backprop outputs, output of the function */ + Tensor* input_backprop, Tensor* input_h_backprop, Tensor* input_c_backprop, + Tensor* params_backprop, ScratchAllocator* workspace_allocator, + ProfileResult* output_profile_result) { + std::unique_ptr input_desc; + std::unique_ptr state_desc; + std::unique_ptr output_desc; + + TF_RETURN_IF_ERROR(CreateForwardAndBackwardIODescriptors( + context, model_shapes, &input_desc, &state_desc, &output_desc)); + + auto input_data = AsDeviceMemory(input); + auto input_h_data = AsDeviceMemory(input_h); + DeviceMemory input_c_data; + if (model_types.HasInputC()) { + input_c_data = AsDeviceMemory(input_c); + } + auto params_data = AsDeviceMemory(params); + auto output_data = AsDeviceMemory(output); + auto output_h_data = AsDeviceMemory(output_h); + DeviceMemory output_c_data; + if (model_types.HasInputC()) { + output_c_data = AsDeviceMemory(output_c); + } + auto output_backprop_data = AsDeviceMemory(output_backprop); + auto output_h_backprop_data = AsDeviceMemory(output_h_backprop); + DeviceMemory output_c_backprop_data; + if (model_types.HasInputC()) { + output_c_backprop_data = AsDeviceMemory(output_c_backprop); + } + auto input_backprop_data = AsDeviceMemory(input_backprop); + auto input_h_backprop_data = AsDeviceMemory(input_h_backprop); + DeviceMemory input_c_backprop_data; + if (model_types.HasInputC()) { + input_c_backprop_data = AsDeviceMemory(input_c_backprop); + } + auto params_backprop_data = AsDeviceMemory(params_backprop); + auto reserve_space_uint8 = + CastDeviceMemory(const_cast(reserve_space)); + + // Creates a memory callback for the workspace. The memory lives to the end + // of this kernel calls. + Stream* stream = context->op_device_context()->stream(); + bool launch_success = + stream + ->ThenRnnBackward(rnn_desc, *input_desc, input_data, *state_desc, + input_h_data, *state_desc, input_c_data, + params_data, *output_desc, output_data, *state_desc, + output_h_data, *state_desc, output_c_data, + output_backprop_data, output_h_backprop_data, + output_c_backprop_data, &input_backprop_data, + &input_h_backprop_data, &input_c_backprop_data, + ¶ms_backprop_data, &reserve_space_uint8, + workspace_allocator, output_profile_result) + .ok(); + return launch_success + ? Status::OK() + : errors::Internal( + "Failed to call ThenRnnBackward with model config: ", + model_types.DebugString(), ", ", model_shapes.DebugString()); +} template void RestoreParams(const OpInputList params_input, const std::vector& params, - DeviceMemoryBase* data_dst, - perftools::gputools::Stream* stream) { + DeviceMemoryBase* data_dst, Stream* stream) { int num_params = params.size(); CHECK(params_input.size() == num_params) << "Number of params mismatch. Expected " << params_input.size() @@ -570,7 +750,7 @@ class CudnnRNNKernelCommon : public OpKernel { TF_RETURN_IF_ERROR( ToRNNInputMode(rnn_input_mode(), num_units, input_size, &input_mode)); - auto* stream = context->op_device_context()->stream(); + Stream* stream = context->op_device_context()->stream(); // ExtracCudnnRNNParamsInfo is only called by op_kernels that do not require // random number generator, therefore set state_allocator to nullptr. const AlgorithmConfig algo_config; @@ -585,6 +765,51 @@ class CudnnRNNKernelCommon : public OpKernel { return Status::OK(); } + template + Status CreateRnnDescriptor(OpKernelContext* context, + const CudnnRnnModelShapes& model_shapes, + const RnnInputMode& input_mode, + const AlgorithmConfig& algo_config, + ScratchAllocator* dropout_state_allocator, + std::unique_ptr* rnn_desc) { + StreamExecutor* executor = context->op_device_context()->stream()->parent(); + ::perftools::gputools::dnn::DataType data_type = ToDataType::value; + auto rnn_desc_s = executor->createRnnDescriptor( + model_shapes.num_layers, model_shapes.num_units, + model_shapes.input_size, input_mode, rnn_direction_mode(), rnn_mode(), + data_type, algo_config, dropout(), seed(), dropout_state_allocator); + TF_RETURN_IF_ERROR(rnn_desc_s.status()); + + *rnn_desc = rnn_desc_s.ConsumeValueOrDie(); + return Status::OK(); + } + + using RnnStateCache = + gtl::FlatMap; + // Returns a raw rnn descriptor pointer. The cache owns the rnn descriptor and + // should outlive the returned pointer. + template + Status GetCachedRnnDescriptor(OpKernelContext* context, + const CudnnRnnModelShapes& model_shapes, + const RnnInputMode& input_mode, + const AlgorithmConfig& algo_config, + RnnStateCache* cache, + RnnDescriptor** rnn_desc) { + RnnScratchSpace& rnn_state = (*cache)[model_shapes]; + if (rnn_state.rnn_desc == nullptr || ResetRndGenState()) { + CudnnRNNPersistentSpaceAllocator* dropout_state_allocator = + new CudnnRNNPersistentSpaceAllocator(context); + rnn_state.dropout_state_allocator.reset(dropout_state_allocator); + Status status = + CreateRnnDescriptor(context, model_shapes, input_mode, algo_config, + dropout_state_allocator, &rnn_state.rnn_desc); + TF_RETURN_IF_ERROR(status); + } + *rnn_desc = rnn_state.rnn_desc.get(); + return Status::OK(); + } + private: int seed_; int seed2_; @@ -648,7 +873,7 @@ class CudnnRNNParamsToCanonical : public CudnnRNNKernelCommon { void Compute(OpKernelContext* context) override { const Tensor& input = context->input(3); auto input_ptr = StreamExecutorUtil::AsDeviceMemory(input); - auto* stream = context->op_device_context()->stream(); + Stream* stream = context->op_device_context()->stream(); std::unique_ptr rnn_desc; OP_REQUIRES_OK(context, ExtractCudnnRNNParamsInfo(context, &rnn_desc)); @@ -789,7 +1014,7 @@ class CudnnRNNCanonicalToParams : public CudnnRNNKernelCommon { OP_REQUIRES_OK(context, context->allocate_output(0, {params_size}, &output)); auto output_ptr = StreamExecutorUtil::AsDeviceMemory(*output); - auto* stream = context->op_device_context()->stream(); + Stream* stream = context->op_device_context()->stream(); OpInputList weights; OP_REQUIRES_OK(context, context->input_list("weights", &weights)); @@ -816,13 +1041,6 @@ TF_CALL_float(REGISTER_GPU); TF_CALL_double(REGISTER_GPU); #undef REGISTER_GPU -// Pointers to RNN scratch space for a specific set of shape parameters (used as -// a hash table value in CudnnRNNForwardOp and CudnnRNNBackwardOp). -struct RnnScratchSpace { - std::unique_ptr rnn_desc; - std::unique_ptr dropout_state_allocator; -}; - // Run the forward operation of the RNN model. template class CudnnRNNForwardOp : public CudnnRNNKernelCommon { @@ -842,115 +1060,71 @@ class CudnnRNNForwardOp : public CudnnRNNKernelCommon { OP_REQUIRES_OK(context, ExtractForwardInput(context, model_types(), &input, &input_h, &input_c, ¶ms, &model_shapes)); - const auto& input_shape = model_shapes.input_shape; - const auto& hidden_state_shape = model_shapes.hidden_state_shape; - const auto& output_shape = model_shapes.output_shape; - - Tensor* output = nullptr; - OP_REQUIRES_OK(context, context->allocate_output(0, output_shape, &output)); - Tensor* output_h = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output(1, hidden_state_shape, &output_h)); - Tensor* output_c = nullptr; - if (HasInputC()) { - // Only LSTM uses input_c and output_c. So for all other models, we only - // need to create dummy outputs. - OP_REQUIRES_OK( - context, context->allocate_output(2, hidden_state_shape, &output_c)); - } else { - OP_REQUIRES_OK(context, context->allocate_output(2, {}, &output_c)); - } - - auto* stream = context->op_device_context()->stream(); - auto* executor = stream->parent(); RnnInputMode input_mode; OP_REQUIRES_OK(context, ToRNNInputMode(rnn_input_mode(), model_shapes.num_units, model_shapes.input_size, &input_mode)); - auto data_type = ToDataType::value; - - auto input_desc_s = executor->createRnnSequenceTensorDescriptor( - input_shape.dim_size(0), input_shape.dim_size(1), - input_shape.dim_size(2), data_type); - OP_REQUIRES_OK(context, FromExecutorStatus(input_desc_s)); - auto input_desc = input_desc_s.ConsumeValueOrDie(); - - auto hidden_state_desc_s = executor->createRnnStateTensorDescriptor( - hidden_state_shape.dim_size(0), hidden_state_shape.dim_size(1), - hidden_state_shape.dim_size(2), data_type); - OP_REQUIRES_OK(context, FromExecutorStatus(hidden_state_desc_s)); - auto hidden_state_desc = hidden_state_desc_s.ConsumeValueOrDie(); - - auto output_desc_s = executor->createRnnSequenceTensorDescriptor( - output_shape.dim_size(0), output_shape.dim_size(1), - output_shape.dim_size(2), data_type); - OP_REQUIRES_OK(context, FromExecutorStatus(output_desc_s)); - auto output_desc = output_desc_s.ConsumeValueOrDie(); - - auto input_data = AsDeviceMemory(input); - auto input_h_data = AsDeviceMemory(input_h); - DeviceMemory input_c_data; - if (HasInputC()) { - input_c_data = AsDeviceMemory(input_c); - } - auto params_data = AsDeviceMemory(params); - auto output_data = AsDeviceMemory(output); - auto output_h_data = AsDeviceMemory(output_h); - DeviceMemory output_c_data; - if (HasInputC()) { - output_c_data = AsDeviceMemory(output_c); - } + Tensor* output = nullptr; + Tensor* output_h = nullptr; + Tensor* output_c = nullptr; + OP_REQUIRES_OK(context, AllocateOutputs(context, model_shapes, &output, + &output_h, &output_c)); + + AlgorithmConfig algo_config; // Creates a memory callback for the reserve_space. The memory lives in the // output of this kernel. And it will be fed into the backward pass when // needed. CudnnRnnAllocatorInOutput reserve_space_allocator(context, 3); - if (!is_training_) { - Tensor* dummy_reserve_space = nullptr; - OP_REQUIRES_OK(context, - context->allocate_output(3, {}, &dummy_reserve_space)); - } // Creates a memory callback for the workspace. The memory lives to the end // of this kernel calls. CudnnRnnAllocatorInTemp workspace_allocator(context); - bool launch_status = false; + Status launch_status; { mutex_lock l(mu_); - RnnScratchSpace& rnn_state = rnn_state_cache_[model_shapes]; - if (rnn_state.rnn_desc == nullptr || ResetRndGenState()) { - CudnnRNNPersistentSpaceAllocator* dropout_state_allocator = - new CudnnRNNPersistentSpaceAllocator(context); - rnn_state.dropout_state_allocator.reset(dropout_state_allocator); - const AlgorithmConfig algo_config; - auto rnn_desc_s = executor->createRnnDescriptor( - model_shapes.num_layers, model_shapes.num_units, - model_shapes.input_size, input_mode, rnn_direction_mode(), - rnn_mode(), data_type, algo_config, dropout(), seed(), - dropout_state_allocator); - OP_REQUIRES_OK(context, FromExecutorStatus(rnn_desc_s)); - rnn_state.rnn_desc = std::move(rnn_desc_s.ConsumeValueOrDie()); - } - launch_status = - stream - ->ThenRnnForward( - *rnn_state.rnn_desc, *input_desc, input_data, - *hidden_state_desc, input_h_data, *hidden_state_desc, - input_c_data, params_data, *output_desc, &output_data, - *hidden_state_desc, &output_h_data, *hidden_state_desc, - &output_c_data, is_training_, &reserve_space_allocator, - &workspace_allocator, /*output_result_profile=*/nullptr) - .ok(); + RnnDescriptor* rnn_desc_ptr = nullptr; + OP_REQUIRES_OK( + context, GetCachedRnnDescriptor(context, model_shapes, input_mode, + algo_config, &rnn_state_cache_, + &rnn_desc_ptr)); + launch_status = DoForward( + context, *rnn_desc_ptr, model_types(), model_shapes, input, input_h, + input_c, params, is_training_, output, output_h, output_c, + &reserve_space_allocator, &workspace_allocator, + /*output_profile_result=*/nullptr); } - OP_REQUIRES(context, launch_status, - errors::Internal("Failed to call ThenRnnForward")); + OP_REQUIRES_OK(context, launch_status); } private: + Status AllocateOutputs(OpKernelContext* context, + const CudnnRnnModelShapes& model_shapes, + Tensor** output, Tensor** output_h, + Tensor** output_c) { + const TensorShape& hidden_state_shape = model_shapes.hidden_state_shape; + const TensorShape& output_shape = model_shapes.output_shape; + + TF_RETURN_IF_ERROR(context->allocate_output(0, output_shape, output)); + TF_RETURN_IF_ERROR( + context->allocate_output(1, hidden_state_shape, output_h)); + if (HasInputC()) { + TF_RETURN_IF_ERROR( + context->allocate_output(2, hidden_state_shape, output_c)); + } else { + // Only LSTM uses input_c and output_c. So for all other models, we only + // need to create dummy outputs. + TF_RETURN_IF_ERROR(context->allocate_output(2, {}, output_c)); + } + if (!is_training_) { + Tensor* dummy_reserve_space = nullptr; + TF_RETURN_IF_ERROR(context->allocate_output(3, {}, &dummy_reserve_space)); + } + return Status::OK(); + } + mutex mu_; bool is_training_; - std::unordered_map - rnn_state_cache_ GUARDED_BY(mu_); + RnnStateCache rnn_state_cache_ GUARDED_BY(mu_); }; #define REGISTER_GPU(T) \ @@ -981,184 +1155,141 @@ class CudnnRNNBackwardOp : public CudnnRNNKernelCommon { OP_REQUIRES_OK(context, ExtractForwardInput(context, model_types(), &input, &input_h, &input_c, ¶ms, &model_shapes)); + RnnInputMode input_mode; + OP_REQUIRES_OK(context, + ToRNNInputMode(rnn_input_mode(), model_shapes.num_units, + model_shapes.input_size, &input_mode)); - const auto& input_shape = model_shapes.input_shape; - const auto& hidden_state_shape = model_shapes.hidden_state_shape; - const auto& output_shape = model_shapes.output_shape; - - auto data_type = ToDataType::value; const Tensor* output = nullptr; - OP_REQUIRES_OK(context, context->input("output", &output)); - OP_REQUIRES(context, output_shape == output->shape(), - errors::InvalidArgument( - "input_h and input_c must have the same shape: ", - input_h->shape().DebugString(), " ", - input_c->shape().DebugString())); const Tensor* output_h = nullptr; - OP_REQUIRES_OK(context, context->input("output_h", &output_h)); - OP_REQUIRES(context, output_h->shape() == hidden_state_shape, - errors::InvalidArgument( - "Invalid output_h shape: ", output_h->shape().DebugString(), - " ", hidden_state_shape.DebugString())); const Tensor* output_c = nullptr; - if (HasInputC()) { - // Only LSTM uses input_c and output_c. So for all other models, we only - // need to create dummy outputs. - OP_REQUIRES_OK(context, context->input("output_c", &output_c)); - OP_REQUIRES(context, output_c->shape() == hidden_state_shape, - errors::InvalidArgument("Invalid output_c shape: ", - output_c->shape().DebugString(), " ", - hidden_state_shape.DebugString())); - } - const Tensor* output_backprop = nullptr; - OP_REQUIRES_OK(context, - context->input("output_backprop", &output_backprop)); - OP_REQUIRES(context, output_backprop->shape() == output_shape, - errors::InvalidArgument("Invalid output_backprop shapes: ", - output_backprop->shape().DebugString(), - " ", output_shape.DebugString())); - const Tensor* output_h_backprop = nullptr; - OP_REQUIRES_OK(context, - context->input("output_h_backprop", &output_h_backprop)); - OP_REQUIRES( - context, output_h_backprop->shape() == hidden_state_shape, - errors::InvalidArgument("Invalid output_h_backprop shapes: ", - output_h_backprop->shape().DebugString(), " ", - hidden_state_shape.DebugString())); const Tensor* output_c_backprop = nullptr; - if (HasInputC()) { - OP_REQUIRES_OK(context, - context->input("output_c_backprop", &output_c_backprop)); - OP_REQUIRES( - context, output_c_backprop->shape() == hidden_state_shape, - errors::InvalidArgument("Invalid output_c_backprop shapes: ", - output_c_backprop->shape().DebugString(), " ", - hidden_state_shape.DebugString())); - } - const Tensor* reserve_space_const = nullptr; - // This is the same "reserve_space" created by the forward op. - // It can also be modified by this backward operation. + const Tensor* reserve_space = nullptr; OP_REQUIRES_OK(context, - context->input("reserve_space", &reserve_space_const)); - // Cudnn needs the reserve space to be writeable. This is fine because they - // are opaque. - Tensor* reserve_space = const_cast(reserve_space_const); + ExtractBackwardInputs(context, model_shapes, model_types(), + &output, &output_h, &output_c, + &output_backprop, &output_h_backprop, + &output_c_backprop, &reserve_space)); Tensor* input_backprop = nullptr; - OP_REQUIRES_OK( - context, context->allocate_output(0, input->shape(), &input_backprop)); Tensor* input_h_backprop = nullptr; - OP_REQUIRES_OK(context, context->allocate_output(1, input_h->shape(), - &input_h_backprop)); Tensor* input_c_backprop = nullptr; - if (HasInputC()) { - OP_REQUIRES_OK(context, context->allocate_output(2, input_c->shape(), - &input_c_backprop)); - } else { - OP_REQUIRES_OK(context, - context->allocate_output(2, {}, &input_c_backprop)); - } Tensor* params_backprop = nullptr; - OP_REQUIRES_OK(context, context->allocate_output(3, params->shape(), - ¶ms_backprop)); - - auto* stream = context->op_device_context()->stream(); - auto* executor = stream->parent(); - RnnInputMode input_mode; OP_REQUIRES_OK(context, - ToRNNInputMode(rnn_input_mode(), model_shapes.num_units, - model_shapes.input_size, &input_mode)); + AllocateOutputs(context, model_shapes, params->shape(), + &input_backprop, &input_h_backprop, + &input_c_backprop, ¶ms_backprop)); - auto input_desc_s = executor->createRnnSequenceTensorDescriptor( - input_shape.dim_size(0), input_shape.dim_size(1), - input_shape.dim_size(2), data_type); - OP_REQUIRES_OK(context, FromExecutorStatus(input_desc_s)); - auto input_desc = input_desc_s.ConsumeValueOrDie(); - - auto hidden_state_desc_s = executor->createRnnStateTensorDescriptor( - hidden_state_shape.dim_size(0), hidden_state_shape.dim_size(1), - hidden_state_shape.dim_size(2), data_type); - OP_REQUIRES_OK(context, FromExecutorStatus(hidden_state_desc_s)); - auto hidden_state_desc = hidden_state_desc_s.ConsumeValueOrDie(); - - auto output_desc_s = executor->createRnnSequenceTensorDescriptor( - output_shape.dim_size(0), output_shape.dim_size(1), - output_shape.dim_size(2), data_type); - OP_REQUIRES_OK(context, FromExecutorStatus(output_desc_s)); - auto output_desc = output_desc_s.ConsumeValueOrDie(); - - auto input_data = AsDeviceMemory(input); - auto input_h_data = AsDeviceMemory(input_h); - DeviceMemory input_c_data; - if (HasInputC()) { - input_c_data = AsDeviceMemory(input_c); - } - auto params_data = AsDeviceMemory(params); - auto output_data = AsDeviceMemory(output); - auto output_h_data = AsDeviceMemory(output_h); - DeviceMemory output_c_data; - if (HasInputC()) { - output_c_data = AsDeviceMemory(output_c); - } - auto output_backprop_data = AsDeviceMemory(output_backprop); - auto output_h_backprop_data = AsDeviceMemory(output_h_backprop); - DeviceMemory output_c_backprop_data; - if (HasInputC()) { - output_c_backprop_data = AsDeviceMemory(output_c_backprop); - } - auto input_backprop_data = AsDeviceMemory(input_backprop); - auto input_h_backprop_data = AsDeviceMemory(input_h_backprop); - DeviceMemory input_c_backprop_data; - if (HasInputC()) { - input_c_backprop_data = AsDeviceMemory(input_c_backprop); - } - auto params_backprop_data = AsDeviceMemory(params_backprop); - auto reserve_space_uint8 = CastDeviceMemory(reserve_space); // Creates a memory callback for the workspace. The memory lives to the end // of this kernel calls. CudnnRnnAllocatorInTemp workspace_allocator(context); - bool launch_status = false; + const AlgorithmConfig default_algo_config; + Status launch_status; { mutex_lock l(mu_); - RnnScratchSpace& rnn_state = rnn_state_cache_[model_shapes]; - if (rnn_state.rnn_desc == nullptr || ResetRndGenState()) { - CudnnRNNPersistentSpaceAllocator* dropout_state_allocator = - new CudnnRNNPersistentSpaceAllocator(context); - rnn_state.dropout_state_allocator.reset(dropout_state_allocator); - const AlgorithmConfig algo_config; - auto rnn_desc_s = executor->createRnnDescriptor( - model_shapes.num_layers, model_shapes.num_units, - model_shapes.input_size, input_mode, rnn_direction_mode(), - rnn_mode(), data_type, algo_config, dropout(), seed(), - dropout_state_allocator); - OP_REQUIRES_OK(context, FromExecutorStatus(rnn_desc_s)); - rnn_state.rnn_desc = std::move(rnn_desc_s.ConsumeValueOrDie()); - } - launch_status = - stream - ->ThenRnnBackward( - *rnn_state.rnn_desc, *input_desc, input_data, - *hidden_state_desc, input_h_data, *hidden_state_desc, - input_c_data, params_data, *output_desc, output_data, - *hidden_state_desc, output_h_data, *hidden_state_desc, - output_c_data, output_backprop_data, output_h_backprop_data, - output_c_backprop_data, &input_backprop_data, - &input_h_backprop_data, &input_c_backprop_data, - ¶ms_backprop_data, &reserve_space_uint8, - &workspace_allocator, /*output_result_profile=*/nullptr) - .ok(); + RnnDescriptor* rnn_desc_ptr = nullptr; + OP_REQUIRES_OK( + context, GetCachedRnnDescriptor(context, model_shapes, input_mode, + default_algo_config, + &rnn_state_cache_, &rnn_desc_ptr)); + launch_status = DoBackward( + context, *rnn_desc_ptr, model_types(), model_shapes, input, input_h, + input_c, params, output, output_h, output_c, output_backprop, + output_h_backprop, output_c_backprop, reserve_space, input_backprop, + input_h_backprop, input_c_backprop, params_backprop, + &workspace_allocator, /*output_profile_result=*/nullptr); } - OP_REQUIRES(context, launch_status, - errors::Internal("Failed to call ThenRnnBackward")); + OP_REQUIRES_OK(context, launch_status); } private: mutex mu_; - std::unordered_map - rnn_state_cache_ GUARDED_BY(mu_); + RnnStateCache rnn_state_cache_ GUARDED_BY(mu_); + + Status ExtractBackwardInputs( + OpKernelContext* context, const CudnnRnnModelShapes& model_shapes, + const CudnnModelTypes& model_types, const Tensor** output, + const Tensor** output_h, const Tensor** output_c, + const Tensor** output_backprop, const Tensor** output_h_backprop, + const Tensor** output_c_backprop, const Tensor** reserve_space) { + TF_RETURN_IF_ERROR(context->input("output", output)); + TF_RETURN_IF_ERROR(context->input("output_backprop", output_backprop)); + TF_RETURN_IF_ERROR(context->input("output_h", output_h)); + TF_RETURN_IF_ERROR(context->input("output_h_backprop", output_h_backprop)); + if (model_types.HasInputC()) { + TF_RETURN_IF_ERROR(context->input("output_c", output_c)); + TF_RETURN_IF_ERROR( + context->input("output_c_backprop", output_c_backprop)); + } + TF_RETURN_IF_ERROR(context->input("reserve_space", reserve_space)); + const TensorShape& hidden_state_shape = model_shapes.hidden_state_shape; + const TensorShape& output_shape = model_shapes.output_shape; + + if (output_shape != (*output)->shape()) { + return errors::InvalidArgument( + "Invalid output shape: ", (*output)->shape().DebugString(), " ", + output_shape.DebugString()); + } + if (hidden_state_shape != (*output_h)->shape()) { + return errors::InvalidArgument( + "Invalid output_h shape: ", (*output_h)->shape().DebugString(), " ", + hidden_state_shape.DebugString()); + } + + if (output_shape != (*output_backprop)->shape()) { + return errors::InvalidArgument("Invalid output_backprop shape: ", + (*output_backprop)->shape().DebugString(), + " ", output_shape.DebugString()); + } + if (hidden_state_shape != (*output_h_backprop)->shape()) { + return errors::InvalidArgument( + "Invalid output_h_backprop shape: ", + (*output_h_backprop)->shape().DebugString(), " ", + hidden_state_shape.DebugString()); + } + + if (model_types.HasInputC()) { + if (hidden_state_shape != (*output_c)->shape()) { + return errors::InvalidArgument( + "Invalid output_c shape: ", (*output_c)->shape().DebugString(), " ", + hidden_state_shape.DebugString()); + } + if (hidden_state_shape != (*output_c_backprop)->shape()) { + return errors::InvalidArgument( + "Invalid output_c_backprop shape: ", + (*output_c_backprop)->shape().DebugString(), " ", + hidden_state_shape.DebugString()); + } + } + return Status::OK(); + } + + Status AllocateOutputs(OpKernelContext* context, + const CudnnRnnModelShapes& model_shapes, + const TensorShape& params_shape, + Tensor** input_backprop, Tensor** input_h_backprop, + Tensor** input_c_backprop, Tensor** params_backprop) { + const TensorShape& input_shape = model_shapes.input_shape; + const TensorShape& hidden_state_shape = model_shapes.hidden_state_shape; + + TF_RETURN_IF_ERROR( + context->allocate_output(0, input_shape, input_backprop)); + TF_RETURN_IF_ERROR( + context->allocate_output(1, hidden_state_shape, input_h_backprop)); + if (HasInputC()) { + TF_RETURN_IF_ERROR( + context->allocate_output(2, hidden_state_shape, input_c_backprop)); + } else { + // Only LSTM uses input_c and output_c. So for all other models, we only + // need to create dummy outputs. + TF_RETURN_IF_ERROR(context->allocate_output(2, {}, input_c_backprop)); + } + TF_RETURN_IF_ERROR( + context->allocate_output(3, params_shape, params_backprop)); + return Status::OK(); + } }; #define REGISTER_GPU(T) \ -- GitLab From a4b408543dd3b882131f522359bcb547c7972e4f Mon Sep 17 00:00:00 2001 From: Jeremy Lau Date: Fri, 13 Apr 2018 17:36:00 -0700 Subject: [PATCH 365/791] VLOG(1) all OutOfRange CtxFailures, and LOG(WARNING) all other CtxFailures. This unifies the logging behavior of the OP_REQUIRES and OP_REQUIRES_OK macros. PiperOrigin-RevId: 192848921 --- tensorflow/core/framework/op_kernel.cc | 48 +++++++++++++++----------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/tensorflow/core/framework/op_kernel.cc b/tensorflow/core/framework/op_kernel.cc index 05171006b0..ca91d68f79 100644 --- a/tensorflow/core/framework/op_kernel.cc +++ b/tensorflow/core/framework/op_kernel.cc @@ -1273,51 +1273,59 @@ const Eigen::SyclDevice& OpKernelContext::eigen_device() const { } #endif +namespace { +template +void CtxFailureInternal(OpKernelT* op_kernel, const char* file, int line, + const Status& s) { + const string logging_prefix = + file == nullptr ? "CtxFailure: " + : strings::StrCat("CtxFailure at ", io::Basename(file), + ":", line, ": "); + + if (errors::IsOutOfRange(s)) { + // VLOG OutOfRange errors. Dataset ops create OutOfRange errors when they + // reach end-of-sequence. + VLOG(1) << logging_prefix << s; + } else { + LOG(WARNING) << logging_prefix << s; + } + op_kernel->SetStatus(s); +} +} // anonymous namespace + void OpKernelConstruction::CtxFailure(const Status& s) { - VLOG(1) << s; - SetStatus(s); + CtxFailureInternal(this, nullptr, 0, s); } void OpKernelConstruction::CtxFailureWithWarning(const Status& s) { - LOG(WARNING) << s; - SetStatus(s); + CtxFailureInternal(this, nullptr, 0, s); } void OpKernelConstruction::CtxFailure(const char* file, int line, const Status& s) { - VLOG(1) << "OP_REQUIRES failed at " << io::Basename(file) << ":" << line - << " : " << s; - SetStatus(s); + CtxFailureInternal(this, file, line, s); } void OpKernelConstruction::CtxFailureWithWarning(const char* file, int line, const Status& s) { - LOG(WARNING) << "OP_REQUIRES failed at " << io::Basename(file) << ":" << line - << " : " << s; - SetStatus(s); + CtxFailureInternal(this, file, line, s); } void OpKernelContext::CtxFailure(const Status& s) { - VLOG(1) << s; - SetStatus(s); + CtxFailureInternal(this, nullptr, 0, s); } void OpKernelContext::CtxFailureWithWarning(const Status& s) { - LOG(WARNING) << s; - SetStatus(s); + CtxFailureInternal(this, nullptr, 0, s); } void OpKernelContext::CtxFailure(const char* file, int line, const Status& s) { - VLOG(1) << "OP_REQUIRES failed at " << io::Basename(file) << ":" << line - << " : " << s; - SetStatus(s); + CtxFailureInternal(this, file, line, s); } void OpKernelContext::CtxFailureWithWarning(const char* file, int line, const Status& s) { - LOG(WARNING) << "OP_REQUIRES failed at " << io::Basename(file) << ":" << line - << " : " << s; - SetStatus(s); + CtxFailureInternal(this, file, line, s); } } // namespace tensorflow -- GitLab From 6e533eb718b33f23ab3f06025cbf680258534d76 Mon Sep 17 00:00:00 2001 From: Igor Saprykin Date: Fri, 13 Apr 2018 17:47:58 -0700 Subject: [PATCH 366/791] Add a caveat about make_initiliazable_iterator to the README. PiperOrigin-RevId: 192850014 --- tensorflow/contrib/distribute/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/contrib/distribute/README.md b/tensorflow/contrib/distribute/README.md index 14de1e8f49..2482731198 100644 --- a/tensorflow/contrib/distribute/README.md +++ b/tensorflow/contrib/distribute/README.md @@ -130,6 +130,8 @@ adjusting your learning rate or batch size according to the number of GPUs. We are working on addressing this limitation by splitting each batch across GPUs instead. * PartitionedVariables are not supported yet. +* Input pipelines with Datasets that capture stateful objects and rely on +`make_initializable_iterator` are not supported yet. ## What's next? -- GitLab From ef24ad14502e992716c49fdd5c63e6b2c2fb6b5a Mon Sep 17 00:00:00 2001 From: Asim Shankar Date: Fri, 13 Apr 2018 17:51:37 -0700 Subject: [PATCH 367/791] Java: Bump release to 1.8.0-rc0 PiperOrigin-RevId: 192850310 --- tensorflow/java/maven/libtensorflow/pom.xml | 2 +- tensorflow/java/maven/libtensorflow_jni/pom.xml | 2 +- tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml | 2 +- tensorflow/java/maven/pom.xml | 2 +- tensorflow/java/maven/proto/pom.xml | 2 +- tensorflow/java/maven/tensorflow/pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tensorflow/java/maven/libtensorflow/pom.xml b/tensorflow/java/maven/libtensorflow/pom.xml index c99d04869a..9c1601753b 100644 --- a/tensorflow/java/maven/libtensorflow/pom.xml +++ b/tensorflow/java/maven/libtensorflow/pom.xml @@ -6,7 +6,7 @@ org.tensorflow parentpom - 1.7.0 + 1.8.0-rc0 ../ libtensorflow diff --git a/tensorflow/java/maven/libtensorflow_jni/pom.xml b/tensorflow/java/maven/libtensorflow_jni/pom.xml index 4561c2c8ad..3d013e12b0 100644 --- a/tensorflow/java/maven/libtensorflow_jni/pom.xml +++ b/tensorflow/java/maven/libtensorflow_jni/pom.xml @@ -6,7 +6,7 @@ org.tensorflow parentpom - 1.7.0 + 1.8.0-rc0 ../ libtensorflow_jni diff --git a/tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml b/tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml index 82a2b8e769..40e44af1f5 100644 --- a/tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml +++ b/tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml @@ -6,7 +6,7 @@ org.tensorflow parentpom - 1.7.0 + 1.8.0-rc0 ../ libtensorflow_jni_gpu diff --git a/tensorflow/java/maven/pom.xml b/tensorflow/java/maven/pom.xml index 4c1ec0cc80..82bfd0c73a 100644 --- a/tensorflow/java/maven/pom.xml +++ b/tensorflow/java/maven/pom.xml @@ -6,7 +6,7 @@ 4.0.0 org.tensorflow parentpom - 1.7.0 + 1.8.0-rc0 pom https://www.tensorflow.org diff --git a/tensorflow/java/maven/proto/pom.xml b/tensorflow/java/maven/proto/pom.xml index fcd8236bad..0a2775a500 100644 --- a/tensorflow/java/maven/proto/pom.xml +++ b/tensorflow/java/maven/proto/pom.xml @@ -6,7 +6,7 @@ org.tensorflow parentpom - 1.7.0 + 1.8.0-rc0 ../ proto diff --git a/tensorflow/java/maven/tensorflow/pom.xml b/tensorflow/java/maven/tensorflow/pom.xml index 241581713a..61961432a7 100644 --- a/tensorflow/java/maven/tensorflow/pom.xml +++ b/tensorflow/java/maven/tensorflow/pom.xml @@ -6,7 +6,7 @@ org.tensorflow parentpom - 1.7.0 + 1.8.0-rc0 ../ tensorflow -- GitLab From 3652556dab3ebfe0152232facc7304fe5754aecb Mon Sep 17 00:00:00 2001 From: Scott Zhu Date: Fri, 13 Apr 2018 17:52:20 -0700 Subject: [PATCH 368/791] Merge changes from github. PiperOrigin-RevId: 192850372 --- tensorflow/BUILD | 5 +- tensorflow/compiler/jit/BUILD | 1 + .../compiler/jit/mark_for_compilation_pass.cc | 4 + tensorflow/contrib/cmake/external/grpc.cmake | 1 + .../copy_graph/python/util/copy_elements.py | 4 +- tensorflow/contrib/data/__init__.py | 2 + .../contrib/data/python/kernel_tests/BUILD | 1 + .../kernel_tests/batch_dataset_op_test.py | 70 ++++ .../kernel_tests/sequence_dataset_op_test.py | 10 + tensorflow/contrib/data/python/ops/BUILD | 1 + .../contrib/data/python/ops/batching.py | 41 ++ .../contrib/distribute/python/values.py | 2 +- .../contrib/kernel_methods/python/losses.py | 6 +- .../python/mappers/random_fourier_features.py | 44 +- .../mappers/random_fourier_features_test.py | 2 +- .../contrib/kfac/python/ops/fisher_blocks.py | 82 ++-- .../contrib/lite/build_ios_universal_lib.sh | 15 +- .../contrib/metrics/python/ops/metric_ops.py | 29 +- tensorflow/contrib/rnn/python/ops/rnn_cell.py | 2 +- .../seq2seq/python/ops/attention_wrapper.py | 4 +- tensorflow/contrib/sparsemax/__init__.py | 2 +- .../contrib/sparsemax/python/ops/sparsemax.py | 2 +- .../contrib/tensorrt/convert/convert_graph.cc | 10 +- .../contrib/tensorrt/convert/convert_nodes.cc | 68 ++- .../base_api/api_def_ClipByValue.pbtxt | 36 ++ .../python_api/api_def_ClipByValue.pbtxt | 4 + .../core/common_runtime/process_util.cc | 21 +- tensorflow/core/grappler/optimizers/BUILD | 23 +- tensorflow/core/kernels/BUILD | 2 + tensorflow/core/kernels/cwise_op_abs.cc | 2 - tensorflow/core/kernels/cwise_op_clip.cc | 225 ++++++++++ tensorflow/core/kernels/cwise_op_clip.h | 61 +++ .../core/kernels/cwise_op_clip_gpu.cu.cc | 134 ++++++ tensorflow/core/kernels/maxpooling_op.cc | 93 ++++- .../core/kernels/segment_reduction_ops.h | 6 + tensorflow/core/ops/dataset_ops.cc | 12 +- tensorflow/core/ops/math_ops.cc | 8 + tensorflow/core/platform/macros.h | 9 +- .../docs_src/community/documentation.md | 18 +- tensorflow/docs_src/extend/adding_an_op.md | 159 +++---- .../docs_src/get_started/custom_estimators.md | 2 +- tensorflow/docs_src/install/install_c.md | 2 +- .../docs_src/performance/performance_guide.md | 8 +- .../docs_src/programmers_guide/debugger.md | 61 ++- tensorflow/python/BUILD | 1 + tensorflow/python/framework/dtypes.py | 10 + tensorflow/python/framework/dtypes_test.py | 5 + tensorflow/python/framework/function_test.py | 3 +- tensorflow/python/framework/tensor_shape.py | 3 + .../python/framework/tensor_shape_test.py | 5 + .../keras/_impl/keras/utils/io_utils.py | 14 +- .../python/kernel_tests/clip_ops_test.py | 124 +++++- .../python/kernel_tests/pooling_ops_test.py | 6 - tensorflow/python/ops/clip_ops.py | 30 ++ tensorflow/python/ops/hidden_ops.txt | 395 ++++++++++++++++++ tensorflow/python/util/tf_inspect.py | 43 +- tensorflow/tensorflow.bzl | 53 ++- .../tools/api/generator/create_python_api.py | 3 +- tensorflow/tools/docker/Dockerfile | 2 +- tensorflow/tools/docker/Dockerfile.devel | 2 + tensorflow/tools/docker/Dockerfile.devel-gpu | 2 + tensorflow/tools/docker/Dockerfile.gpu | 2 +- .../notebooks/3_mnist_from_scratch.ipynb | 6 +- .../docker/parameterized_docker_build.sh | 4 +- tensorflow/tools/docs/BUILD | 2 +- tensorflow/tools/docs/build_docs_test.py | 5 - tensorflow/tools/docs/generate_lib.py | 19 +- tensorflow/tools/docs/generate_lib_test.py | 3 - tensorflow/tools/docs/parser.py | 56 ++- tensorflow/tools/docs/parser_test.py | 80 +++- tensorflow/tools/docs/pretty_docs.py | 12 +- tensorflow/tools/docs/py_guide_parser.py | 2 +- tensorflow/workspace.bzl | 13 +- 73 files changed, 1797 insertions(+), 402 deletions(-) create mode 100644 tensorflow/core/api_def/base_api/api_def_ClipByValue.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_ClipByValue.pbtxt create mode 100644 tensorflow/core/kernels/cwise_op_clip.cc create mode 100644 tensorflow/core/kernels/cwise_op_clip.h create mode 100644 tensorflow/core/kernels/cwise_op_clip_gpu.cu.cc create mode 100644 tensorflow/python/ops/hidden_ops.txt diff --git a/tensorflow/BUILD b/tensorflow/BUILD index cfafffdd13..f2ad16fa04 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -450,11 +450,12 @@ tf_cc_shared_object( linkstatic = 1, visibility = ["//visibility:public"], deps = [ + "//tensorflow/core:core_cpu_impl", "//tensorflow/core:framework_internal_impl", + "//tensorflow/core:gpu_runtime_impl", + "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry_impl", "//tensorflow/core:lib_internal_impl", - "//tensorflow/core:core_cpu_impl", "//tensorflow/stream_executor:stream_executor_impl", - "//tensorflow/core:gpu_runtime_impl", ] + tf_additional_binary_deps(), ) diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD index 6edeb7047f..50fa95c4f3 100644 --- a/tensorflow/compiler/jit/BUILD +++ b/tensorflow/compiler/jit/BUILD @@ -318,6 +318,7 @@ cc_library( "//tensorflow/core:lib", "//tensorflow/core:lib_internal", "//tensorflow/core:protos_all_cc", + "//tensorflow/core/kernels:bounds_check", ], ) diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc index 0c9fbf3d54..8e2ee0f1d7 100644 --- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc +++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc @@ -35,6 +35,7 @@ limitations under the License. #include "tensorflow/core/framework/types.h" #include "tensorflow/core/graph/algorithm.h" #include "tensorflow/core/graph/control_flow.h" +#include "tensorflow/core/kernels/bounds_check.h" #include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/public/version.h" @@ -441,6 +442,9 @@ string DescribeCycle(const GraphCycles& cycles, const Graph& graph, int src, } auto node_name = [&cycles, &graph](int node_id) { + if (!FastBoundsCheck(node_id, graph.num_node_ids())) { + return string("(null)"); + } auto* node = graph.FindNodeId(node_id); if (node == nullptr) { return string("(null)"); diff --git a/tensorflow/contrib/cmake/external/grpc.cmake b/tensorflow/contrib/cmake/external/grpc.cmake index bec8177a3f..35c2a294ec 100644 --- a/tensorflow/contrib/cmake/external/grpc.cmake +++ b/tensorflow/contrib/cmake/external/grpc.cmake @@ -35,6 +35,7 @@ else() set(grpc_STATIC_LIBRARIES ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgrpc++_unsecure.a ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgrpc_unsecure.a + ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libaddress_sorting.a ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/third_party/cares/cares/lib/libcares.a ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgpr.a) endif() diff --git a/tensorflow/contrib/copy_graph/python/util/copy_elements.py b/tensorflow/contrib/copy_graph/python/util/copy_elements.py index b806799202..102bc460fd 100644 --- a/tensorflow/contrib/copy_graph/python/util/copy_elements.py +++ b/tensorflow/contrib/copy_graph/python/util/copy_elements.py @@ -201,7 +201,7 @@ def copy_op_to_graph(org_instance, to_graph, variables, scope=''): #An instance of tensorflow.core.framework.node_def_pb2.NodeDef, it #stores String-based info such as name, device and type of the op. #Unique to every Operation instance. - new_node_def = deepcopy(op._node_def) + new_node_def = deepcopy(op.node_def) #Change the name new_node_def.name = new_name @@ -211,7 +211,7 @@ def copy_op_to_graph(org_instance, to_graph, variables, scope=''): #Make a copy of the op_def too. #Its unique to every _type_ of Operation. - op_def = deepcopy(op._op_def) + op_def = deepcopy(op.op_def) #Initialize a new Operation instance new_op = ops.Operation(new_node_def, to_graph, new_inputs, output_types, diff --git a/tensorflow/contrib/data/__init__.py b/tensorflow/contrib/data/__init__.py index f58e5ec1f0..637b1dc46c 100644 --- a/tensorflow/contrib/data/__init__.py +++ b/tensorflow/contrib/data/__init__.py @@ -25,6 +25,7 @@ See the @{$datasets$Importing Data} Programmer's Guide for an overview. @@Counter @@SqlDataset +@@assert_element_shape @@batch_and_drop_remainder @@bucket_by_sequence_length @@dense_to_sparse_batch @@ -55,6 +56,7 @@ from __future__ import print_function # pylint: disable=unused-import +from tensorflow.contrib.data.python.ops.batching import assert_element_shape from tensorflow.contrib.data.python.ops.batching import batch_and_drop_remainder from tensorflow.contrib.data.python.ops.batching import dense_to_sparse_batch from tensorflow.contrib.data.python.ops.batching import map_and_batch diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD index a8481dc90a..b475c9fa6b 100644 --- a/tensorflow/contrib/data/python/kernel_tests/BUILD +++ b/tensorflow/contrib/data/python/kernel_tests/BUILD @@ -21,6 +21,7 @@ py_test( "//tensorflow/python:dtypes", "//tensorflow/python:errors", "//tensorflow/python:math_ops", + "//tensorflow/python:script_ops", "//tensorflow/python:sparse_tensor", "//tensorflow/python:string_ops", "//tensorflow/python:tensor_shape", diff --git a/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py index 75482f67da..413d873797 100644 --- a/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py @@ -28,8 +28,10 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops import script_ops from tensorflow.python.ops import string_ops from tensorflow.python.platform import test @@ -579,5 +581,73 @@ class PaddedBatchDatasetSerializationTest( lambda: build_dataset(seq_lens2), 8) +class RestructuredDatasetTest(test.TestCase): + + def test_assert_element_shape(self): + + def create_unknown_shape_dataset(x): + return script_ops.py_func(lambda _: (np.ones(2, dtype=np.float32), + np.zeros((3, 4), dtype=np.int32)), + [x], + [dtypes.float32, dtypes.int32]) + + dataset = dataset_ops.Dataset.range(5).map(create_unknown_shape_dataset) + unknown_shapes = (tensor_shape.TensorShape(None), + tensor_shape.TensorShape(None)) + self.assertEqual(unknown_shapes, dataset.output_shapes) + + expected_shapes = (tensor_shape.TensorShape(2), + tensor_shape.TensorShape((3, 4))) + result = dataset.apply(batching.assert_element_shape(expected_shapes)) + self.assertEqual(expected_shapes, result.output_shapes) + + iterator = result.make_initializable_iterator() + init_op = iterator.initializer + get_next = iterator.get_next() + with self.test_session() as sess: + sess.run(init_op) + for _ in range(5): + sess.run(get_next) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def test_assert_wrong_element_shape(self): + + def create_dataset(_): + return (array_ops.ones(2, dtype=dtypes.float32), + array_ops.zeros((3, 4), dtype=dtypes.int32)) + + dataset = dataset_ops.Dataset.range(3).map(create_dataset) + wrong_shapes = (tensor_shape.TensorShape(2), + tensor_shape.TensorShape((3, 10))) + with self.assertRaises(ValueError): + dataset.apply(batching.assert_element_shape(wrong_shapes)) + + def test_assert_wrong_element_shape_on_unknown_shape_dataset(self): + + def create_unknown_shape_dataset(x): + return script_ops.py_func(lambda _: (np.ones(2, dtype=np.float32), + np.zeros((3, 4), dtype=np.int32)), + [x], + [dtypes.float32, dtypes.int32]) + + dataset = dataset_ops.Dataset.range(3).map(create_unknown_shape_dataset) + unknown_shapes = (tensor_shape.TensorShape(None), + tensor_shape.TensorShape(None)) + self.assertEqual(unknown_shapes, dataset.output_shapes) + + wrong_shapes = (tensor_shape.TensorShape(2), + tensor_shape.TensorShape((3, 10))) + iterator = ( + dataset.apply(batching.assert_element_shape(wrong_shapes)) + .make_initializable_iterator()) + init_op = iterator.initializer + get_next = iterator.get_next() + with self.test_session() as sess: + sess.run(init_op) + with self.assertRaises(errors.InvalidArgumentError): + sess.run(get_next) + + if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/data/python/kernel_tests/sequence_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/sequence_dataset_op_test.py index b044ff1775..d0cb203a3a 100644 --- a/tensorflow/contrib/data/python/kernel_tests/sequence_dataset_op_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/sequence_dataset_op_test.py @@ -47,6 +47,11 @@ class SequenceDatasetSerializationTest( # Skip nothing self.run_core_tests(lambda: self._build_skip_dataset(0), None, 10) + def testInvalidSkip(self): + with self.assertRaisesRegexp(ValueError, + 'Shape must be rank 0 but is rank 1'): + self.run_core_tests(lambda: self._build_skip_dataset([1, 2]), None, 0) + def _build_take_dataset(self, count): components = (np.arange(10),) return dataset_ops.Dataset.from_tensor_slices(components).take(count) @@ -69,6 +74,11 @@ class SequenceDatasetSerializationTest( # Take nothing self.run_core_tests(lambda: self._build_take_dataset(0), None, 0) + def testInvalidTake(self): + with self.assertRaisesRegexp(ValueError, + 'Shape must be rank 0 but is rank 1'): + self.run_core_tests(lambda: self._build_take_dataset([1, 2]), None, 0) + def _build_repeat_dataset(self, count, take_count=3): components = (np.arange(10),) return dataset_ops.Dataset.from_tensor_slices(components).take( diff --git a/tensorflow/contrib/data/python/ops/BUILD b/tensorflow/contrib/data/python/ops/BUILD index 7c28d1f005..0e4590829b 100644 --- a/tensorflow/contrib/data/python/ops/BUILD +++ b/tensorflow/contrib/data/python/ops/BUILD @@ -112,6 +112,7 @@ py_library( srcs = ["batching.py"], srcs_version = "PY2AND3", deps = [ + "//tensorflow/contrib/framework:framework_py", "//tensorflow/python:array_ops", "//tensorflow/python:dataset_ops_gen", "//tensorflow/python:dtypes", diff --git a/tensorflow/contrib/data/python/ops/batching.py b/tensorflow/contrib/data/python/ops/batching.py index a212adf6cf..28db949da9 100644 --- a/tensorflow/contrib/data/python/ops/batching.py +++ b/tensorflow/contrib/data/python/ops/batching.py @@ -17,6 +17,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.contrib.framework import with_shape from tensorflow.python.data.ops import dataset_ops from tensorflow.python.data.util import nest from tensorflow.python.data.util import sparse @@ -345,6 +346,46 @@ class _RestructuredDataset(dataset_ops.Dataset): return self._output_shapes +def assert_element_shape(expected_shapes): + """Assert the shape of this `Dataset`. + + ```python + shapes = [tf.TensorShape([16, 256]), tf.TensorShape(None)] + result = dataset.apply(tf.contrib.data.assert_element_shape(shapes)) + print(result.output_shapes) # ==> "((16, 256), )" + ``` + + If dataset shapes and expected_shape, are fully defined, assert they match. + Otherwise, add assert op that will validate the shapes when tensors are + evaluated, and set shapes on tensors, respectively. + + Args: + expected_shapes: A nested structure of `tf.TensorShape` objects. + + Returns: + A `Dataset` transformation function, which can be passed to + @{tf.data.Dataset.apply} + """ + + def _check_shape(*elements): + flatten_tensors = nest.flatten(elements) + flatten_shapes = nest.flatten(expected_shapes) + checked_tensors = [ + with_shape(shape, tensor) + for shape, tensor in zip(flatten_shapes, flatten_tensors) + ] + return nest.pack_sequence_as(elements, checked_tensors) + + def _apply_fn(dataset): + return _RestructuredDataset( + dataset.map(_check_shape), + dataset.output_types, + output_shapes=expected_shapes, + output_classes=dataset.output_classes) + + return _apply_fn + + class _MapAndBatchDataset(dataset_ops.MapDataset): """A `Dataset` that maps a function over a batch of elements.""" diff --git a/tensorflow/contrib/distribute/python/values.py b/tensorflow/contrib/distribute/python/values.py index 9acb6a9db9..87bf059038 100644 --- a/tensorflow/contrib/distribute/python/values.py +++ b/tensorflow/contrib/distribute/python/values.py @@ -73,7 +73,7 @@ class DistributedValues(object): @property def devices(self): - return self._index.keys() + return list(self._index.keys()) def __str__(self): return "%s:%s" % (self.__class__.__name__, self._index) diff --git a/tensorflow/contrib/kernel_methods/python/losses.py b/tensorflow/contrib/kernel_methods/python/losses.py index f182fef067..4ef0a66a52 100644 --- a/tensorflow/contrib/kernel_methods/python/losses.py +++ b/tensorflow/contrib/kernel_methods/python/losses.py @@ -43,10 +43,10 @@ def sparse_multiclass_hinge_loss( This is a generalization of standard (binary) hinge loss. For a given instance with correct label c*, the loss is given by: - loss = max_{c != c*} logits_c - logits_{c*} + 1. + $$loss = max_{c != c*} logits_c - logits_{c*} + 1.$$ or equivalently - loss = max_c { logits_c - logits_{c*} + I_{c != c*} } - where I_{c != c*} = 1 if c != c* and 0 otherwise. + $$loss = max_c { logits_c - logits_{c*} + I_{c != c*} }$$ + where \\(I_{c != c*} = 1\ \text{if}\ c != c*\\) and 0 otherwise. Args: labels: `Tensor` of shape [batch_size] or [batch_size, 1]. Corresponds to diff --git a/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features.py b/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features.py index 9dc01124ab..9a721a9d44 100644 --- a/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features.py +++ b/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features.py @@ -34,33 +34,31 @@ class RandomFourierFeatureMapper(dkm.DenseKernelMapper): r"""Class that implements Random Fourier Feature Mapping (RFFM) in TensorFlow. The RFFM mapping is used to approximate the Gaussian (RBF) kernel: - ``` - exp(-||x-y||_2^2 / (2 * sigma^2)) - ``` + $$(exp(-||x-y||_2^2 / (2 * \sigma^2))$$ The implementation of RFFM is based on the following paper: "Random Features for Large-Scale Kernel Machines" by Ali Rahimi and Ben Recht. (link: https://people.eecs.berkeley.edu/~brecht/papers/07.rah.rec.nips.pdf) - The mapping uses a matrix `Omega \in R^{d x D}` and a bias vector `b \in R^D` - where `d` is the input dimension (number of dense input features) and `D` is - the output dimension (i.e., dimension of the feature space the input is mapped - to). Each entry of `Omega` is sampled i.i.d. from a (scaled) Gaussian - distribution and each entry of `b` is sampled independently and uniformly from - [0, 2 * pi]. - - For a single input feature vector x in R^d, its RFFM is defined as: - ``` - sqrt(2/D) * cos(x * Omega + b) - ``` - where `cos` is the element-wise cosine function and `x, b` are represented as - row vectors. The aforementioned paper shows that the linear kernel of - RFFM-mapped vectors approximates the Gaussian kernel of the initial vectors. + The mapping uses a matrix \\(\Omega \in R^{d x D}\\) and a bias vector + \\(b \in R^D\\) where \\(d\\) is the input dimension (number of dense input + features) and \\(D\\) is the output dimension (i.e., dimension of the feature + space the input is mapped to). Each entry of \\(\Omega\\) is sampled i.i.d. + from a (scaled) Gaussian distribution and each entry of \\(b\\) is sampled + independently and uniformly from [0, \\(2 * \pi\\)]. + + For a single input feature vector \\(x \in R^d\\), its RFFM is defined as: + $$\sqrt(2/D) * cos(x * \Omega + b)$$ + + where \\(cos\\) is the element-wise cosine function and \\(x, b\\) are + represented as row vectors. The aforementioned paper shows that the linear + kernel of RFFM-mapped vectors approximates the Gaussian kernel of the initial + vectors. """ def __init__(self, input_dim, output_dim, stddev=1.0, seed=1, name=None): - """Constructs a RandomFourierFeatureMapper instance. + r"""Constructs a RandomFourierFeatureMapper instance. Args: input_dim: The dimension (number of features) of the tensors to be mapped. @@ -68,11 +66,11 @@ class RandomFourierFeatureMapper(dkm.DenseKernelMapper): stddev: The standard deviation of the Gaussian kernel to be approximated. The error of the classifier trained using this approximation is very sensitive to this parameter. - seed: An integer used to initialize the parameters (`Omega` and `b`) of - the mapper. For repeatable sequences across different invocations of the - mapper object (for instance, to ensure consistent mapping both at - training and eval/inference if these happen in different invocations), - set this to the same integer. + seed: An integer used to initialize the parameters (\\(\Omega\\) and + \\(b\\)) of the mapper. For repeatable sequences across different + invocations of the mapper object (for instance, to ensure consistent + mapping both at training and eval/inference if these happen in + different invocations), set this to the same integer. name: name for the mapper object. """ # TODO(sibyl-vie3Poto): Maybe infer input_dim and/or output_dim (if not explicitly diff --git a/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features_test.py b/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features_test.py index 6f4a264485..91929184a2 100644 --- a/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features_test.py +++ b/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features_test.py @@ -34,7 +34,7 @@ def _inner_product(x, y): """Inner product between tensors x and y. The input tensors are assumed to be in ROW representation, that is, the method - returns x * y^T. + returns \\(x * y^T\\). Args: x: input tensor in row format diff --git a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py index e0d9cb5ea9..00b3673a74 100644 --- a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py +++ b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py @@ -19,11 +19,11 @@ Information matrix. Suppose one has a model that parameterizes a posterior distribution over 'y' given 'x' with parameters 'params', p(y | x, params). Its Fisher Information matrix is given by, - F(params) = E[ v(x, y, params) v(x, y, params)^T ] + $$F(params) = E[ v(x, y, params) v(x, y, params)^T ]$$ where, - v(x, y, params) = (d / d params) log p(y | x, params) + $$v(x, y, params) = (d / d params) log p(y | x, params)$$ and the expectation is taken with respect to the data's distribution for 'x' and the model's posterior distribution for 'y', @@ -85,7 +85,7 @@ def normalize_damping(damping, num_replications): def compute_pi_tracenorm(left_cov, right_cov): """Computes the scalar constant pi for Tikhonov regularization/damping. - pi = sqrt( (trace(A) / dim(A)) / (trace(B) / dim(B)) ) + $$\pi = \sqrt{ (trace(A) / dim(A)) / (trace(B) / dim(B)) }$$ See section 6.3 of https://arxiv.org/pdf/1503.05671.pdf for details. Args: @@ -462,14 +462,14 @@ class FullyConnectedDiagonalFB(InputOutputMultiTower, FisherBlock): Let 'params' be a vector parameterizing a model and 'i' an arbitrary index into it. We are interested in Fisher(params)[i, i]. This is, - Fisher(params)[i, i] = E[ v(x, y, params) v(x, y, params)^T ][i, i] - = E[ v(x, y, params)[i] ^ 2 ] + $$Fisher(params)[i, i] = E[ v(x, y, params) v(x, y, params)^T ][i, i] + = E[ v(x, y, params)[i] ^ 2 ]$$ Consider fully connected layer in this model with (unshared) weight matrix 'w'. For an example 'x' that produces layer inputs 'a' and output preactivations 's', - v(x, y, w) = vec( a (d loss / d s)^T ) + $$v(x, y, w) = vec( a (d loss / d s)^T )$$ This FisherBlock tracks Fisher(params)[i, i] for all indices 'i' corresponding to the layer's parameters 'w'. @@ -532,14 +532,14 @@ class ConvDiagonalFB(InputOutputMultiTower, FisherBlock): Let 'params' be a vector parameterizing a model and 'i' an arbitrary index into it. We are interested in Fisher(params)[i, i]. This is, - Fisher(params)[i, i] = E[ v(x, y, params) v(x, y, params)^T ][i, i] - = E[ v(x, y, params)[i] ^ 2 ] + $$Fisher(params)[i, i] = E[ v(x, y, params) v(x, y, params)^T ][i, i] + = E[ v(x, y, params)[i] ^ 2 ]$$ Consider a convoluational layer in this model with (unshared) filter matrix 'w'. For an example image 'x' that produces layer inputs 'a' and output preactivations 's', - v(x, y, w) = vec( sum_{loc} a_{loc} (d loss / d s_{loc})^T ) + $$v(x, y, w) = vec( sum_{loc} a_{loc} (d loss / d s_{loc})^T )$$ where 'loc' is a single (x, y) location in an image. @@ -805,12 +805,12 @@ class ConvKFCBasicFB(InputOutputMultiTower, KroneckerProductFB): 'w'. For a minibatch that produces inputs 'a' and output preactivations 's', this FisherBlock estimates, - F(w) = #locations * kronecker(E[flat(a) flat(a)^T], - E[flat(ds) flat(ds)^T]) + $$F(w) = \#locations * kronecker(E[flat(a) flat(a)^T], + E[flat(ds) flat(ds)^T])$$ where - ds = (d / ds) log p(y | x, w) + $$ds = (d / ds) log p(y | x, w)$$ #locations = number of (x, y) locations where 'w' is applied. where the expectation is taken over all examples and locations and flat() @@ -1567,7 +1567,7 @@ class FullyConnectedSeriesFB(InputOutputMultiTowerMultiUse, if self._option == SeriesFBApproximation.option1: - # Note that L_A = A0^(-1/2) * U_A and L_G = G0^(-1/2) * U_G. + # Note that \\(L_A = A0^{-1/2} * U_A and L_G = G0^{-1/2} * U_G.\\) L_A, psi_A = self._input_factor.get_option1quants( self._input_damping_func) L_G, psi_G = self._output_factor.get_option1quants( @@ -1581,33 +1581,33 @@ class FullyConnectedSeriesFB(InputOutputMultiTowerMultiUse, T = self._num_timesteps return (1 - x)**2 / (T * (1 - x**2) - 2 * x * (1 - x**T)) - # Y = gamma( psi_G*psi_A^T ) (computed element-wise) + # \\(Y = \gamma( psi_G*psi_A^T )\\) (computed element-wise) # Even though Y is Z-independent we are recomputing it from the psi's # each since Y depends on both A and G quantities, and it is relatively # cheap to compute. Y = gamma(array_ops.reshape(psi_G, [int(psi_G.shape[0]), -1]) * psi_A) - # Z = L_G^T * Z * L_A + # \\(Z = L_G^T * Z * L_A\\) # This is equivalent to the following computation from the original # pseudo-code: - # Z = G0^(-1/2) * Z * A0^(-1/2) - # Z = U_G^T * Z * U_A + # \\(Z = G0^{-1/2} * Z * A0^{-1/2}\\) + # \\(Z = U_G^T * Z * U_A\\) Z = math_ops.matmul(L_G, math_ops.matmul(Z, L_A), transpose_a=True) - # Z = Z .* Y + # \\(Z = Z .* Y\\) Z *= Y - # Z = L_G * Z * L_A^T + # \\(Z = L_G * Z * L_A^T\\) # This is equivalent to the following computation from the original # pseudo-code: - # Z = U_G * Z * U_A^T - # Z = G0^(-1/2) * Z * A0^(-1/2) + # \\(Z = U_G * Z * U_A^T\\) + # \\(Z = G0^{-1/2} * Z * A0^{-1/2}\\) Z = math_ops.matmul(L_G, math_ops.matmul(Z, L_A, transpose_b=True)) elif self._option == SeriesFBApproximation.option2: - # Note that P_A = A_1^T * A_0^(-1) and P_G = G_1^T * G_0^(-1), - # and K_A = A_0^(-1/2) * E_A and K_G = G_0^(-1/2) * E_G. + # Note that \\(P_A = A_1^T * A_0^{-1} and P_G = G_1^T * G_0^{-1}\\), + # and \\(K_A = A_0^{-1/2} * E_A\ and\ K_G = G_0^{-1/2} * E_G.\\) P_A, K_A, mu_A = self._input_factor.get_option2quants( self._input_damping_func) P_G, K_G, mu_G = self._output_factor.get_option2quants( @@ -1616,26 +1616,26 @@ class FullyConnectedSeriesFB(InputOutputMultiTowerMultiUse, # Our approach differs superficially from the pseudo-code in the paper # in order to reduce the total number of matrix-matrix multiplies. # In particular, the first three computations in the pseudo code are - # Z = G0^(-1/2) * Z * A0^(-1/2) - # Z = Z - hPsi_G^T * Z * hPsi_A - # Z = E_G^T * Z * E_A - # Noting that hPsi = C0^(-1/2) * C1 * C0^(-1/2), so that - # C0^(-1/2) * hPsi = C0^(-1) * C1 * C0^(-1/2) = P^T * C0^(-1/2) + # \\(Z = G0^{-1/2} * Z * A0^{-1/2}\\) + # \\(Z = Z - hPsi_G^T * Z * hPsi_A\\) + # \\(Z = E_G^T * Z * E_A\\) + # Noting that hPsi = C0^{-1/2} * C1 * C0^{-1/2}\\), so that + # \\(C0^{-1/2} * hPsi = C0^{-1} * C1 * C0^{-1/2} = P^T * C0^{-1/2}\\) # the entire computation can be written as - # Z = E_G^T * (G0^(-1/2) * Z * A0^(-1/2) - # - hPsi_G^T * G0^(-1/2) * Z * A0^(-1/2) * hPsi_A) * E_A - # = E_G^T * (G0^(-1/2) * Z * A0^(-1/2) - # - G0^(-1/2) * P_G * Z * P_A^T * A0^(-1/2)) * E_A - # = E_G^T * G0^(-1/2) * Z * A0^(-1/2) * E_A - # - E_G^T* G0^(-1/2) * P_G * Z * P_A^T * A0^(-1/2) * E_A - # = K_G^T * Z * K_A - K_G^T * P_G * Z * P_A^T * K_A + # \\(Z = E_G^T * (G0^{-1/2} * Z * A0^{-1/2}\\) + # \\( - hPsi_G^T * G0^{-1/2} * Z * A0^{-1/2} * hPsi_A) * E_A\\) + # \\( = E_G^T * (G0^{-1/2} * Z * A0^{-1/2}\\) + # \\( - G0^{-1/2} * P_G * Z * P_A^T * A0^{-1/2}) * E_A\\) + # \\( = E_G^T * G0^{-1/2} * Z * A0^{-1/2} * E_A\\) + # \\( - E_G^T* G0^{-1/2} * P_G * Z * P_A^T * A0^{-1/2} * E_A\\) + # \\( = K_G^T * Z * K_A - K_G^T * P_G * Z * P_A^T * K_A\\) # This final expression is computed by the following two lines: - # Z = Z - P_G * Z * P_A^T + # \\(Z = Z - P_G * Z * P_A^T\\) Z -= math_ops.matmul(P_G, math_ops.matmul(Z, P_A, transpose_b=True)) - # Z = K_G^T * Z * K_A + # \\(Z = K_G^T * Z * K_A\\) Z = math_ops.matmul(K_G, math_ops.matmul(Z, K_A), transpose_a=True) - # Z = Z ./ (1*1^T - mu_G*mu_A^T) + # \\(Z = Z ./ (1*1^T - mu_G*mu_A^T)\\) # Be careful with the outer product. We don't want to accidentally # make it an inner-product instead. tmp = 1.0 - array_ops.reshape(mu_G, [int(mu_G.shape[0]), -1]) * mu_A @@ -1646,13 +1646,13 @@ class FullyConnectedSeriesFB(InputOutputMultiTowerMultiUse, # We now perform the transpose/reverse version of the operations # derived above, whose derivation from the original pseudo-code is # analgous. - # Z = K_G * Z * K_A^T + # \\(Z = K_G * Z * K_A^T\\) Z = math_ops.matmul(K_G, math_ops.matmul(Z, K_A, transpose_b=True)) - # Z = Z - P_G^T * Z * P_A + # \\(Z = Z - P_G^T * Z * P_A\\) Z -= math_ops.matmul(P_G, math_ops.matmul(Z, P_A), transpose_a=True) - # Z = normalize (1/E[T]) * Z + # \\(Z = normalize (1/E[T]) * Z\\) # Note that this normalization is done because we compute the statistics # by averaging, not summing, over time. (And the gradient is presumably # summed over time, not averaged, and thus their scales are different.) diff --git a/tensorflow/contrib/lite/build_ios_universal_lib.sh b/tensorflow/contrib/lite/build_ios_universal_lib.sh index 4a9023ff33..9f398f4a9f 100755 --- a/tensorflow/contrib/lite/build_ios_universal_lib.sh +++ b/tensorflow/contrib/lite/build_ios_universal_lib.sh @@ -19,11 +19,16 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "$SCRIPT_DIR/../../.." -make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=x86_64 -j 8 -make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=i386 -j 8 -make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=armv7 -j 8 -make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=armv7s -j 8 -make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=arm64 -j 8 +make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=x86_64 -j 8 \ +$SCRIPT_DIR/gen/lib/ios_x86_64/libtensorflow-lite.a +make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=i386 -j 8 \ +$SCRIPT_DIR/gen/lib/ios_i386/libtensorflow-lite.a +make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=armv7 -j 8 \ +$SCRIPT_DIR/gen/lib/ios_armv7/libtensorflow-lite.a +make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=armv7s -j 8 \ +$SCRIPT_DIR/gen/lib/ios_armv7s/libtensorflow-lite.a +make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=arm64 -j 8 \ +$SCRIPT_DIR/gen/lib/ios_arm64/libtensorflow-lite.a lipo \ tensorflow/contrib/lite/gen/lib/ios_x86_64/libtensorflow-lite.a \ diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py index 81f05e7ce5..9c8ae48094 100644 --- a/tensorflow/contrib/metrics/python/ops/metric_ops.py +++ b/tensorflow/contrib/metrics/python/ops/metric_ops.py @@ -63,6 +63,8 @@ def _safe_div(numerator, denominator, name): name=name) +@deprecated(None, 'Please switch to tf.metrics.true_positives. Note that the ' + 'order of the labels and predictions arguments has been switched.') def streaming_true_positives(predictions, labels, weights=None, @@ -107,6 +109,8 @@ def streaming_true_positives(predictions, name=name) +@deprecated(None, 'Please switch to tf.metrics.true_negatives. Note that the ' + 'order of the labels and predictions arguments has been switched.') def streaming_true_negatives(predictions, labels, weights=None, @@ -151,6 +155,8 @@ def streaming_true_negatives(predictions, name=name) +@deprecated(None, 'Please switch to tf.metrics.false_positives. Note that the ' + 'order of the labels and predictions arguments has been switched.') def streaming_false_positives(predictions, labels, weights=None, @@ -195,6 +201,8 @@ def streaming_false_positives(predictions, name=name) +@deprecated(None, 'Please switch to tf.metrics.false_negatives. Note that the ' + 'order of the labels and predictions arguments has been switched.') def streaming_false_negatives(predictions, labels, weights=None, @@ -238,6 +246,7 @@ def streaming_false_negatives(predictions, name=name) +@deprecated(None, 'Please switch to tf.metrics.mean') def streaming_mean(values, weights=None, metrics_collections=None, @@ -287,6 +296,7 @@ def streaming_mean(values, name=name) +@deprecated(None, 'Please switch to tf.metrics.mean_tensor') def streaming_mean_tensor(values, weights=None, metrics_collections=None, @@ -340,9 +350,8 @@ def streaming_mean_tensor(values, name=name) -@deprecated(None, - 'Please switch to tf.metrics.accuracy. Note that the order of the ' - 'labels and predictions arguments has been switched.') +@deprecated(None, 'Please switch to tf.metrics.accuracy. Note that the order ' + 'of the labels and predictions arguments has been switched.') def streaming_accuracy(predictions, labels, weights=None, @@ -400,6 +409,8 @@ def streaming_accuracy(predictions, name=name) +@deprecated(None, 'Please switch to tf.metrics.precision. Note that the order ' + 'of the labels and predictions arguments has been switched.') def streaming_precision(predictions, labels, weights=None, @@ -456,6 +467,8 @@ def streaming_precision(predictions, name=name) +@deprecated(None, 'Please switch to tf.metrics.recall. Note that the order ' + 'of the labels and predictions arguments has been switched.') def streaming_recall(predictions, labels, weights=None, @@ -975,8 +988,8 @@ def streaming_curve_points(labels=None, return points, update_op -@deprecated(None, 'Please switch to tf.metrics.auc. Note that the order of the ' - 'labels and predictions arguments has been switched.') +@deprecated(None, 'Please switch to tf.metrics.auc. Note that the order of ' + 'the labels and predictions arguments has been switched.') def streaming_auc(predictions, labels, weights=None, @@ -1797,9 +1810,9 @@ def streaming_sensitivity_at_specificity(predictions, name=name) -@deprecated( - None, 'Please switch to tf.metrics.precision_at_thresholds. Note that the ' - 'order of the labels and predictions arguments has been switched.') +@deprecated(None, + 'Please switch to tf.metrics.precision_at_thresholds. Note that ' + 'the order of the labels and predictions arguments are switched.') def streaming_precision_at_thresholds(predictions, labels, thresholds, diff --git a/tensorflow/contrib/rnn/python/ops/rnn_cell.py b/tensorflow/contrib/rnn/python/ops/rnn_cell.py index 2f6ae9f367..b12e2cd5ed 100644 --- a/tensorflow/contrib/rnn/python/ops/rnn_cell.py +++ b/tensorflow/contrib/rnn/python/ops/rnn_cell.py @@ -2891,7 +2891,7 @@ class WeightNormLSTMCell(rnn_cell_impl.RNNCell): output_size = weight.get_shape().as_list()[1] g = vs.get_variable(name, [output_size], dtype=weight.dtype) - return nn_impl.l2_normalize(weight, dim=0) * g + return nn_impl.l2_normalize(weight, axis=0) * g def _linear(self, args, diff --git a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py index 9e0d69593f..f0f143ddfc 100644 --- a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py +++ b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py @@ -610,8 +610,8 @@ def monotonic_attention(p_choose_i, previous_attention, mode): addition, once an input sequence element is attended to at a given output timestep, elements occurring before it cannot be attended to at subsequent output timesteps. This function generates attention distributions according - to these assumptions. For more information, see ``Online and Linear-Time - Attention by Enforcing Monotonic Alignments''. + to these assumptions. For more information, see `Online and Linear-Time + Attention by Enforcing Monotonic Alignments`. Args: p_choose_i: Probability of choosing input sequence/memory element i. Should diff --git a/tensorflow/contrib/sparsemax/__init__.py b/tensorflow/contrib/sparsemax/__init__.py index 19d213fb3e..7bc726f4a8 100644 --- a/tensorflow/contrib/sparsemax/__init__.py +++ b/tensorflow/contrib/sparsemax/__init__.py @@ -14,7 +14,7 @@ # ============================================================================== """Module that implements sparsemax and sparsemax loss, see [1]. -[1] https://arxiv.org/abs/1602.02068 +[1]: https://arxiv.org/abs/1602.02068 ## Sparsemax diff --git a/tensorflow/contrib/sparsemax/python/ops/sparsemax.py b/tensorflow/contrib/sparsemax/python/ops/sparsemax.py index 890ca20f4c..e617af2ff1 100644 --- a/tensorflow/contrib/sparsemax/python/ops/sparsemax.py +++ b/tensorflow/contrib/sparsemax/python/ops/sparsemax.py @@ -31,7 +31,7 @@ def sparsemax(logits, name=None): """Computes sparsemax activations [1]. For each batch `i` and class `j` we have - sparsemax[i, j] = max(logits[i, j] - tau(logits[i, :]), 0) + $$sparsemax[i, j] = max(logits[i, j] - tau(logits[i, :]), 0)$$ [1]: https://arxiv.org/abs/1602.02068 diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc index ff8cc6374d..b412b296e0 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc +++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc @@ -405,7 +405,13 @@ tensorflow::Status ConvertGraphDefToTensorRT( max_mem_per_engine, static_graph_properties, &output_edge_map, precision_mode); if (precision_mode == INT8MODE) { - TF_RETURN_IF_ERROR(GetCalibNode(&p)); + tensorflow::Status status = GetCalibNode(&p); + if (status != tensorflow::Status::OK()) { + LOG(WARNING) << "subgraph conversion error for subgraph_index:" << count + << " due to: \"" << status.ToString() + << "\" SKIPPING......( " << subgraph_node_names.size() + << " nodes)"; + } } else { tensorflow::Status status = ConvertSubGraphToTensorRT(&p); if (status != tensorflow::Status::OK()) { @@ -414,8 +420,8 @@ tensorflow::Status ConvertGraphDefToTensorRT( << "\" SKIPPING......( " << subgraph_node_names.size() << " nodes)"; } - count++; } + count++; } graph.ToGraphDef(new_graph_def); return tensorflow::Status::OK(); diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc index e920a797fe..b81ae9dc3e 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc +++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc @@ -443,7 +443,9 @@ class Converter { * 2) Control dependency inputs contain caret at the beginning and we * remove this and annotate the edge as a control dependency. ************************************************************************/ - string name = input_name[0] == '^' ? input_name.substr(1) : input_name; + // skip control nodes + if (input_name[0] == '^') continue; + string name = input_name; auto first = name.find_first_of(':'); if (first != string::npos && first + 2 == name.size() && name[first + 1] == '0') @@ -2262,6 +2264,7 @@ tensorflow::Status InjectCalibrationNode(tensorrt::convert::SubGraphParams& s) { auto ws = new tensorflow::tensorrt::TRTWeightStore(); TF_CHECK_OK(weight_rmgr->Create(calib_op_name, calib_op_name, ws)); Converter converter(op_res->network_, ws, s.precision_mode == FP16MODE); + std::vector input_names; std::vector input_dtypes; for (const std::pair& input : s.input_inds) { @@ -2270,20 +2273,41 @@ tensorflow::Status InjectCalibrationNode(tensorrt::convert::SubGraphParams& s) { int output_idx = input.second; tensorflow::Node* node = s.graph.FindNodeId(node_id); auto node_name = node->name(); - input_names.push_back(node_name); // insert original node name without port - // TODO(jie): alternative :) - if (!s.graph_properties.HasOutputProperties(node_name)) + // input_names should use the node name in the graph + // here it should be the input tensor name -> matching the binding + // insert original node name without port + auto tensor_name = node_name; + if (output_idx != 0) { + tensor_name = StrCat(tensor_name, ":", output_idx); + } + + VLOG(2) << "input name: " << node_name << " tensor_name: " << tensor_name + << " idx: " << output_idx; + + auto shape_inference_node_name = node_name; + auto shape_inference_output_idx = output_idx; + // rewire the shape inference to original node in the graph + if (s.output_edge_map->count(tensor_name)) { + shape_inference_node_name = s.output_edge_map->at(tensor_name).second; + shape_inference_output_idx = s.output_edge_map->at(tensor_name).first; + } + if (shape_inference_output_idx < 0) continue; + VLOG(2) << "shapeinference name: " << shape_inference_node_name + << " idx: " << shape_inference_output_idx; + + if (!s.graph_properties.HasOutputProperties(shape_inference_node_name)) return tensorflow::errors::Internal("failed to find input node: " + - node_name); + shape_inference_node_name); - auto op_info_vec = s.graph_properties.GetOutputProperties(node_name); - if (static_cast(op_info_vec.size()) < output_idx) + auto op_info_vec = + s.graph_properties.GetOutputProperties(shape_inference_node_name); + if (static_cast(op_info_vec.size()) <= shape_inference_output_idx) return tensorflow::errors::Internal( - "accessing output index of: ", output_idx, ", at node: ", node_name, - "with output entry from shape_map: ", op_info_vec.size()); - - auto op_info = op_info_vec.at(output_idx); + "accessing output index of: ", shape_inference_output_idx, + ", at node: ", shape_inference_node_name, + " with output entry from shape_map: ", op_info_vec.size()); + auto op_info = op_info_vec.at(shape_inference_output_idx); tensorflow::DataType tf_dtype = op_info.dtype(); input_dtypes.push_back(tf_dtype); @@ -2294,16 +2318,23 @@ tensorflow::Status InjectCalibrationNode(tensorrt::convert::SubGraphParams& s) { << "' failed"; return type_status; } - TF_CHECK_OK(ConvertDType(tf_dtype, &dtype)); VLOG(2) << "accessing output index of: " << output_idx << ", at node: " << node_name << "with output entry from shape_map: " << op_info_vec.size(); - // TODO(ben,jie): update TRT input format/dimension nvinfer1::DimsCHW input_dim_psuedo_chw; for (int i = 0; i < 3; i++) input_dim_psuedo_chw.d[i] = 1; + // TODO(jie): TRT 3.x only support 4 dimensional input tensor. + // update the code once TRT 4.0 comes out. + if (op_info.shape().dim_size() != 4) { + string err_str = "Require 4 dimensional input."; + StrAppend(&err_str, " Got ", op_info.shape().dim_size(), " ", + shape_inference_node_name); + return tensorflow::errors::Unimplemented(err_str); + } + for (int i = 1; i < op_info.shape().dim_size(); i++) { VLOG(2) << "dimension: " << i << " , size: " << op_info.shape().dim(i).size(); @@ -2312,8 +2343,11 @@ tensorflow::Status InjectCalibrationNode(tensorrt::convert::SubGraphParams& s) { // TODO(ben,jie): proper way to restore input tensor name? auto input_tensor_name = node_name; - if (output_idx != 0) input_tensor_name = StrCat(node_name, ":", output_idx); + if (output_idx != 0) { + input_tensor_name = StrCat(node_name, ":", output_idx); + } + input_names.push_back(input_tensor_name); nvinfer1::ITensor* input_tensor = converter.network()->addInput( input_tensor_name.c_str(), dtype, input_dim_psuedo_chw); @@ -2377,11 +2411,13 @@ tensorflow::Status InjectCalibrationNode(tensorrt::convert::SubGraphParams& s) { tensor->setType(trt_dtype); } - VLOG(2) << "finished output"; + VLOG(2) << "Finished processing outputs"; // Build the engine op_res->builder_->setMaxBatchSize(s.max_batch_size); op_res->builder_->setMaxWorkspaceSize(s.max_workspace_size_bytes); + VLOG(0) << "Max batch size= " << s.max_batch_size + << " max workspace size= " << s.max_workspace_size_bytes; // Build the TRT op // TODO(sami,ben,jie): proper naming! @@ -2475,7 +2511,7 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef( std::vector input_names; std::vector input_dtypes; for (const std::pair& input : s.input_inds) { - VLOG(2) << "parsing input!!!!!"; + VLOG(2) << "parsing input. Node id= " << input.first; int node_id = input.first; int output_idx = input.second; tensorflow::Node* node = s.graph.FindNodeId(node_id); diff --git a/tensorflow/core/api_def/base_api/api_def_ClipByValue.pbtxt b/tensorflow/core/api_def/base_api/api_def_ClipByValue.pbtxt new file mode 100644 index 0000000000..803d8970ab --- /dev/null +++ b/tensorflow/core/api_def/base_api/api_def_ClipByValue.pbtxt @@ -0,0 +1,36 @@ +op { + graph_op_name: "ClipByValue" + in_arg { + name: "t" + description: <
Version:CPU/GPU:Python Version:Compiler:Build Tools:cuDNN:CUDA:
tensorflow-1.7.0rc1CPU3.5-3.6MSVC 2015 update 3Cmake v3.6.3N/AN/A
tensorflow_gpu-1.7.0rc1GPU3.5-3.6MSVC 2015 update 3Cmake v3.6.379
tensorflow-1.7.0CPU3.5-3.6MSVC 2015 update 3Cmake v3.6.3N/AN/A
tensorflow_gpu-1.7.0GPU3.5-3.6MSVC 2015 update 3Cmake v3.6.379
tensorflow-1.6.0CPU3.5-3.6MSVC 2015 update 3Cmake v3.6.3N/AN/A
tensorflow_gpu-1.6.0GPU3.5-3.6MSVC 2015 update 3Cmake v3.6.379
tensorflow-1.5.0CPU3.5-3.6MSVC 2015 update 3Cmake v3.6.3N/AN/A
tensorflow-1.0.0CPU3.5MSVC 2015 update 3Cmake v3.6.3N/AN/A
tensorflow_gpu-1.0.0GPU3.5MSVC 2015 update 3Cmake v3.6.35.18
+ + +## Build the C or Java libraries + +The instructions above are tailored to building the TensorFlow Python packages. + +If you're interested in building the libraries for the TensorFlow C API, do the +following: + +1. Follow the steps up to [Configure the installation](#ConfigureInstallation) +2. Build the C libraries following instructions in the [README](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/lib_package/README.md). + +If you're interested inv building the libraries for the TensorFlow Java API, +do the following: + +1. Follow the steps up to [Configure the installation](#ConfigureInstallation) +2. Build the Java library following instructions in the [README](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/lib_package/README.md). -- GitLab From d218339e6a05a984ef7b9a49d66db219d862936e Mon Sep 17 00:00:00 2001 From: Yifei Feng Date: Thu, 19 Apr 2018 01:26:07 -0700 Subject: [PATCH 637/791] Remove proto import in header files for core/kernels/boosted_trees. Move implementations that requires declaration of TreeEnsemble to .cc files. The goal is to make kernels mostly independent of proto headers, which will let us lock down our .so import PiperOrigin-RevId: 193478404 --- .../core/kernels/boosted_trees/resources.cc | 138 ++++++++++++++++ .../core/kernels/boosted_trees/resources.h | 154 +++++------------- 2 files changed, 178 insertions(+), 114 deletions(-) diff --git a/tensorflow/core/kernels/boosted_trees/resources.cc b/tensorflow/core/kernels/boosted_trees/resources.cc index 2ea12c522c..c410748c27 100644 --- a/tensorflow/core/kernels/boosted_trees/resources.cc +++ b/tensorflow/core/kernels/boosted_trees/resources.cc @@ -21,6 +21,35 @@ limitations under the License. namespace tensorflow { +// Constructor. +BoostedTreesEnsembleResource::BoostedTreesEnsembleResource() + : tree_ensemble_( + protobuf::Arena::CreateMessage( + &arena_)) {} + +string BoostedTreesEnsembleResource::DebugString() { + return strings::StrCat("TreeEnsemble[size=", tree_ensemble_->trees_size(), + "]"); +} + +bool BoostedTreesEnsembleResource::InitFromSerialized(const string& serialized, + const int64 stamp_token) { + CHECK_EQ(stamp(), -1) << "Must Reset before Init."; + if (ParseProtoUnlimited(tree_ensemble_, serialized)) { + set_stamp(stamp_token); + return true; + } + return false; +} + +string BoostedTreesEnsembleResource::SerializeAsString() const { + return tree_ensemble_->SerializeAsString(); +} + +int32 BoostedTreesEnsembleResource::num_trees() const { + return tree_ensemble_->trees_size(); +} + int32 BoostedTreesEnsembleResource::next_node( const int32 tree_id, const int32 node_id, const int32 index_in_batch, const std::vector::ConstVec>& bucketized_features) const { @@ -49,6 +78,115 @@ float BoostedTreesEnsembleResource::node_value(const int32 tree_id, } } +int32 BoostedTreesEnsembleResource::GetNumLayersGrown( + const int32 tree_id) const { + DCHECK_LT(tree_id, tree_ensemble_->trees_size()); + return tree_ensemble_->tree_metadata(tree_id).num_layers_grown(); +} + +void BoostedTreesEnsembleResource::SetNumLayersGrown( + const int32 tree_id, int32 new_num_layers) const { + DCHECK_LT(tree_id, tree_ensemble_->trees_size()); + tree_ensemble_->mutable_tree_metadata(tree_id)->set_num_layers_grown( + new_num_layers); +} + +void BoostedTreesEnsembleResource::UpdateLastLayerNodesRange( + const int32 node_range_start, int32 node_range_end) const { + tree_ensemble_->mutable_growing_metadata()->set_last_layer_node_start( + node_range_start); + tree_ensemble_->mutable_growing_metadata()->set_last_layer_node_end( + node_range_end); +} + +void BoostedTreesEnsembleResource::GetLastLayerNodesRange( + int32* node_range_start, int32* node_range_end) const { + *node_range_start = + tree_ensemble_->growing_metadata().last_layer_node_start(); + *node_range_end = tree_ensemble_->growing_metadata().last_layer_node_end(); +} + +int64 BoostedTreesEnsembleResource::GetNumNodes(const int32 tree_id) { + DCHECK_LT(tree_id, tree_ensemble_->trees_size()); + return tree_ensemble_->trees(tree_id).nodes_size(); +} + +int32 BoostedTreesEnsembleResource::GetNumLayersAttempted() { + return tree_ensemble_->growing_metadata().num_layers_attempted(); +} + +bool BoostedTreesEnsembleResource::is_leaf(const int32 tree_id, + const int32 node_id) const { + DCHECK_LT(tree_id, tree_ensemble_->trees_size()); + DCHECK_LT(node_id, tree_ensemble_->trees(tree_id).nodes_size()); + const auto& node = tree_ensemble_->trees(tree_id).nodes(node_id); + return node.node_case() == boosted_trees::Node::kLeaf; +} + +int32 BoostedTreesEnsembleResource::feature_id(const int32 tree_id, + const int32 node_id) const { + const auto node = tree_ensemble_->trees(tree_id).nodes(node_id); + DCHECK_EQ(node.node_case(), boosted_trees::Node::kBucketizedSplit); + return node.bucketized_split().feature_id(); +} + +int32 BoostedTreesEnsembleResource::bucket_threshold( + const int32 tree_id, const int32 node_id) const { + const auto node = tree_ensemble_->trees(tree_id).nodes(node_id); + DCHECK_EQ(node.node_case(), boosted_trees::Node::kBucketizedSplit); + return node.bucketized_split().threshold(); +} + +int32 BoostedTreesEnsembleResource::left_id(const int32 tree_id, + const int32 node_id) const { + const auto node = tree_ensemble_->trees(tree_id).nodes(node_id); + DCHECK_EQ(node.node_case(), boosted_trees::Node::kBucketizedSplit); + return node.bucketized_split().left_id(); +} + +int32 BoostedTreesEnsembleResource::right_id(const int32 tree_id, + const int32 node_id) const { + const auto node = tree_ensemble_->trees(tree_id).nodes(node_id); + DCHECK_EQ(node.node_case(), boosted_trees::Node::kBucketizedSplit); + return node.bucketized_split().right_id(); +} + +std::vector BoostedTreesEnsembleResource::GetTreeWeights() const { + return {tree_ensemble_->tree_weights().begin(), + tree_ensemble_->tree_weights().end()}; +} + +float BoostedTreesEnsembleResource::GetTreeWeight(const int32 tree_id) const { + return tree_ensemble_->tree_weights(tree_id); +} + +float BoostedTreesEnsembleResource::IsTreeFinalized(const int32 tree_id) const { + DCHECK_LT(tree_id, tree_ensemble_->trees_size()); + return tree_ensemble_->tree_metadata(tree_id).is_finalized(); +} + +float BoostedTreesEnsembleResource::IsTreePostPruned( + const int32 tree_id) const { + DCHECK_LT(tree_id, tree_ensemble_->trees_size()); + return tree_ensemble_->tree_metadata(tree_id).post_pruned_nodes_meta_size() > + 0; +} + +void BoostedTreesEnsembleResource::SetIsFinalized(const int32 tree_id, + const bool is_finalized) { + DCHECK_LT(tree_id, tree_ensemble_->trees_size()); + return tree_ensemble_->mutable_tree_metadata(tree_id)->set_is_finalized( + is_finalized); +} + +// Sets the weight of i'th tree. +void BoostedTreesEnsembleResource::SetTreeWeight(const int32 tree_id, + const float weight) { + DCHECK_GE(tree_id, 0); + DCHECK_LT(tree_id, num_trees()); + tree_ensemble_->set_tree_weights(tree_id, weight); +} + void BoostedTreesEnsembleResource::UpdateGrowingMetadata() const { tree_ensemble_->mutable_growing_metadata()->set_num_layers_attempted( tree_ensemble_->growing_metadata().num_layers_attempted() + 1); diff --git a/tensorflow/core/kernels/boosted_trees/resources.h b/tensorflow/core/kernels/boosted_trees/resources.h index 561ca3a18a..df78d3f275 100644 --- a/tensorflow/core/kernels/boosted_trees/resources.h +++ b/tensorflow/core/kernels/boosted_trees/resources.h @@ -17,12 +17,16 @@ limitations under the License. #define TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_RESOURCES_H_ #include "tensorflow/core/framework/resource_mgr.h" -#include "tensorflow/core/kernels/boosted_trees/boosted_trees.pb.h" #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/protobuf.h" namespace tensorflow { +// Forward declaration for proto class TreeEnsemble +namespace boosted_trees { +class TreeEnsemble; +} // namespace boosted_trees + // A StampedResource is a resource that has a stamp token associated with it. // Before reading from or applying updates to the resource, the stamp should // be checked to verify that the update is not stale. @@ -42,31 +46,15 @@ class StampedResource : public ResourceBase { // Keep a tree ensemble in memory for efficient evaluation and mutation. class BoostedTreesEnsembleResource : public StampedResource { public: - // Constructor. - BoostedTreesEnsembleResource() - : tree_ensemble_( - protobuf::Arena::CreateMessage( - &arena_)) {} - - string DebugString() override { - return strings::StrCat("TreeEnsemble[size=", tree_ensemble_->trees_size(), - "]"); - } - - bool InitFromSerialized(const string& serialized, const int64 stamp_token) { - CHECK_EQ(stamp(), -1) << "Must Reset before Init."; - if (ParseProtoUnlimited(tree_ensemble_, serialized)) { - set_stamp(stamp_token); - return true; - } - return false; - } - - string SerializeAsString() const { - return tree_ensemble_->SerializeAsString(); - } - - int32 num_trees() const { return tree_ensemble_->trees_size(); } + BoostedTreesEnsembleResource(); + + string DebugString() override; + + bool InitFromSerialized(const string& serialized, const int64 stamp_token); + + string SerializeAsString() const; + + int32 num_trees() const; // Find the next node to which the example (specified by index_in_batch) // traverses down from the current node indicated by tree_id and node_id. @@ -82,73 +70,31 @@ class BoostedTreesEnsembleResource : public StampedResource { float node_value(const int32 tree_id, const int32 node_id) const; - int32 GetNumLayersGrown(const int32 tree_id) const { - DCHECK_LT(tree_id, tree_ensemble_->trees_size()); - return tree_ensemble_->tree_metadata(tree_id).num_layers_grown(); - } + int32 GetNumLayersGrown(const int32 tree_id) const; - void SetNumLayersGrown(const int32 tree_id, int32 new_num_layers) const { - DCHECK_LT(tree_id, tree_ensemble_->trees_size()); - tree_ensemble_->mutable_tree_metadata(tree_id)->set_num_layers_grown( - new_num_layers); - } + void SetNumLayersGrown(const int32 tree_id, int32 new_num_layers) const; void UpdateLastLayerNodesRange(const int32 node_range_start, - int32 node_range_end) const { - tree_ensemble_->mutable_growing_metadata()->set_last_layer_node_start( - node_range_start); - tree_ensemble_->mutable_growing_metadata()->set_last_layer_node_end( - node_range_end); - } + int32 node_range_end) const; void GetLastLayerNodesRange(int32* node_range_start, - int32* node_range_end) const { - *node_range_start = - tree_ensemble_->growing_metadata().last_layer_node_start(); - *node_range_end = tree_ensemble_->growing_metadata().last_layer_node_end(); - } + int32* node_range_end) const; - int64 GetNumNodes(const int32 tree_id) { - DCHECK_LT(tree_id, tree_ensemble_->trees_size()); - return tree_ensemble_->trees(tree_id).nodes_size(); - } + int64 GetNumNodes(const int32 tree_id); void UpdateGrowingMetadata() const; - int32 GetNumLayersAttempted() { - return tree_ensemble_->growing_metadata().num_layers_attempted(); - } - - bool is_leaf(const int32 tree_id, const int32 node_id) const { - DCHECK_LT(tree_id, tree_ensemble_->trees_size()); - DCHECK_LT(node_id, tree_ensemble_->trees(tree_id).nodes_size()); - const auto& node = tree_ensemble_->trees(tree_id).nodes(node_id); - return node.node_case() == boosted_trees::Node::kLeaf; - } - - int32 feature_id(const int32 tree_id, const int32 node_id) const { - const auto node = tree_ensemble_->trees(tree_id).nodes(node_id); - DCHECK_EQ(node.node_case(), boosted_trees::Node::kBucketizedSplit); - return node.bucketized_split().feature_id(); - } - - int32 bucket_threshold(const int32 tree_id, const int32 node_id) const { - const auto node = tree_ensemble_->trees(tree_id).nodes(node_id); - DCHECK_EQ(node.node_case(), boosted_trees::Node::kBucketizedSplit); - return node.bucketized_split().threshold(); - } - - int32 left_id(const int32 tree_id, const int32 node_id) const { - const auto node = tree_ensemble_->trees(tree_id).nodes(node_id); - DCHECK_EQ(node.node_case(), boosted_trees::Node::kBucketizedSplit); - return node.bucketized_split().left_id(); - } - - int32 right_id(const int32 tree_id, const int32 node_id) const { - const auto node = tree_ensemble_->trees(tree_id).nodes(node_id); - DCHECK_EQ(node.node_case(), boosted_trees::Node::kBucketizedSplit); - return node.bucketized_split().right_id(); - } + int32 GetNumLayersAttempted(); + + bool is_leaf(const int32 tree_id, const int32 node_id) const; + + int32 feature_id(const int32 tree_id, const int32 node_id) const; + + int32 bucket_threshold(const int32 tree_id, const int32 node_id) const; + + int32 left_id(const int32 tree_id, const int32 node_id) const; + + int32 right_id(const int32 tree_id, const int32 node_id) const; // Add a tree to the ensemble and returns a new tree_id. int32 AddNewTree(const float weight); @@ -163,38 +109,18 @@ class BoostedTreesEnsembleResource : public StampedResource { // Retrieves tree weights and returns as a vector. // It involves a copy, so should be called only sparingly (like once per // iteration, not per example). - std::vector GetTreeWeights() const { - return {tree_ensemble_->tree_weights().begin(), - tree_ensemble_->tree_weights().end()}; - } - - float GetTreeWeight(const int32 tree_id) const { - return tree_ensemble_->tree_weights(tree_id); - } - - float IsTreeFinalized(const int32 tree_id) const { - DCHECK_LT(tree_id, tree_ensemble_->trees_size()); - return tree_ensemble_->tree_metadata(tree_id).is_finalized(); - } - - float IsTreePostPruned(const int32 tree_id) const { - DCHECK_LT(tree_id, tree_ensemble_->trees_size()); - return tree_ensemble_->tree_metadata(tree_id) - .post_pruned_nodes_meta_size() > 0; - } - - void SetIsFinalized(const int32 tree_id, const bool is_finalized) { - DCHECK_LT(tree_id, tree_ensemble_->trees_size()); - return tree_ensemble_->mutable_tree_metadata(tree_id)->set_is_finalized( - is_finalized); - } + std::vector GetTreeWeights() const; + + float GetTreeWeight(const int32 tree_id) const; + + float IsTreeFinalized(const int32 tree_id) const; + + float IsTreePostPruned(const int32 tree_id) const; + + void SetIsFinalized(const int32 tree_id, const bool is_finalized); // Sets the weight of i'th tree. - void SetTreeWeight(const int32 tree_id, const float weight) { - DCHECK_GE(tree_id, 0); - DCHECK_LT(tree_id, num_trees()); - tree_ensemble_->set_tree_weights(tree_id, weight); - } + void SetTreeWeight(const int32 tree_id, const float weight); // Resets the resource and frees the protos in arena. // Caller needs to hold the mutex lock while calling this. -- GitLab From b2536f05bb156612c96f204041ea31980b711fc8 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 01:56:31 -0700 Subject: [PATCH 638/791] Update feature_util's GetFeatures to show compile-time error for unsupported types instead of a link-time error. PiperOrigin-RevId: 193480683 --- tensorflow/core/example/feature_util.h | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/example/feature_util.h b/tensorflow/core/example/feature_util.h index d977935b8a..2265498b5e 100644 --- a/tensorflow/core/example/feature_util.h +++ b/tensorflow/core/example/feature_util.h @@ -182,13 +182,25 @@ struct FeatureTrait< // Returns true if sequence_example has a feature_list with the specified key. bool HasFeatureList(const string& key, const SequenceExample& sequence_example); +template +struct TypeHasFeatures : std::false_type {}; + +template <> +struct TypeHasFeatures : std::true_type {}; + +template <> +struct TypeHasFeatures : std::true_type {}; + // A family of template functions to return mutable Features proto from a // container proto. Supported ProtoTypes: Example, Features. template -Features* GetFeatures(ProtoType* proto); +typename std::enable_if::value, Features*>::type +GetFeatures(ProtoType* proto); template -const Features& GetFeatures(const ProtoType& proto); +typename std::enable_if::value, + const Features&>::type +GetFeatures(const ProtoType& proto); // Base declaration of a family of template functions to return a read only // repeated field of feature values. @@ -300,7 +312,7 @@ bool HasFeature(const string& key, const Features& features); template bool HasFeature(const string& key, const Example& example) { return HasFeature(key, GetFeatures(example)); -}; +} // DEPRECATED: use HasFeature instead. // TODO(gorban): update all clients in a followup CL. -- GitLab From 5fb3c64421f53aa7ef58ffcee6de47cd4a40fe2d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 02:58:31 -0700 Subject: [PATCH 639/791] Set the random seed in on-demand mode. PiperOrigin-RevId: 193488103 --- tensorflow/compiler/jit/xla_compile_on_demand_op.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/compiler/jit/xla_compile_on_demand_op.cc b/tensorflow/compiler/jit/xla_compile_on_demand_op.cc index 682d6ea8cc..6c2782e28e 100644 --- a/tensorflow/compiler/jit/xla_compile_on_demand_op.cc +++ b/tensorflow/compiler/jit/xla_compile_on_demand_op.cc @@ -67,6 +67,7 @@ Status XlaCompileOnDemandOp::Run(OpKernelContext* ctx, run_options.set_stream(stream); run_options.set_allocator(client->backend().memory_allocator()); run_options.set_intra_op_thread_pool(&ctx->eigen_cpu_device()); + run_options.set_rng_seed(ctx->step_id()); auto run_result = executable->Run(launch_context.arguments(), run_options); TF_RETURN_IF_ERROR(run_result.status()); -- GitLab From bf86d3a46b4e2ef4dabcba211c1ce36cb81ac315 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 04:27:38 -0700 Subject: [PATCH 640/791] Handle corner case in Python 3: members annotated with @classmethod. PiperOrigin-RevId: 193495506 --- tensorflow/contrib/autograph/pyct/inspect_utils.py | 12 +++++++----- .../contrib/autograph/pyct/inspect_utils_test.py | 7 +++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/tensorflow/contrib/autograph/pyct/inspect_utils.py b/tensorflow/contrib/autograph/pyct/inspect_utils.py index a0f56a6c1f..eef74599a7 100644 --- a/tensorflow/contrib/autograph/pyct/inspect_utils.py +++ b/tensorflow/contrib/autograph/pyct/inspect_utils.py @@ -75,13 +75,15 @@ def getdefiningclass(m, owner_class): """Resolves the class (e.g. one of the superclasses) that defined a method.""" # Normalize bound functions to their respective unbound versions. m = _get_unbound_function(m) - last_defining = owner_class - for superclass in tf_inspect.getmro(owner_class): + for superclass in owner_class.__bases__: if hasattr(superclass, m.__name__): superclass_m = getattr(superclass, m.__name__) - if _get_unbound_function(superclass_m) == m: - last_defining = superclass - return last_defining + if _get_unbound_function(superclass_m) is m: + return superclass + elif hasattr(m, '__self__') and m.__self__ == owner_class: + # Python 3 class methods only work this way it seems :S + return superclass + return owner_class def getmethodclass(m): diff --git a/tensorflow/contrib/autograph/pyct/inspect_utils_test.py b/tensorflow/contrib/autograph/pyct/inspect_utils_test.py index cf841dae81..1a212f676a 100644 --- a/tensorflow/contrib/autograph/pyct/inspect_utils_test.py +++ b/tensorflow/contrib/autograph/pyct/inspect_utils_test.py @@ -243,6 +243,10 @@ class InspectUtilsTest(test.TestCase): def bar(self): pass + @classmethod + def class_method(cls): + pass + class Subclass(Superclass): def foo(self): @@ -257,6 +261,9 @@ class InspectUtilsTest(test.TestCase): inspect_utils.getdefiningclass(Subclass.bar, Subclass) is Superclass) self.assertTrue( inspect_utils.getdefiningclass(Subclass.baz, Subclass) is Subclass) + self.assertTrue( + inspect_utils.getdefiningclass(Subclass.class_method, Subclass) is + Superclass) def test_isbuiltin(self): self.assertTrue(inspect_utils.isbuiltin(range)) -- GitLab From 06d802ab61987bde76a30098ff7930c27d561375 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 05:11:30 -0700 Subject: [PATCH 641/791] Support for converting entire class hierarchies: * limit the methods being converted to those that have not been inherited from the superclass * include the (possibly compiled) superclass in the definition of the compiled class * either mark the superclass for conversion or generate an absolute aliased import line, depending on whether it's whitelisted PiperOrigin-RevId: 193499204 --- .../autograph/converters/call_trees.py | 10 ++-- tensorflow/contrib/autograph/impl/api.py | 2 +- .../contrib/autograph/impl/conversion.py | 58 +++++++++++++++--- .../contrib/autograph/impl/conversion_test.py | 60 +++++++++++++++++++ 4 files changed, 117 insertions(+), 13 deletions(-) diff --git a/tensorflow/contrib/autograph/converters/call_trees.py b/tensorflow/contrib/autograph/converters/call_trees.py index e390d1a262..2e5590b46c 100644 --- a/tensorflow/contrib/autograph/converters/call_trees.py +++ b/tensorflow/contrib/autograph/converters/call_trees.py @@ -245,8 +245,6 @@ class CallTreeTransformer(transformer.Base): new_call.keywords = node.keywords return new_call - # pylint:disable=invalid-name - def visit_Expr(self, node): if isinstance(node.value, gast.Call): if anno.hasanno(node.value.func, 'live_val'): @@ -294,15 +292,17 @@ class CallTreeTransformer(transformer.Base): raise NotImplementedError( 'py_func with return values (unknown function)') else: - if self.context.recursive: + if ast_util.matches(node, 'super(_)'): + # super() calls are preserved. The class conversion mechanism will + # ensure that they return the correct value. + pass + elif self.context.recursive: node = self._insert_dynamic_conversion(node) else: # Unresolved functions are allowed in non-recursive mode. pass return node - # pylint:enable=invalid-name - def transform(node, context, uncompiled_modules, nocompile_decorators): """Transform function call to the compiled counterparts. diff --git a/tensorflow/contrib/autograph/impl/api.py b/tensorflow/contrib/autograph/impl/api.py index f97a33326e..d874ef15c9 100644 --- a/tensorflow/contrib/autograph/impl/api.py +++ b/tensorflow/contrib/autograph/impl/api.py @@ -241,7 +241,7 @@ def to_graph(e, module = gast.Module([]) for import_line in config.COMPILED_IMPORT_STATEMENTS: module.body.extend(parser.parse_str(import_line).body) - for dep in conversion_map.dependency_cache.values(): + for dep in reversed(conversion_map.dependency_cache.values()): module.body.append(dep) compiled_node, compiled_src = compiler.ast_to_object(module) diff --git a/tensorflow/contrib/autograph/impl/conversion.py b/tensorflow/contrib/autograph/impl/conversion.py index 5653e991f6..e7230a5f45 100644 --- a/tensorflow/contrib/autograph/impl/conversion.py +++ b/tensorflow/contrib/autograph/impl/conversion.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import collections import imp import gast @@ -39,6 +40,7 @@ from tensorflow.contrib.autograph.converters import side_effect_guards from tensorflow.contrib.autograph.converters import single_return from tensorflow.contrib.autograph.impl import config from tensorflow.contrib.autograph.impl import naming +from tensorflow.contrib.autograph.pyct import ast_util from tensorflow.contrib.autograph.pyct import context from tensorflow.contrib.autograph.pyct import inspect_utils from tensorflow.contrib.autograph.pyct import parser @@ -81,7 +83,9 @@ class ConversionMap(object): self.recursive = recursive self.nocompile_decorators = nocompile_decorators self.partial_types = partial_types if partial_types else () - self.dependency_cache = {} + # Required to output dependencies in discovery order, which should match + # the reverse dependency order. + self.dependency_cache = collections.OrderedDict() self.additional_imports = set() self.name_map = {} self.api_module = api_module @@ -201,6 +205,9 @@ def class_to_graph(c, conversion_map): class_namespace = {} for _, m in members: + # Only convert the members that are directly defined by the class. + if inspect_utils.getdefiningclass(m, c) is not c: + continue node, _, namespace = function_to_graph( m, conversion_map=conversion_map, @@ -214,12 +221,49 @@ def class_to_graph(c, conversion_map): converted_members[m] = node namer = conversion_map.new_namer(class_namespace) class_name = namer.compiled_class_name(c.__name__, c) - node = gast.ClassDef( - class_name, - bases=[], - keywords=[], - body=list(converted_members.values()), - decorator_list=[]) + + # TODO(mdan): This needs to be explained more thoroughly. + # Process any base classes: if the sueprclass if of a whitelisted type, an + # absolute import line is generated. Otherwise, it is marked for conversion + # (as a side effect of the call to namer.compiled_class_name() followed by + # conversion_map.update_name_map(namer)). + output_nodes = [] + renames = {} + bases = [] + for base in c.__bases__: + if isinstance(object, base): + bases.append('object') + continue + if is_whitelisted_for_graph(base): + alias = namer.new_symbol(base.__name__, ()) + output_nodes.append( + gast.ImportFrom( + module=base.__module__, + names=[gast.alias(name=base.__name__, asname=alias)], + level=0)) + else: + # This will trigger a conversion into a class with this name. + alias = namer.compiled_class_name(base.__name__, base) + bases.append(alias) + renames[qual_names.QN(base.__name__)] = qual_names.QN(alias) + conversion_map.update_name_map(namer) + + # Generate the definition of the converted class. + output_nodes.append( + gast.ClassDef( + class_name, + bases=bases, + keywords=[], + body=list(converted_members.values()), + decorator_list=[])) + node = gast.Module(output_nodes) + + # Make a final pass to replace references to the class or its base classes. + # Most commonly, this occurs when making super().__init__() calls. + # TODO(mdan): Making direct references to superclass' superclass will fail. + node = qual_names.resolve(node) + renames[qual_names.QN(c.__name__)] = qual_names.QN(class_name) + node = ast_util.rename_symbols(node, renames) return node, class_name, class_namespace diff --git a/tensorflow/contrib/autograph/impl/conversion_test.py b/tensorflow/contrib/autograph/impl/conversion_test.py index da3220892f..5edd8e74a8 100644 --- a/tensorflow/contrib/autograph/impl/conversion_test.py +++ b/tensorflow/contrib/autograph/impl/conversion_test.py @@ -24,6 +24,7 @@ from tensorflow.contrib.autograph import utils from tensorflow.contrib.autograph.impl import api from tensorflow.contrib.autograph.impl import conversion from tensorflow.python.framework import constant_op +from tensorflow.python.keras._impl.keras.engine import training from tensorflow.python.platform import test @@ -78,6 +79,65 @@ class ConversionTest(test.TestCase): conversion_map.dependency_cache[f].body[0].body[0].value.func.id) self.assertEqual('tf__g', conversion_map.dependency_cache[g].name) + def test_entity_to_graph_class_hierarchy(self): + + class TestBase(object): + + def __init__(self, x='base'): + self.x = x + + def foo(self): + return self.x + + def bar(self): + return self.x + + class TestSubclass(TestBase): + + def __init__(self, y): + super(TestSubclass, self).__init__('sub') + self.y = y + + def foo(self): + return self.y + + def baz(self): + return self.y + + conversion_map = self._simple_conversion_map() + conversion.entity_to_graph(TestSubclass, conversion_map, None, None) + + self.assertTrue(TestBase in conversion_map.dependency_cache) + self.assertTrue(TestSubclass in conversion_map.dependency_cache) + self.assertEqual('TfTestBase', + conversion_map.dependency_cache[TestBase].body[-1].name) + self.assertEqual( + 'TfTestSubclass', + conversion_map.dependency_cache[TestSubclass].body[-1].name) + + def test_entity_to_graph_class_hierarchy_whitelisted(self): + + class TestSubclass(training.Model): + + def __init__(self, y): + super(TestSubclass, self).__init__() + self.built = False + + def call(self, x): + return 3 * x + + conversion_map = self._simple_conversion_map() + conversion.entity_to_graph(TestSubclass, conversion_map, None, None) + + self.assertTrue(TestSubclass in conversion_map.dependency_cache) + self.assertFalse(training.Model in conversion_map.dependency_cache) + self.assertEqual( + 'Model', + conversion_map.dependency_cache[TestSubclass].body[0].names[0].name) + self.assertEqual( + 'TfTestSubclass', + conversion_map.dependency_cache[TestSubclass].body[-1].name) + def test_entity_to_graph_lambda(self): f = lambda a: a -- GitLab From 40f77655affb162d32b7d4861fa68c35fc3d8f7a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 06:58:34 -0700 Subject: [PATCH 642/791] Update the Colorbot demo to use a Keras model in addition to the Estimator. PiperOrigin-RevId: 193508874 --- ...imator.ipynb => rnn_keras_estimator.ipynb} | 677 +++++------------- 1 file changed, 167 insertions(+), 510 deletions(-) rename tensorflow/contrib/autograph/examples/notebooks/{rnn_colorbot_estimator.ipynb => rnn_keras_estimator.ipynb} (50%) diff --git a/tensorflow/contrib/autograph/examples/notebooks/rnn_colorbot_estimator.ipynb b/tensorflow/contrib/autograph/examples/notebooks/rnn_keras_estimator.ipynb similarity index 50% rename from tensorflow/contrib/autograph/examples/notebooks/rnn_colorbot_estimator.ipynb rename to tensorflow/contrib/autograph/examples/notebooks/rnn_keras_estimator.ipynb index 7f5e4d4ac1..324b23c24b 100644 --- a/tensorflow/contrib/autograph/examples/notebooks/rnn_colorbot_estimator.ipynb +++ b/tensorflow/contrib/autograph/examples/notebooks/rnn_keras_estimator.ipynb @@ -62,7 +62,7 @@ } }, "source": [ - "# Case study: building an RNN\n" + "# Case study: training a custom RNN, using Keras and Estimators\n" ] }, { @@ -118,6 +118,16 @@ " length = tf.cast(tf.shape(chars)[0], dtype=tf.int64)\n", " return rgb, chars, length\n", "\n", + "\n", + "def set_static_batch_shape(batch_size):\n", + " def apply(rgb, chars, length):\n", + " rgb.set_shape((batch_size, None))\n", + " chars.set_shape((batch_size, None, 256))\n", + " length.set_shape((batch_size,))\n", + " return rgb, chars, length\n", + " return apply\n", + "\n", + "\n", "def load_dataset(data_dir, url, batch_size, training=True):\n", " \"\"\"Loads the colors data at path into a tf.PaddedDataset.\"\"\"\n", " path = tf.keras.utils.get_file(os.path.basename(url), url, cache_dir=data_dir)\n", @@ -129,7 +139,10 @@ " if training:\n", " dataset = dataset.shuffle(buffer_size=3000)\n", " dataset = dataset.padded_batch(\n", - " batch_size, padded_shapes=([None], [None, None], []))\n", + " batch_size, padded_shapes=((None,), (None, 256), ()))\n", + " # To simplify the model code, we statically set as many of the shapes that we\n", + " # know.\n", + " dataset = dataset.map(set_static_batch_shape(batch_size))\n", " return dataset" ] }, @@ -145,7 +158,8 @@ "source": [ "To show the use of control flow, we write the RNN loop by hand, rather than using a pre-built RNN model.\n", "\n", - "Note how we write the model code in Eager style, with regular `if` and `while` statements. Then, we annotate the functions with `@autograph.convert` to have them automatically compiled to run in graph mode." + "Note how we write the model code in Eager style, with regular `if` and `while` statements. Then, we annotate the functions with `@autograph.convert` to have them automatically compiled to run in graph mode.\n", + "We use Keras to define the model, and we will train it using Estimators." ] }, { @@ -166,70 +180,72 @@ }, "outputs": [], "source": [ - "class RnnColorbot(object):\n", - " \"\"\"Holds the parameters of the colorbot model.\"\"\"\n", + "@autograph.convert()\n", + "class RnnColorbot(tf.keras.Model):\n", + " \"\"\"RNN Colorbot model.\"\"\"\n", "\n", " def __init__(self):\n", + " super(RnnColorbot, self).__init__()\n", " self.lower_cell = tf.contrib.rnn.LSTMBlockCell(256)\n", " self.upper_cell = tf.contrib.rnn.LSTMBlockCell(128)\n", " self.relu_layer = tf.layers.Dense(3, activation=tf.nn.relu)\n", "\n", + "\n", + " def _rnn_layer(self, chars, cell, batch_size, training):\n", + " \"\"\"A single RNN layer.\n", + "\n", + " Args:\n", + " chars: A Tensor of shape (max_sequence_length, batch_size, input_size)\n", + " cell: An object of type tf.contrib.rnn.LSTMBlockCell\n", + " batch_size: Int, the batch size to use\n", + " training: Boolean, whether the layer is used for training\n", + "\n", + " Returns:\n", + " A Tensor of shape (max_sequence_length, batch_size, output_size).\n", + " \"\"\"\n", + " hidden_outputs = []\n", + " autograph.utils.set_element_type(hidden_outputs, tf.float32)\n", + " state, output = cell.zero_state(batch_size, tf.float32)\n", + " for ch in chars:\n", + " cell_output, (state, output) = cell.call(ch, (state, output))\n", + " hidden_outputs.append(cell_output)\n", + " hidden_outputs = hidden_outputs.stack()\n", + " if training:\n", + " hidden_outputs = tf.nn.dropout(hidden_outputs, 0.5)\n", + " return hidden_outputs\n", + "\n", + " def build(self, _):\n", + " \"\"\"Creates the model variables. See keras.Model.build().\"\"\"\n", " self.lower_cell.build(tf.TensorShape((None, 256)))\n", " self.upper_cell.build(tf.TensorShape((None, 256)))\n", - " self.relu_layer.build(tf.TensorShape((None, 128)))\n", + " self.relu_layer.build(tf.TensorShape((None, 128))) \n", + " self.built = True\n", "\n", "\n", - "def rnn_layer(chars, cell, batch_size, training):\n", - " \"\"\"A simple RNN layer.\n", - " \n", - " Args:\n", - " chars: A Tensor of shape (max_sequence_length, batch_size, input_size)\n", - " cell: An object of type tf.contrib.rnn.LSTMBlockCell\n", - " batch_size: Int, the batch size to use\n", - " training: Boolean, whether the layer is used for training\n", + " def call(self, inputs, training=False):\n", + " \"\"\"The RNN model code. Uses Eager and \n", "\n", - " Returns:\n", - " A Tensor of shape (max_sequence_length, batch_size, output_size).\n", - " \"\"\"\n", - " hidden_outputs = []\n", - " autograph.utils.set_element_type(hidden_outputs, tf.float32)\n", - " state, output = cell.zero_state(batch_size, tf.float32)\n", - " for ch in chars:\n", - " cell_output, (state, output) = cell.call(ch, (state, output))\n", - " hidden_outputs.append(cell_output)\n", - " hidden_outputs = hidden_outputs.stack()\n", - " if training:\n", - " hidden_outputs = tf.nn.dropout(hidden_outputs, 0.5)\n", - " return hidden_outputs\n", + " The model consists of two RNN layers (made by lower_cell and upper_cell),\n", + " followed by a fully connected layer with ReLU activation.\n", "\n", + " Args:\n", + " inputs: A tuple (chars, length)\n", + " training: Boolean, whether the layer is used for training\n", "\n", - "@autograph.convert(recursive=True)\n", - "def model(inputs, colorbot, batch_size, training):\n", - " \"\"\"RNNColorbot model.\n", - " \n", - " The model consists of two RNN layers (made by lower_cell and upper_cell),\n", - " followed by a fully connected layer with ReLU activation.\n", - " \n", - " Args:\n", - " inputs: A tuple (chars, length)\n", - " colorbot: An object of type RnnColorbot\n", - " batch_size: Int, the batch size to use\n", - " training: Boolean, whether the layer is used for training\n", - " \n", - " Returns:\n", - " A Tensor of shape (batch_size, 3) - the model predictions.\n", - " \"\"\"\n", - " (chars, length) = inputs\n", - " seq = tf.transpose(chars, [1, 0, 2])\n", - " seq.set_shape((None, batch_size, 256))\n", + " Returns:\n", + " A Tensor of shape (batch_size, 3) - the model predictions.\n", + " \"\"\"\n", + " chars, length = inputs\n", + " batch_size = chars.shape[0]\n", + " seq = tf.transpose(chars, (1, 0, 2))\n", "\n", - " seq = rnn_layer(seq, colorbot.lower_cell, batch_size, training)\n", - " seq = rnn_layer(seq, colorbot.upper_cell, batch_size, training)\n", + " seq = self._rnn_layer(seq, self.lower_cell, batch_size, training)\n", + " seq = self._rnn_layer(seq, self.upper_cell, batch_size, training)\n", "\n", - " # Grab just the end-of-sequence from each output.\n", - " indices = tf.stack([length - 1, range(batch_size)], axis=1)\n", - " sequence_ends = tf.gather_nd(seq, indices)\n", - " return colorbot.relu_layer(sequence_ends)\n", + " # Grab just the end-of-sequence from each output.\n", + " indices = tf.stack([length - 1, range(batch_size)], axis=1)\n", + " sequence_ends = tf.gather_nd(seq, indices)\n", + " return self.relu_layer(sequence_ends)\n", "\n", "@autograph.convert()\n", "def loss_fn(labels, predictions):\n", @@ -246,9 +262,9 @@ } }, "source": [ - "We will now create the model function for the estimator.\n", + "We will now create the model function for the custom Estimator.\n", "\n", - "In the model function, we simply call the converted functions that we defined above - that's it!" + "In the model function, we simply use the model class we defined above - that's it!" ] }, { @@ -275,14 +291,12 @@ " sequence_length = features['sequence_length']\n", " inputs = (chars, sequence_length)\n", "\n", - " # Create the model components.\n", - " # Simply calling the AutoGraph-ed functions and objects just works!\n", + " # Create the model. Simply using the AutoGraph-ed class just works!\n", " colorbot = RnnColorbot()\n", - " \n", - " batch_size = params['batch_size']\n", + " colorbot.build(None)\n", "\n", " if mode == tf.estimator.ModeKeys.TRAIN:\n", - " predictions = model(inputs, colorbot, batch_size, training=True)\n", + " predictions = colorbot(inputs, training=True)\n", " loss = loss_fn(labels, predictions)\n", "\n", " learning_rate = params['learning_rate']\n", @@ -292,14 +306,13 @@ " return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)\n", "\n", " elif mode == tf.estimator.ModeKeys.EVAL:\n", - " predictions = model(inputs, colorbot, batch_size, training=False)\n", + " predictions = colorbot(inputs)\n", " loss = loss_fn(labels, predictions)\n", "\n", " return tf.estimator.EstimatorSpec(mode, loss=loss)\n", - " \n", + "\n", " elif mode == tf.estimator.ModeKeys.PREDICT:\n", - " # For prediction, we expect single tensors.\n", - " predictions = model(inputs, colorbot, 1, training=False)\n", + " predictions = colorbot(inputs)\n", "\n", " predictions = tf.minimum(predictions, 1.0)\n", " return tf.estimator.EstimatorSpec(mode, predictions=predictions)" @@ -368,7 +381,7 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": 7, "metadata": { "colab": { "autoexec": { @@ -379,9 +392,9 @@ }, "colab_type": "code", "executionInfo": { - "elapsed": 10064, + "elapsed": 10604, "status": "ok", - "timestamp": 1523580419240, + "timestamp": 1524095272039, "user": { "displayName": "", "photoUrl": "", @@ -390,7 +403,7 @@ "user_tz": 240 }, "id": "2pg1AfbxBJQq", - "outputId": "41894b16-3d3a-4e30-f6e4-5a9c837a2210", + "outputId": "9c924b4f-06e1-4538-976c-a3e1ddac5660", "slideshow": { "slide_type": "-" } @@ -400,7 +413,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Eval loss at step 100: 0.0665446\n" + "Eval loss at step 100: 0.0674834\n" ] } ], @@ -444,7 +457,7 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": 8, "metadata": { "colab": { "autoexec": { @@ -455,9 +468,9 @@ }, "colab_type": "code", "executionInfo": { - "elapsed": 31286, + "elapsed": 7990, "status": "ok", - "timestamp": 1523580450579, + "timestamp": 1524095280105, "user": { "displayName": "", "photoUrl": "", @@ -466,7 +479,7 @@ "user_tz": 240 }, "id": "dxHex2tUN_10", - "outputId": "b3dc558d-b800-4e9b-e60e-3441124e80d8", + "outputId": "2b889e5a-b9ed-4645-bf03-d98f26c72101", "slideshow": { "slide_type": "slide" } @@ -478,7 +491,7 @@ "\u003clink rel=stylesheet type=text/css href='/nbextensions/google.colab/tabbar.css'\u003e\u003c/link\u003e" ], "text/plain": [ - "\u003cIPython.core.display.HTML at 0x7f4112527e90\u003e" + "\u003cIPython.core.display.HTML at 0x7f3f36aa6cd0\u003e" ] }, "metadata": { @@ -494,7 +507,7 @@ "\u003cscript src='/nbextensions/google.colab/tabbar_main.min.js'\u003e\u003c/script\u003e" ], "text/plain": [ - "\u003cIPython.core.display.HTML at 0x7f4112527f10\u003e" + "\u003cIPython.core.display.HTML at 0x7f3eca67f7d0\u003e" ] }, "metadata": { @@ -510,7 +523,7 @@ "\u003cdiv id=\"id1\"\u003e\u003c/div\u003e" ], "text/plain": [ - "\u003cIPython.core.display.HTML at 0x7f4112527f50\u003e" + "\u003cIPython.core.display.HTML at 0x7f3eca67f8d0\u003e" ] }, "metadata": { @@ -523,11 +536,11 @@ { "data": { "application/javascript": [ - "window[\"2c60f474-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = colab_lib.createTabBar({\"initialSelection\": 0, \"location\": \"top\", \"contentHeight\": [\"initial\"], \"borderColor\": [\"#a7a7a7\"], \"contentBorder\": [\"0px\"], \"tabNames\": [\"RNN Colorbot\"], \"elementId\": \"id1\"});\n", - "//# sourceURL=js_a0db480422" + "window[\"e8ddfa22-4362-11e8-91ec-c8d3ffb5fbe0\"] = colab_lib.createTabBar({\"contentBorder\": [\"0px\"], \"elementId\": \"id1\", \"borderColor\": [\"#a7a7a7\"], \"contentHeight\": [\"initial\"], \"tabNames\": [\"RNN Colorbot\"], \"location\": \"top\", \"initialSelection\": 0});\n", + "//# sourceURL=js_71b9087b6d" ], "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f410f8fd1d0\u003e" + "\u003cIPython.core.display.Javascript at 0x7f3eca67f950\u003e" ] }, "metadata": { @@ -540,11 +553,11 @@ { "data": { "application/javascript": [ - "window[\"2c60f475-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", - "//# sourceURL=js_d2a46ea291" + "window[\"e8ddfa23-4362-11e8-91ec-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", + "//# sourceURL=js_e390445f33" ], "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f410f8fd0d0\u003e" + "\u003cIPython.core.display.Javascript at 0x7f3eca67f990\u003e" ] }, "metadata": { @@ -557,11 +570,11 @@ { "data": { "application/javascript": [ - "window[\"2c60f476-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", - "//# sourceURL=js_0a8262c6e9" + "window[\"e8ddfa24-4362-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", + "//# sourceURL=js_241dd76d85" ], "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f410f8fd390\u003e" + "\u003cIPython.core.display.Javascript at 0x7f3eca67fc50\u003e" ] }, "metadata": { @@ -575,11 +588,11 @@ { "data": { "application/javascript": [ - "window[\"2c60f477-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = document.querySelector(\"#id1_content_0\");\n", - "//# sourceURL=js_e32f85ccd2" + "window[\"e8ddfa25-4362-11e8-91ec-c8d3ffb5fbe0\"] = document.querySelector(\"#id1_content_0\");\n", + "//# sourceURL=js_60c64e3d50" ], "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f410f8fd490\u003e" + "\u003cIPython.core.display.Javascript at 0x7f3eca67fd90\u003e" ] }, "metadata": { @@ -593,11 +606,11 @@ { "data": { "application/javascript": [ - "window[\"2c60f478-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"2c60f477-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_eaee748b21" + "window[\"e8ddfa26-4362-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"e8ddfa25-4362-11e8-91ec-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_14ea437cbd" ], "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f410f8fd550\u003e" + "\u003cIPython.core.display.Javascript at 0x7f3eca67fe10\u003e" ] }, "metadata": { @@ -611,11 +624,11 @@ { "data": { "application/javascript": [ - "window[\"2c60f479-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", - "//# sourceURL=js_2befe06587" + "window[\"e8ddfa27-4362-11e8-91ec-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", + "//# sourceURL=js_09294c2226" ], "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f4112527f10\u003e" + "\u003cIPython.core.display.Javascript at 0x7f3eca67fcd0\u003e" ] }, "metadata": { @@ -629,11 +642,11 @@ { "data": { "application/javascript": [ - "window[\"354d7b1a-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"2c60f476-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_8ec4aeeb25" + "window[\"ec965514-4362-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"e8ddfa24-4362-11e8-91ec-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_e5e8266997" ], "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f410f8fd690\u003e" + "\u003cIPython.core.display.Javascript at 0x7f3eca67fe10\u003e" ] }, "metadata": { @@ -647,11 +660,11 @@ { "data": { "application/javascript": [ - "window[\"354d7b1b-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", - "//# sourceURL=js_9f9f4574f1" + "window[\"ec965515-4362-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", + "//# sourceURL=js_07a097f0ee" ], "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f410f8fd350\u003e" + "\u003cIPython.core.display.Javascript at 0x7f3eca67fc90\u003e" ] }, "metadata": { @@ -665,11 +678,11 @@ { "data": { "application/javascript": [ - "window[\"354d7b1c-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = document.querySelector(\"#id1_content_0\");\n", - "//# sourceURL=js_bcccd8f300" + "window[\"ec965516-4362-11e8-91ec-c8d3ffb5fbe0\"] = document.querySelector(\"#id1_content_0\");\n", + "//# sourceURL=js_790d669ca8" ], "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f410f8fd6d0\u003e" + "\u003cIPython.core.display.Javascript at 0x7f3eca67f8d0\u003e" ] }, "metadata": { @@ -683,11 +696,11 @@ { "data": { "application/javascript": [ - "window[\"354d7b1d-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"354d7b1c-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_2c056cee72" + "window[\"ec965517-4362-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"ec965516-4362-11e8-91ec-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_d30df771f0" ], "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f410f8fd490\u003e" + "\u003cIPython.core.display.Javascript at 0x7f3eca67fd90\u003e" ] }, "metadata": { @@ -701,11 +714,11 @@ { "data": { "application/javascript": [ - "window[\"354d7b1e-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", - "//# sourceURL=js_c853c3f58b" + "window[\"ec965518-4362-11e8-91ec-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", + "//# sourceURL=js_8a43a2da4b" ], "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f410f8fd610\u003e" + "\u003cIPython.core.display.Javascript at 0x7f3eca67fc50\u003e" ] }, "metadata": { @@ -718,369 +731,9 @@ }, { "data": { - "application/javascript": [ - "window[\"354d7b1f-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"354d7b1b-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_e5730ab00d" - ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQwAAAENCAYAAAD60Fs2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAACMBJREFUeJzt3F+I1XX+x/G32zjiFERUpgaFd2JBzOg5joX4h0SiMgmM\n/uhVGIlgFBlERGB3hUEkhkRdtDfRP1ACL6KpLBqcguxCjEAkmGamQcSohFHzsxe7O6zssvsydtff\n+ns8rs758j3f8z7fiyef7/k3o7XWCiDwh4s9APC/QzCAmGAAMcEAYoIBxAQDiAkGF8XTTz9d3W63\n7rvvvhoZGakVK1Zc7JEICMYlbvXq1TU8PHyxxzjPV199VcPDw/XZZ5/V22+/XVVVM2bMuMhTkRAM\n/qt+++23+uGHH+r666+vWbNmXexxuECCcQl76qmnanx8vLZs2VIDAwP1+uuv1zfffFP3339/dTqd\nWr9+fY2MjEzvv2nTpnr55ZfrgQceqIGBgXr44Yfr5MmTVVV1+vTp2r59ey1durQ6nU5t2LChTpw4\nUVVVk5OTtWXLllq6dGmtXbu23nnnnelj7tq1q7Zt21bbt2+vJUuW1HvvvVfPPvtsHTp0qAYGBmrX\nrl1/N/fRo0dr06ZN1el06u67766hoaGqqhodHa1OpzO93zPPPFO33nrr9P3t27fXm2+++e89iZyv\ncUlbtWpVGx4ebq21NjEx0brdbjtw4EBrrbUvvviidbvdduLEidZaaxs3bmxr1qxp33//fZuammob\nN25sO3fubK219tZbb7VHH320TU1NtXPnzrXDhw+3X375pbXW2kMPPdR27NjRTp8+3Y4cOdIGBwen\nn/OVV15pN910U/voo49aa61NTU21999/vz344IPTMx48eLCtWLGitdbamTNn2po1a9qePXvamTNn\n2vDwcOvv72/Hjh2bfj2HDx9urbW2du3advvtt7ejR4+21lpbuXJlO3LkyH/qVNJas8L4f6D95edC\n+/btq5UrV9by5curqmrZsmV1880316effjq977333ls33HBD9fb21h133FFHjhypqqqenp46efJk\nHTt2rGbMmFGLFi2qyy+/vCYmJurrr7+uJ598smbOnFkLFy6sDRs21N69e6eP2d/fX6tXr66qqt7e\n3n8666FDh+rUqVP1yCOPVE9PTw0ODtaqVavqgw8+qKqqJUuW1MjISB0/fryqqtauXVtffvlljY6O\n1q+//loLFy78N501/pGeiz0A/z1jY2O1f//++vjjj6vqzyE5e/ZsLVu2bHqfa665Zvr27Nmz69Sp\nU1VVdc8999TExEQ98cQT9fPPP9e6devq8ccfr8nJybryyitr9uzZ04+bP39+HT58ePr+3Llz4xkn\nJydr3rx5522bP39+TU5OVlVVp9OpoaGhuu6666rb7Va32629e/dWb29vLV68+ALOBr+HYFzi/vbT\nh3nz5tX69etrx44dF3ycnp6e2rp1a23durXGxsZq8+bNtWDBgrrtttvqp59+qlOnTlVfX19VVY2P\nj9ecOXP+4Qz/ypw5c2p8fPy8bWNjY7VgwYKqqup2u/Xiiy/WvHnzqtPp1MDAQD333HPV29tb3W73\ngl8XF8YlySXu2muvrdHR0aqqWrduXQ0NDdXnn39e586dq6mpqRoZGakff/zxXx7n4MGD9d1339W5\nc+eqr6+venp66rLLLqu5c+dWf39/vfTSS3X69On69ttv6913361169b9rnlvueWW6uvrq9dee63O\nnj1bBw8erE8++aTuvPPOqqq68cYba9asWbVv377qdDp1xRVX1NVXX10ffvjheW+I8p8hGJe4zZs3\n1+7du6vb7db+/ftr9+7dtWfPnlq2bFmtWrWq3njjjen3OP7ZSuD48eO1bdu2Wrx4cd111121dOnS\n6Sjs3LmzRkdHa/ny5bVt27Z67LHHzrvMuRAzZ86sV199tQ4cOFCDg4P1/PPP1wsvvDC9wqj68yrj\nqquumr7U+WsoFi1a9Luek9yM1vyBDpCxwgBiggHEBAOICQYQ+z/7PYzjf/QRGVxM12z68u+2WWEA\nMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHE\nBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhAT\nDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEww\ngJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEA\nYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOI\nCQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAm\nGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhg\nADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIB\nxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOICQYQ\nEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBM\nMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHB\nAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQD\niAkGEBMMIDajtdYu9hDA/wYrDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEA\nYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4j9CY2LTAbbRbWuAAAAAElFTkSuQmCC\n", "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f41127a2050\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"354d7b20-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", - "//# sourceURL=js_a897ef7e24" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f41127a2250\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"354d7b21-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = document.querySelector(\"#id1_content_0\");\n", - "//# sourceURL=js_565fa3d154" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f4113124d90\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"354d7b22-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"354d7b21-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_222e0dc6af" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f4113124c10\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"354d7b23-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", - "//# sourceURL=js_831db7458f" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f4113124310\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"3803fab4-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"354d7b20-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_adb576c6eb" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f410f990850\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"3803fab5-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", - "//# sourceURL=js_9418f2d32f" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f410f990850\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"3803fab6-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = document.querySelector(\"#id1_content_0\");\n", - "//# sourceURL=js_3fad25f306" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f4112527ed0\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"3803fab7-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"3803fab6-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_45b9340e7b" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f410f990c90\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"3803fab8-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", - "//# sourceURL=js_bec9896d44" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f410f990a10\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"3803fab9-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"3803fab5-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_460b91ad4a" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f41b21d3a10\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"3803faba-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", - "//# sourceURL=js_7dedd0b037" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f41b21d3890\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"3803fabb-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = document.querySelector(\"#id1_content_0\");\n", - "//# sourceURL=js_4b1c977dc7" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f41b21d3bd0\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"3803fabc-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"3803fabb-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_d64fedfcf9" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f41b21d3410\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"3803fabd-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", - "//# sourceURL=js_3e8c929c3f" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f41b21d3c50\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"3b9b986c-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"3803faba-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_9f9cf2b76f" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f410f8fd590\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"3b9b986d-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", - "//# sourceURL=js_b402e6b587" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f41b21d3d90\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"3b9b986e-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = document.querySelector(\"#id1_content_0\");\n", - "//# sourceURL=js_9b7d66db72" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f41b21d3b10\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"3b9b986f-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"3b9b986e-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_11ec213a3f" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f41b21d3950\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "window[\"3b9b9870-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", - "//# sourceURL=js_9c055e4bc0" - ], - "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f41b21d3850\u003e" - ] - }, - "metadata": { - "tags": [ - "id1_content_0", - "outputarea_id1" - ] - }, - "output_type": "display_data" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQwAAAENCAYAAAD60Fs2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAACMRJREFUeJzt3F+IlfW+x/Gvp3FECyIqU4PCO7EgZnQtnUJ0JJGoTDoY\n/dGrMBJhosggIgK7KwwiMdxRF11F/0AJvIisLBqcguxCjEAkmNQGcRvVwIzm71zsc4Yje7P3x9h7\nz97u1+tqrYdnPeu7nos3v2f9m9FaawUQ+K/pHgD49yEYQEwwgJhgADHBAGKCAcQEg2nx9NNPV7fb\nrfvuu69GRkZq5cqV0z0SAcG4xK1evbqGh4ene4wLfPXVVzU8PFyfffZZvf3221VVNWPGjGmeioRg\n8E/122+/1Q8//FDXX399zZo1a7rH4SIJxiXsqaeeqhMnTtSWLVuqv7+/Xn/99frmm2/q/vvvr06n\nU+vXr6+RkZGp/Tdt2lQvv/xyPfDAA9Xf318PP/xwnTlzpqqqJicna9u2bbVs2bLqdDq1YcOGOn36\ndFVVjY2N1ZYtW2rZsmW1du3aeuedd6aOuXPnzhoaGqpt27bV0qVL67333qtnn322Dh06VP39/bVz\n584/m/vo0aO1adOm6nQ6dffdd9f+/furqmp0dLQ6nc7Ufs8880zdeuutU/e3bdtWb7755t/3JHKh\nxiVtcHCwDQ8Pt9ZaO3nyZOt2u+3AgQOttda++OKL1u122+nTp1trrW3cuLGtWbOmff/9921iYqJt\n3Lix7dixo7XW2ltvvdUeffTRNjEx0c6fP98OHz7cfvnll9Zaaw899FDbvn17m5ycbEeOHGnLly+f\nes5XXnml3XTTTe2jjz5qrbU2MTHR3n///fbggw9OzXjw4MG2cuXK1lprZ8+ebWvWrGm7d+9uZ8+e\nbcPDw62vr68dO3Zs6vUcPny4tdba2rVr2+23396OHj3aWmtt1apV7ciRI/+oU0lrzQrjP0D7358L\n7d27t1atWlUrVqyoqqqBgYG6+eab69NPP53a9957760bbrihent764477qgjR45UVVVPT0+dOXOm\njh07VjNmzKjFixfX5ZdfXidPnqyvv/66nnzyyZo5c2YtWrSoNmzYUHv27Jk6Zl9fX61evbqqqnp7\ne//qrIcOHarx8fF65JFHqqenp5YvX16Dg4P1wQcfVFXV0qVLa2RkpE6dOlVVVWvXrq0vv/yyRkdH\n69dff61Fixb9nc4af0nPdA/AP8/x48dr37599fHHH1fVn0Jy7ty5GhgYmNrnmmuumbo9e/bsGh8f\nr6qqe+65p06ePFlPPPFE/fzzz7Vu3bp6/PHHa2xsrK688sqaPXv21OMWLFhQhw8fnro/b968eMax\nsbGaP3/+BdsWLFhQY2NjVVXV6XRq//79dd1111W3261ut1t79uyp3t7eWrJkyUWcDX4PwbjE/f9P\nH+bPn1/r16+v7du3X/Rxenp6auvWrbV169Y6fvx4bd68uRYuXFi33XZb/fTTTzU+Pl5z5sypqqoT\nJ07U3Llz/+IMf8vcuXPrxIkTF2w7fvx4LVy4sKqqut1uvfjiizV//vzqdDrV399fzz33XPX29la3\n273o18XFcUlyibv22mtrdHS0qqrWrVtX+/fvr88//7zOnz9fExMTNTIyUj/++OPfPM7Bgwfru+++\nq/Pnz9ecOXOqp6enLrvsspo3b1719fXVSy+9VJOTk/Xtt9/Wu+++W+vWrftd895yyy01Z86ceu21\n1+rcuXN18ODB+uSTT+rOO++sqqobb7yxZs2aVXv37q1Op1NXXHFFXX311fXhhx9e8IYo/xiCcYnb\nvHlz7dq1q7rdbu3bt6927dpVu3fvroGBgRocHKw33nhj6j2Ov7YSOHXqVA0NDdWSJUvqrrvuqmXL\nlk1FYceOHTU6OlorVqyooaGheuyxxy64zLkYM2fOrFdffbUOHDhQy5cvr+eff75eeOGFqRVG1Z9W\nGVddddXUpc7/hWLx4sW/6znJzWjNH+gAGSsMICYYQEwwgJhgALF/2e9h/PEP/z3dI8B/tKseee/P\ntllhADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEA\nYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOI\nCQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAm\nGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhg\nADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIB\nxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOICQYQ\nEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBM\nMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHB\nAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQD\niAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwg\nJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICY\nYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKC\nAcQEA4gJBhATDCA2o7XWpnsI4N+DFQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEww\ngJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHE/gfh60wGjfc7LQAAAABJRU5ErkJg\ngg==\n", - "text/plain": [ - "\u003cmatplotlib.figure.Figure at 0x7f4113124310\u003e" + "\u003cmatplotlib.figure.Figure at 0x7f3ecc00bf10\u003e" ] }, "metadata": { @@ -1095,11 +748,11 @@ { "data": { "application/javascript": [ - "window[\"3b9b9871-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"3b9b986d-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_ba6a061307" + "window[\"ec965519-4362-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"ec965515-4362-11e8-91ec-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_893ad561f4" ], "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f410f8fd890\u003e" + "\u003cIPython.core.display.Javascript at 0x7f3f31b55c90\u003e" ] }, "metadata": { @@ -1113,11 +766,11 @@ { "data": { "application/javascript": [ - "window[\"3b9b9872-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", - "//# sourceURL=js_83e3496927" + "window[\"ec96551a-4362-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", + "//# sourceURL=js_2d99e0ac17" ], "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f410f8fd590\u003e" + "\u003cIPython.core.display.Javascript at 0x7f3eca67fe50\u003e" ] }, "metadata": { @@ -1131,11 +784,11 @@ { "data": { "application/javascript": [ - "window[\"3b9b9873-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = document.querySelector(\"#id1_content_0\");\n", - "//# sourceURL=js_f437bab20d" + "window[\"ec96551b-4362-11e8-91ec-c8d3ffb5fbe0\"] = document.querySelector(\"#id1_content_0\");\n", + "//# sourceURL=js_5c19462e32" ], "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f41127a22d0\u003e" + "\u003cIPython.core.display.Javascript at 0x7f3f31b55dd0\u003e" ] }, "metadata": { @@ -1149,11 +802,11 @@ { "data": { "application/javascript": [ - "window[\"3b9b9874-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"3b9b9873-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_93aa63450e" + "window[\"ec96551c-4362-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"ec96551b-4362-11e8-91ec-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_b9c8b7567b" ], "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f41127a2b90\u003e" + "\u003cIPython.core.display.Javascript at 0x7f3f31b55a50\u003e" ] }, "metadata": { @@ -1167,11 +820,11 @@ { "data": { "application/javascript": [ - "window[\"3b9b9875-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", - "//# sourceURL=js_aca189bea5" + "window[\"ec96551d-4362-11e8-91ec-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", + "//# sourceURL=js_fd05186348" ], "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f410f8fd4d0\u003e" + "\u003cIPython.core.display.Javascript at 0x7f3f31b55810\u003e" ] }, "metadata": { @@ -1185,10 +838,10 @@ { "data": { "text/html": [ - "\u003cdiv class=id_100313201 style=\"margin-right:10px; display:flex;align-items:center;\"\u003e\u003cspan style=\"margin-right: 3px;\"\u003e\u003c/span\u003e\u003c/div\u003e" + "\u003cdiv class=id_888646481 style=\"margin-right:10px; display:flex;align-items:center;\"\u003e\u003cspan style=\"margin-right: 3px;\"\u003e\u003c/span\u003e\u003c/div\u003e" ], "text/plain": [ - "\u003cIPython.core.display.HTML at 0x7f410f990a90\u003e" + "\u003cIPython.core.display.HTML at 0x7f3f32414810\u003e" ] }, "metadata": { @@ -1203,11 +856,11 @@ { "data": { "application/javascript": [ - "window[\"3b9b9876-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = jQuery(\".id_100313201 span\");\n", - "//# sourceURL=js_5df1fe383e" + "window[\"ec96551e-4362-11e8-91ec-c8d3ffb5fbe0\"] = jQuery(\".id_888646481 span\");\n", + "//# sourceURL=js_efef96e882" ], "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f410f8fd490\u003e" + "\u003cIPython.core.display.Javascript at 0x7f3f31b55710\u003e" ] }, "metadata": { @@ -1222,11 +875,11 @@ { "data": { "application/javascript": [ - "window[\"3b9b9877-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = window[\"3b9b9876-3eb4-11e8-91ec-c8d3ffb5fbe0\"].text(\"Give me a color name (or press 'enter' to exit): \");\n", - "//# sourceURL=js_c62c7174ad" + "window[\"ec96551f-4362-11e8-91ec-c8d3ffb5fbe0\"] = window[\"ec96551e-4362-11e8-91ec-c8d3ffb5fbe0\"].text(\"Give me a color name (or press 'enter' to exit): \");\n", + "//# sourceURL=js_6eca889864" ], "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f41127a2390\u003e" + "\u003cIPython.core.display.Javascript at 0x7f3eca67f990\u003e" ] }, "metadata": { @@ -1241,11 +894,11 @@ { "data": { "application/javascript": [ - "window[\"3ed76584-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = jQuery(\".id_100313201 input\");\n", - "//# sourceURL=js_2e2201ddc4" + "window[\"ed8ea972-4362-11e8-91ec-c8d3ffb5fbe0\"] = jQuery(\".id_888646481 input\");\n", + "//# sourceURL=js_f02070cc60" ], "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f41127a2810\u003e" + "\u003cIPython.core.display.Javascript at 0x7f3f31b553d0\u003e" ] }, "metadata": { @@ -1260,11 +913,11 @@ { "data": { "application/javascript": [ - "window[\"3ed76585-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = window[\"3ed76584-3eb4-11e8-91ec-c8d3ffb5fbe0\"].remove();\n", - "//# sourceURL=js_288e5283d6" + "window[\"ed8ea973-4362-11e8-91ec-c8d3ffb5fbe0\"] = window[\"ed8ea972-4362-11e8-91ec-c8d3ffb5fbe0\"].remove();\n", + "//# sourceURL=js_ed9faba660" ], "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f41127a26d0\u003e" + "\u003cIPython.core.display.Javascript at 0x7f3f31a95450\u003e" ] }, "metadata": { @@ -1279,11 +932,11 @@ { "data": { "application/javascript": [ - "window[\"3ed76586-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = jQuery(\".id_100313201 span\");\n", - "//# sourceURL=js_2f31d19cde" + "window[\"ed8ea974-4362-11e8-91ec-c8d3ffb5fbe0\"] = jQuery(\".id_888646481 span\");\n", + "//# sourceURL=js_f3458d7074" ], "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f41127a2fd0\u003e" + "\u003cIPython.core.display.Javascript at 0x7f3f31a95250\u003e" ] }, "metadata": { @@ -1298,11 +951,11 @@ { "data": { "application/javascript": [ - "window[\"3ed76587-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = window[\"3ed76586-3eb4-11e8-91ec-c8d3ffb5fbe0\"].text(\"Give me a color name (or press 'enter' to exit): \");\n", - "//# sourceURL=js_2fbbcda050" + "window[\"ed8ea975-4362-11e8-91ec-c8d3ffb5fbe0\"] = window[\"ed8ea974-4362-11e8-91ec-c8d3ffb5fbe0\"].text(\"Give me a color name (or press 'enter' to exit): \");\n", + "//# sourceURL=js_3ffd97bd6f" ], "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f4112527e90\u003e" + "\u003cIPython.core.display.Javascript at 0x7f3f31a953d0\u003e" ] }, "metadata": { @@ -1317,11 +970,11 @@ { "data": { "application/javascript": [ - "window[\"3ed76588-3eb4-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"3b9b9872-3eb4-11e8-91ec-c8d3ffb5fbe0\"]);\n", - "//# sourceURL=js_f94d975cf3" + "window[\"ed8ea976-4362-11e8-91ec-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"ec96551a-4362-11e8-91ec-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_7f73e8bcca" ], "text/plain": [ - "\u003cIPython.core.display.Javascript at 0x7f41127a2fd0\u003e" + "\u003cIPython.core.display.Javascript at 0x7f3f31b55710\u003e" ] }, "metadata": { @@ -1337,7 +990,7 @@ "def predict_input_fn(color_name):\n", " \"\"\"An input function for prediction.\"\"\"\n", " _, chars, sequence_length = parse(color_name)\n", - " \n", + "\n", " # We create a batch of a single element.\n", " features = {\n", " 'chars': tf.expand_dims(chars, 0),\n", @@ -1385,7 +1038,11 @@ "colab": { "collapsed_sections": [], "default_view": {}, - "name": "RNN Colorbot using Estimators", + "last_runtime": { + "build_target": "", + "kind": "local" + }, + "name": "RNN Colorbot using Keras and Estimators", "provenance": [ { "file_id": "1CtzefX39ffFibX_BqE6cRbT0UW_DdVKl", -- GitLab From b4c37a452d2ed1d1c29ceb70127c4ef6434c44ca Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 07:13:03 -0700 Subject: [PATCH 643/791] Teach the conditinal simplifier about sharding. PiperOrigin-RevId: 193510638 --- tensorflow/compiler/xla/service/conditional_simplifier.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/compiler/xla/service/conditional_simplifier.cc b/tensorflow/compiler/xla/service/conditional_simplifier.cc index f35de08085..e560abc87f 100644 --- a/tensorflow/compiler/xla/service/conditional_simplifier.cc +++ b/tensorflow/compiler/xla/service/conditional_simplifier.cc @@ -69,7 +69,7 @@ static StatusOr TryRemoveConditional(HloInstruction* conditional) { conditional->shape(), {conditional->mutable_operand(2)}, conditional->false_computation())); } - + conditional->SetupDerivedInstruction(call_op); TF_RETURN_IF_ERROR(computation->ReplaceInstruction(conditional, call_op)); TF_RETURN_IF_ERROR(CallInliner::Inline(call_op).status()); -- GitLab From 1a2eb108a3e513a4f4609b9d421277bc222e5eb0 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 19 Apr 2018 15:03:05 +0000 Subject: [PATCH 644/791] Update docs for tf.unstack with respect to numpy. In 18692 an issue was raised over whether tf.unstack is compatible with numpy.unstack (specified in current docs) or numpy.split. It looks like there is no numpy.unstack. And for numpy.split, it is not compatible with tf.unstack. The tf.split is very close to numpy.split. However, the second arg `num_or_size_splits` in `tf.split` requires the number of the splits, while the second arg `indices_or_sections` in `numpy.split` requires the index of the splits. For that reason the tf.split is not compatible with numpy.split as well. According to the above this fix simply removes `The numpy equivalent` part in the docs of tf.unstack. This fix fixes 18692. Signed-off-by: Yong Tang --- tensorflow/python/ops/array_ops.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index ceeabe090d..23202ae28e 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -1057,9 +1057,7 @@ def unstack(value, num=None, axis=0, name="unstack"): `value[:, i, :, :]` and each tensor in `output` will have shape `(A, C, D)`. Etc. - This is the opposite of stack. The numpy equivalent is - - tf.unstack(x, n) = np.unstack(x) + This is the opposite of stack. Args: value: A rank `R > 0` `Tensor` to be unstacked. -- GitLab From 50f6683ca50e6d4e7008d6d1b437b407d6a62e92 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 19 Apr 2018 09:13:21 -0700 Subject: [PATCH 645/791] Add shape check for batch related Dataset ops (#18683) * Add shape check for PrefetchDataset Signed-off-by: Yong Tang * Add BatchDataset shape check Signed-off-by: Yong Tang * Add shape check for SlideDataset Signed-off-by: Yong Tang * Add shape check for DenseToSparseBatchDataset Signed-off-by: Yong Tang * Sanitize with clang-format -i --style=Google Signed-off-by: Yong Tang --- tensorflow/core/ops/dataset_ops.cc | 31 ++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc index 34f2c612ec..c63e485f6c 100644 --- a/tensorflow/core/ops/dataset_ops.cc +++ b/tensorflow/core/ops/dataset_ops.cc @@ -199,7 +199,12 @@ REGISTER_OP("PrefetchDataset") .Output("handle: variant") .Attr("output_types: list(type) >= 1") .Attr("output_shapes: list(shape) >= 1") - .SetShapeFn(shape_inference::ScalarShape); + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle unused; + // buffer_size should be a scalar. + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); + return shape_inference::ScalarShape(c); + }); REGISTER_OP("ScanDataset") .Input("input_dataset: variant") @@ -283,7 +288,12 @@ REGISTER_OP("BatchDataset") .Output("handle: variant") .Attr("output_types: list(type) >= 1") .Attr("output_shapes: list(shape) >= 1") - .SetShapeFn(shape_inference::ScalarShape); + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle unused; + // batch_size should be a scalar. + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); + return shape_inference::ScalarShape(c); + }); // TODO(mrry): move SlideDataset to contrib in the future. REGISTER_OP("SlideDataset") @@ -293,7 +303,13 @@ REGISTER_OP("SlideDataset") .Output("handle: variant") .Attr("output_types: list(type) >= 1") .Attr("output_shapes: list(shape) >= 1") - .SetShapeFn(shape_inference::ScalarShape); + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle unused; + // window_size and stride should be scalars. + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused)); + return shape_inference::ScalarShape(c); + }); REGISTER_OP("PaddedBatchDataset") .Input("input_dataset: variant") @@ -323,7 +339,14 @@ REGISTER_OP("DenseToSparseBatchDataset") .Output("handle: variant") .Attr("output_types: list(type) >= 1") .Attr("output_shapes: list(shape) >= 1") - .SetShapeFn(shape_inference::ScalarShape); + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle unused; + // batch_size should be a scalar. + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); + // row_shape should be a 1-D vector. + TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 1, &unused)); + return shape_inference::ScalarShape(c); + }); REGISTER_OP("RangeDataset") .Input("start: int64") -- GitLab From b71b6b8ca9ade8b39d77f0373210fe58dfccf4f4 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 19 Apr 2018 09:13:35 -0700 Subject: [PATCH 646/791] Shape validation with random/shuffle related Dataset ops (#18682) * Add shape check for CacheDataset Signed-off-by: Yong Tang * Add shape check for ShuffleAndRepeatDataset Signed-off-by: Yong Tang * Add check for ShuffleDataset Signed-off-by: Yong Tang * Add shape check for RandomDataset Signed-off-by: Yong Tang * Add RangeDataset shape check Signed-off-by: Yong Tang * Sanitize with clang-format -i --style=Google Signed-off-by: Yong Tang --- tensorflow/core/ops/dataset_ops.cc | 43 ++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc index c63e485f6c..dae0c0eae4 100644 --- a/tensorflow/core/ops/dataset_ops.cc +++ b/tensorflow/core/ops/dataset_ops.cc @@ -357,7 +357,14 @@ REGISTER_OP("RangeDataset") .Attr("output_shapes: list(shape) >= 1") .SetIsStateful() // TODO(b/65524810): Source dataset ops must be marked // stateful to inhibit constant folding. - .SetShapeFn(shape_inference::ScalarShape); + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle unused; + // start, stop, and step should be scalars. + TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused)); + return shape_inference::ScalarShape(c); + }); REGISTER_OP("RandomDataset") .Input("seed: int64") @@ -367,7 +374,13 @@ REGISTER_OP("RandomDataset") .Attr("output_shapes: list(shape) >= 1") .SetIsStateful() // TODO(b/65524810): Source dataset ops must be marked // stateful to inhibit constant folding. - .SetShapeFn(shape_inference::ScalarShape); + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle unused; + // buffer_size, seed, and seed2 should be scalars. + TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); + return shape_inference::ScalarShape(c); + }); REGISTER_OP("ShuffleDataset") .Input("input_dataset: variant") @@ -378,7 +391,14 @@ REGISTER_OP("ShuffleDataset") .Attr("reshuffle_each_iteration: bool = true") .Attr("output_types: list(type) >= 1") .Attr("output_shapes: list(shape) >= 1") - .SetShapeFn(shape_inference::ScalarShape); + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle unused; + // buffer_size, seed, and seed2 should be scalars. + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &unused)); + return shape_inference::ScalarShape(c); + }); REGISTER_OP("ShuffleAndRepeatDataset") .Input("input_dataset: variant") @@ -389,7 +409,15 @@ REGISTER_OP("ShuffleAndRepeatDataset") .Output("handle: variant") .Attr("output_types: list(type) >= 1") .Attr("output_shapes: list(shape) >= 1") - .SetShapeFn(shape_inference::ScalarShape); + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle unused; + // buffer_size, seed, seed2, and count should be scalars. + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(4), 0, &unused)); + return shape_inference::ScalarShape(c); + }); REGISTER_OP("CacheDataset") .Input("input_dataset: variant") @@ -397,7 +425,12 @@ REGISTER_OP("CacheDataset") .Output("handle: variant") .Attr("output_types: list(type) >= 1") .Attr("output_shapes: list(shape) >= 1") - .SetShapeFn(shape_inference::ScalarShape); + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle unused; + // filename should be a scalar. + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); + return shape_inference::ScalarShape(c); + }); REGISTER_OP("TextLineDataset") .Input("filenames: string") -- GitLab From 76619c8dea0e480fd48e3b4dcfe0249eb24216b8 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 19 Apr 2018 09:13:53 -0700 Subject: [PATCH 647/791] Validation in shape functions of Dataset ops (#18680) * Add shape check for PrependFromQueueAndPaddedBatchDataset Signed-off-by: Yong Tang * Add comment for shape check Signed-off-by: Yong Tang * Add shape check for FixedLengthRecordDataset Signed-off-by: Yong Tang * Add check for filenames as well Signed-off-by: Yong Tang * Clang-format -i --style=google for file format Signed-off-by: Yong Tang * Add shape check for SqlDataset Signed-off-by: Yong Tang --- tensorflow/core/ops/dataset_ops.cc | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc index dae0c0eae4..869bef8040 100644 --- a/tensorflow/core/ops/dataset_ops.cc +++ b/tensorflow/core/ops/dataset_ops.cc @@ -459,7 +459,14 @@ REGISTER_OP("SqlDataset") .Attr("output_shapes: list(shape) >= 1") .SetIsStateful() // TODO(b/65524810): Source dataset ops must be marked // stateful to inhibit constant folding. - .SetShapeFn(shape_inference::ScalarShape); + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle unused; + // driver_name, data_source_name, and query should be scalars. + TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused)); + return shape_inference::ScalarShape(c); + }); REGISTER_OP("FixedLengthRecordDataset") .Input("filenames: string") @@ -470,7 +477,18 @@ REGISTER_OP("FixedLengthRecordDataset") .Output("handle: variant") .SetIsStateful() // TODO(b/65524810): Source dataset ops must be marked // stateful to inhibit constant folding. - .SetShapeFn(shape_inference::ScalarShape); + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle unused; + // `filenames` must be a scalar or a vector. + TF_RETURN_IF_ERROR(c->WithRankAtMost(c->input(0), 1, &unused)); + // header_bytes, record_bytes, footer_bytes, buffer_size should be + // scalars. + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(4), 0, &unused)); + return shape_inference::ScalarShape(c); + }); REGISTER_OP("TFRecordDataset") .Input("filenames: string") @@ -609,7 +627,12 @@ REGISTER_OP("PrependFromQueueAndPaddedBatchDataset") // length of `output_types` is `N`, the `output_shapes` are // (as far as possible to tell statically) compatible with `padded_shapes`, // and that `padding_values` are all scalars. - .SetShapeFn(shape_inference::ScalarShape); + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle unused; + // batch_size should be a scalar. + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); + return shape_inference::ScalarShape(c); + }); REGISTER_OP("EnqueueInQueueDataset") .Input("queue: variant") -- GitLab From 7e735e5be811bacfa4e16aeae2e8aa53ef209ea6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 09:13:47 -0700 Subject: [PATCH 648/791] Pin pip to version 9.0.3 * This is because pip 10 is still unstable in some distros * reference: https://github.com/pypa/pip/issues/5240 PiperOrigin-RevId: 193525542 --- tensorflow/tools/ci_build/install/install_pip_packages.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/tools/ci_build/install/install_pip_packages.sh b/tensorflow/tools/ci_build/install/install_pip_packages.sh index fc137aeeed..9644277fab 100755 --- a/tensorflow/tools/ci_build/install/install_pip_packages.sh +++ b/tensorflow/tools/ci_build/install/install_pip_packages.sh @@ -19,11 +19,11 @@ set -e # We don't apt-get install so that we can install a newer version of pip. # Only needed for Ubuntu 14.04 ,and not needed for Ubuntu 16.04 / Debian 8,9 if $(cat /etc/*-release | grep -q 14.04); then - easy_install -U pip - easy_install3 -U pip + easy_install -U pip==9.0.3 + easy_install3 -U pip==9.0.3 else - pip2 install --upgrade pip - pip3 install --upgrade pip + pip2 install --upgrade pip==9.0.3 + pip3 install --upgrade pip==9.0.3 fi # Install pip packages from whl files to avoid the time-consuming process of -- GitLab From 51a26bb2f3e66fc79a5870f6eed88f60de995d4a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 09:23:35 -0700 Subject: [PATCH 649/791] [TF:XLA] Change HloTestBase::ExecuteNoHloPasses to return a literal directly. PiperOrigin-RevId: 193526900 --- tensorflow/compiler/xla/tests/hlo_test_base.cc | 8 +++++--- tensorflow/compiler/xla/tests/hlo_test_base.h | 2 +- tensorflow/compiler/xla/tests/tuple_test.cc | 3 +-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tensorflow/compiler/xla/tests/hlo_test_base.cc b/tensorflow/compiler/xla/tests/hlo_test_base.cc index c5afe0c3e0..9984aba089 100644 --- a/tensorflow/compiler/xla/tests/hlo_test_base.cc +++ b/tensorflow/compiler/xla/tests/hlo_test_base.cc @@ -113,11 +113,13 @@ StatusOr> HloTestBase::Execute( return test_runner_.Execute(std::move(module), arguments); } -StatusOr> HloTestBase::ExecuteNoHloPasses( +std::unique_ptr HloTestBase::ExecuteNoHloPasses( std::unique_ptr module, tensorflow::gtl::ArraySlice arguments) { - return test_runner_.Execute(std::move(module), arguments, - /*run_hlo_passes=*/false); + return test_runner_ + .Execute(std::move(module), arguments, + /*run_hlo_passes=*/false) + .ValueOrDie(); } std::unique_ptr HloTestBase::ExecuteAndTransfer( diff --git a/tensorflow/compiler/xla/tests/hlo_test_base.h b/tensorflow/compiler/xla/tests/hlo_test_base.h index 28d7ab09cb..79fcea9403 100644 --- a/tensorflow/compiler/xla/tests/hlo_test_base.h +++ b/tensorflow/compiler/xla/tests/hlo_test_base.h @@ -99,7 +99,7 @@ class HloTestBase : public ::testing::Test { // Same as above, except the module will be executed without running any HLO // passes on it. - StatusOr> ExecuteNoHloPasses( + std::unique_ptr ExecuteNoHloPasses( std::unique_ptr module, tensorflow::gtl::ArraySlice arguments); diff --git a/tensorflow/compiler/xla/tests/tuple_test.cc b/tensorflow/compiler/xla/tests/tuple_test.cc index 098be6d7aa..61d0fa02ab 100644 --- a/tensorflow/compiler/xla/tests/tuple_test.cc +++ b/tensorflow/compiler/xla/tests/tuple_test.cc @@ -535,8 +535,7 @@ TEST_F(TupleHloTest, HloRunner::CreateModuleFromString(testcase, GetDebugOptionsForTest()) .ValueOrDie(); auto param = Literal::MakeTupleOwned(Literal::CreateR1({1, 2, 3})); - TF_ASSERT_OK_AND_ASSIGN(auto result, - ExecuteNoHloPasses(std::move(module), {param.get()})); + auto result = ExecuteNoHloPasses(std::move(module), {param.get()}); EXPECT_TRUE(LiteralTestUtil::Equal( *result, *Literal::MakeTupleOwned(Literal::CreateR2({{1, 2, 3}})))); -- GitLab From 0b3950d67bcb07c11f87bd3c2da554017bff0674 Mon Sep 17 00:00:00 2001 From: imsheridan Date: Fri, 20 Apr 2018 00:35:54 +0800 Subject: [PATCH 650/791] Fix code block rendering in several api definitions --- tensorflow/core/api_def/base_api/api_def_Pad.pbtxt | 1 + tensorflow/core/api_def/base_api/api_def_QuantizeV2.pbtxt | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/tensorflow/core/api_def/base_api/api_def_Pad.pbtxt b/tensorflow/core/api_def/base_api/api_def_Pad.pbtxt index e45e2375eb..ee4aad7899 100644 --- a/tensorflow/core/api_def/base_api/api_def_Pad.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_Pad.pbtxt @@ -24,5 +24,6 @@ pad(t, paddings) ==> [[0, 0, 0, 0, 0, 0] [0, 0, 2, 2, 0, 0] [0, 0, 0, 0, 0, 0]] ``` + END } diff --git a/tensorflow/core/api_def/base_api/api_def_QuantizeV2.pbtxt b/tensorflow/core/api_def/base_api/api_def_QuantizeV2.pbtxt index b9e75caf02..37ac10dddb 100644 --- a/tensorflow/core/api_def/base_api/api_def_QuantizeV2.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_QuantizeV2.pbtxt @@ -44,6 +44,7 @@ In 'MIN_COMBINED' mode, each value of the tensor will undergo the following: out[i] = (in[i] - min_range) * range(T) / (max_range - min_range) if T == qint8, out[i] -= (range(T) + 1) / 2.0 ``` + here `range(T) = numeric_limits::max() - numeric_limits::min()` *MIN_COMBINED Mode Example* @@ -87,6 +88,7 @@ choosing to elide the lowest possible value for symmetry (e.g., output range is We first find the range of values in our tensor. The range we use is always centered on 0, so we find m such that + ```c++ m = max(abs(input_min), abs(input_max)) ``` @@ -95,6 +97,7 @@ Our input tensor range is then `[-m, m]`. Next, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`. If T is signed, this is + ``` num_bits = sizeof(T) * 8 [min_fixed, max_fixed] = @@ -102,16 +105,19 @@ If T is signed, this is ``` Otherwise, if T is unsigned, the fixed-point range is + ``` [min_fixed, max_fixed] = [0, (1 << num_bits) - 1] ``` From this we compute our scaling factor, s: + ```c++ s = (max_fixed - min_fixed) / (2 * m) ``` Now we can quantize the elements of our tensor: + ```c++ result = round(input * s) ``` -- GitLab From 1f1d7b88717847f590987ee40efbe970bb591275 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 09:34:24 -0700 Subject: [PATCH 651/791] Disable dlopen error of libneuralnetworks for non-Android platforms. PiperOrigin-RevId: 193528346 --- tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h b/tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h index 85aca36874..ace4827d8c 100644 --- a/tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h +++ b/tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h @@ -34,10 +34,13 @@ limitations under the License. inline void* loadLibrary(const char* name) { // TODO: change RTLD_LOCAL? Assumes there can be multiple instances of nn // api RT - void* handle = dlopen(name, RTLD_LAZY | RTLD_LOCAL); + void* handle = nullptr; +#ifdef __ANDROID__ + handle = dlopen(name, RTLD_LAZY | RTLD_LOCAL); if (handle == nullptr) { NNAPI_LOG("nnapi error: unable to open library %s", name); } +#endif return handle; } -- GitLab From c173157bdc132460c6f424a9803221e74fc73f59 Mon Sep 17 00:00:00 2001 From: Saurabh Saxena Date: Thu, 19 Apr 2018 09:37:20 -0700 Subject: [PATCH 652/791] [tf.data] Add checkpointing support for MapAndBatchDataset. PiperOrigin-RevId: 193528712 --- .../kernel_tests/batch_dataset_op_test.py | 31 ++ .../kernels/data/map_and_batch_dataset_op.cc | 277 +++++++++++++++++- 2 files changed, 302 insertions(+), 6 deletions(-) diff --git a/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py index e1ec60d7c9..a4a0ce79b6 100644 --- a/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py @@ -681,6 +681,37 @@ class UnbatchDatasetSerializationTest( num_outputs) +class MapAndBatchDatasetSerializationTest( + dataset_serialization_test_base.DatasetSerializationTestBase): + + def testSerializationCore(self): + range_size = 11 + num_repeats = 2 + batch_size = 5 + total_outputs = range_size * num_repeats + num_outputs_drop_remainder = total_outputs // batch_size + num_outputs_keep_remainder = int(math.ceil(total_outputs / batch_size)) + num_parallel_batches = 2 + + def build_ds(range_start, drop_remainder=False): + + def _map_fn(x): + return math_ops.square(x) + + return dataset_ops.Dataset.range( + range_start, range_start + range_size).repeat(num_repeats).apply( + batching.map_and_batch( + map_func=_map_fn, + batch_size=batch_size, + num_parallel_batches=num_parallel_batches, + drop_remainder=drop_remainder)) + + self.run_core_tests(lambda: build_ds(10), lambda: build_ds(15), + num_outputs_keep_remainder) + self.run_core_tests(lambda: build_ds(10, True), lambda: build_ds(15, True), + num_outputs_drop_remainder) + + class PaddedBatchDatasetSerializationTest( dataset_serialization_test_base.DatasetSerializationTestBase): diff --git a/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc index aaf4dc7341..b8105552a0 100644 --- a/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc +++ b/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc @@ -74,26 +74,29 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel { OP_REQUIRES_OK(ctx, CapturedFunction::Create( func_, std::move(other_arguments), &captured_func)); - *output = new Dataset(input, batch_size, num_parallel_batches, - drop_remainder, output_types_, output_shapes_, + *output = new Dataset(ctx, input, batch_size, num_parallel_batches, + drop_remainder, output_types_, output_shapes_, func_, std::move(captured_func), &ctx->eigen_cpu_device()); } private: - class Dataset : public DatasetBase { + class Dataset : public GraphDatasetBase { public: - Dataset(const DatasetBase* input, int64 batch_size, + Dataset(OpKernelContext* ctx, const DatasetBase* input, int64 batch_size, int64 num_parallel_batches, bool drop_remainder, const DataTypeVector& output_types, const std::vector& output_shapes, + const NameAttrList& func, std::unique_ptr captured_func, const Eigen::ThreadPoolDevice* device) - : input_(input), + : GraphDatasetBase(ctx), + input_(input), batch_size_(batch_size), num_parallel_batches_(num_parallel_batches), drop_remainder_(drop_remainder), output_types_(output_types), output_shapes_(output_shapes), + map_fn_(func), captured_func_(std::move(captured_func)), device_(device) { input_->Ref(); @@ -117,6 +120,48 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel { string DebugString() override { return "MapAndBatchDatasetOp::Dataset"; } + protected: + Status AsGraphDefInternal(OpKernelContext* ctx, DatasetGraphDefBuilder* b, + Node** output) const override { + TF_RETURN_IF_ERROR(b->AddFunction(ctx, map_fn_.name())); + Node* input_graph_node = nullptr; + TF_RETURN_IF_ERROR(b->AddParentDataset(ctx, input_, &input_graph_node)); + Node* batch_size_node; + TF_RETURN_IF_ERROR(b->AddScalar(batch_size_, &batch_size_node)); + Node* num_parallel_batches_node; + TF_RETURN_IF_ERROR( + b->AddScalar(num_parallel_batches_, &num_parallel_batches_node)); + Node* drop_remainder_node; + TF_RETURN_IF_ERROR(b->AddScalar(drop_remainder_, &drop_remainder_node)); + + DataTypeVector other_arguments_types; + other_arguments_types.reserve(captured_func_->captured_inputs().size()); + std::vector other_arguments; + other_arguments.reserve(captured_func_->captured_inputs().size()); + for (const Tensor& t : captured_func_->captured_inputs()) { + Node* node; + TF_RETURN_IF_ERROR(b->AddTensor(t, &node)); + other_arguments.emplace_back(node); + other_arguments_types.emplace_back(t.dtype()); + } + AttrValue f; + b->BuildAttrValue(map_fn_, &f); + AttrValue other_arguments_types_attr; + b->BuildAttrValue(other_arguments_types, &other_arguments_types_attr); + + TF_RETURN_IF_ERROR(b->AddDataset( + this, + {std::make_pair(0, input_graph_node), + std::make_pair(2, batch_size_node), + std::make_pair(3, num_parallel_batches_node), + std::make_pair(4, drop_remainder_node)}, // Single tensor inputs. + {std::make_pair(1, other_arguments)}, // Tensor list inputs. + {std::make_pair("f", f), + std::make_pair("Targuments", other_arguments_types_attr)}, // Attrs + output)); + return Status::OK(); + } + private: class Iterator : public DatasetIterator { public: @@ -217,9 +262,83 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel { return status; } + protected: + Status SaveInternal(IteratorStateWriter* writer) override { + mutex_lock l(mu_); + if (current_batch_index_ == -1) { + // Iterator has not been used. Nothing to save. + return Status::OK(); + } + TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("current_batch_index"), + current_batch_index_)); + TF_RETURN_IF_ERROR(SaveParent(writer, input_impl_)); + TF_RETURN_IF_ERROR(writer->WriteScalar( + full_name("invocation_results_size"), invocation_results_.size())); + for (size_t i = 0; i < invocation_results_.size(); ++i) { + TF_RETURN_IF_ERROR(WriteInvocationResultLocked(writer, i)); + } + TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("batch_results_size"), + batch_results_.size())); + for (size_t i = 0; i < batch_results_.size(); ++i) { + TF_RETURN_IF_ERROR(WriteBatchResultLocked(writer, i)); + } + return Status::OK(); + } + + Status RestoreInternal(IteratorContext* ctx, + IteratorStateReader* reader) override { + mutex_lock l(mu_); + if (!reader->Contains(full_name("current_batch_index"))) { + // Iterator was never used so nothing to restore. + return Status::OK(); + } + { + int64 temp; + TF_RETURN_IF_ERROR( + reader->ReadScalar(full_name("current_batch_index"), &temp)); + current_batch_index_ = static_cast(temp); + if (current_batch_index_ != temp) { + return errors::Internal("Invalid value for current_batch_index ", + temp); + } + } + TF_RETURN_IF_ERROR(RestoreParent(ctx, reader, input_impl_)); + size_t invocation_results_size; + { + int64 temp; + TF_RETURN_IF_ERROR( + reader->ReadScalar(full_name("invocation_results_size"), &temp)); + invocation_results_size = static_cast(temp); + if (invocation_results_size != temp) { + return errors::Internal( + "Invalid value for invocation_results_size ", temp); + } + } + CHECK_EQ(invocation_results_.size(), invocation_results_size); + for (size_t i = 0; i < invocation_results_size; ++i) { + TF_RETURN_IF_ERROR(ReadInvocationResultLocked(reader, i)); + } + size_t batch_results_size; + { + int64 temp; + TF_RETURN_IF_ERROR( + reader->ReadScalar(full_name("batch_results_size"), &temp)); + batch_results_size = static_cast(temp); + if (batch_results_size != temp) { + return errors::Internal("Invalid value for batch_results_size ", + temp); + } + } + CHECK_EQ(batch_results_.size(), batch_results_size); + for (size_t i = 0; i < batch_results_size; ++i) { + TF_RETURN_IF_ERROR(ReadBatchResultLocked(reader, i)); + } + return Status::OK(); + } + private: struct BatchResult { - mutex mu; + mutex mu ACQUIRED_AFTER(mu_); bool output_allocated GUARDED_BY(mu); std::vector output; std::unique_ptr counter; @@ -393,6 +512,151 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel { return status; } + Status WriteInvocationResultLocked(IteratorStateWriter* writer, + size_t index) + EXCLUSIVE_LOCKS_REQUIRED(mu_) { + const InvocationResult& result = invocation_results_[index]; + string prefix = strings::StrCat("invocation_results_", index); + TF_RETURN_IF_ERROR(WriteStatusLocked( + writer, full_name(strings::StrCat(prefix, "_status")), + result.status)); + if (result.end_of_input) { + TF_RETURN_IF_ERROR(writer->WriteScalar( + full_name(strings::StrCat(prefix, "_end_of_input")), "")); + } + TF_RETURN_IF_ERROR(writer->WriteScalar( + full_name(strings::StrCat(prefix, "_return_values_size")), + result.return_values.size())); + for (size_t i = 0; i < result.return_values.size(); i++) { + TF_RETURN_IF_ERROR(writer->WriteTensor( + full_name(strings::StrCat(prefix, "_return_values_", i)), + result.return_values[i])); + } + return Status::OK(); + } + + Status ReadInvocationResultLocked(IteratorStateReader* reader, + size_t index) + EXCLUSIVE_LOCKS_REQUIRED(mu_) { + InvocationResult* result = &invocation_results_[index]; + string prefix = strings::StrCat("invocation_results_", index); + TF_RETURN_IF_ERROR(ReadStatusLocked( + reader, full_name(strings::StrCat(prefix, "_status")), + &result->status)); + result->end_of_input = reader->Contains( + full_name(strings::StrCat(prefix, "_end_of_input"))); + size_t return_values_size; + { + int64 temp; + TF_RETURN_IF_ERROR(reader->ReadScalar( + full_name(strings::StrCat(prefix, "_return_values_size")), + &temp)); + return_values_size = static_cast(temp); + if (temp != return_values_size) { + return errors::Internal("Invalid value for return_values_size ", + return_values_size); + } + } + result->return_values.reserve(return_values_size); + for (size_t i = 0; i < return_values_size; i++) { + result->return_values.emplace_back(); + TF_RETURN_IF_ERROR(reader->ReadTensor( + full_name(strings::StrCat(prefix, "_return_values_", i)), + &result->return_values.back())); + } + return Status::OK(); + } + + Status WriteBatchResultLocked(IteratorStateWriter* writer, size_t index) + EXCLUSIVE_LOCKS_REQUIRED(mu_) { + // Wait for the map_fn dispatches made in `InvokeFunctionLocked` to + // finish. This may delay saving a checkpoint by a bit but keeps the + // code clean and also saves us from checkpointing the state of the + // `BlockingCounter`. + batch_results_[index].counter->Wait(); + const BatchResult& result = batch_results_[index]; + string prefix = strings::StrCat("batch_results_", index); + { + mutex_lock l(batch_results_[index].mu); + if (result.output_allocated) { + TF_RETURN_IF_ERROR(writer->WriteScalar( + full_name(strings::StrCat(prefix, "_output_allocated")), "")); + } + } + TF_RETURN_IF_ERROR(writer->WriteScalar( + full_name(strings::StrCat(prefix, "_output_size")), + result.output.size())); + for (size_t i = 0; i < result.output.size(); i++) { + TF_RETURN_IF_ERROR(writer->WriteTensor( + full_name(strings::StrCat(prefix, "_output_", i)), + result.output[i])); + } + return Status::OK(); + } + + Status ReadBatchResultLocked(IteratorStateReader* reader, size_t index) + EXCLUSIVE_LOCKS_REQUIRED(mu_) { + BatchResult* result = &batch_results_[index]; + string prefix = strings::StrCat("batch_results_", index); + { + mutex_lock l(batch_results_[index].mu); + result->output_allocated = reader->Contains( + full_name(strings::StrCat(prefix, "_output_allocated"))); + // Simulate that the batch was fully generated. + batch_results_[index].counter.reset(new BlockingCounter(0)); + } + size_t output_size; + { + int64 temp; + TF_RETURN_IF_ERROR(reader->ReadScalar( + full_name(strings::StrCat(prefix, "_output_size")), &temp)); + output_size = static_cast(temp); + if (temp != output_size) { + return errors::Internal("Invalid value for output_size ", + output_size); + } + } + result->output.reserve(output_size); + for (size_t i = 0; i < output_size; i++) { + result->output.emplace_back(); + TF_RETURN_IF_ERROR(reader->ReadTensor( + full_name(strings::StrCat(prefix, "_output_", i)), + &result->output.back())); + } + return Status::OK(); + } + + Status WriteStatusLocked(IteratorStateWriter* writer, + const string& prefix, const Status& status) + EXCLUSIVE_LOCKS_REQUIRED(mu_) { + TF_RETURN_IF_ERROR( + writer->WriteScalar(full_name(strings::StrCat(prefix, "_code")), + static_cast(status.code()))); + if (!status.ok()) { + TF_RETURN_IF_ERROR( + writer->WriteScalar(full_name(strings::StrCat(prefix, "_msg")), + status.error_message())); + } + return Status::OK(); + } + + Status ReadStatusLocked(IteratorStateReader* reader, const string& prefix, + Status* status) EXCLUSIVE_LOCKS_REQUIRED(mu_) { + int64 code_int; + TF_RETURN_IF_ERROR(reader->ReadScalar( + full_name(strings::StrCat(prefix, "_code")), &code_int)); + error::Code code = static_cast(code_int); + + if (code != error::Code::OK) { + string error_message; + TF_RETURN_IF_ERROR(reader->ReadScalar( + full_name(strings::StrCat(prefix, "_msg")), &error_message)); + *status = Status(code, error_message); + } else { + *status = Status::OK(); + } + return Status::OK(); + } mutex mu_; int32 current_batch_index_ GUARDED_BY(mu_) = -1; const std::unique_ptr input_impl_ GUARDED_BY(mu_); @@ -407,6 +671,7 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel { const bool drop_remainder_; const DataTypeVector output_types_; const std::vector output_shapes_; + const NameAttrList map_fn_; const std::unique_ptr captured_func_; const Eigen::ThreadPoolDevice* device_; // not owned }; -- GitLab From 436f1434060d7f370baae9661baacc6cf27415ec Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Thu, 19 Apr 2018 09:54:40 -0700 Subject: [PATCH 653/791] Create a skeleton tf.contrib.checkpoint. My plan for this is to incubate tools for working with object-based checkpoints: - Tools for managing dependency graphs, e.g. checkpointable lists/dictionaries - Inspecting/visualizing checkpoints - Listing variables and gathering initializers from a Checkpointable object and its dependencies - Verifying all variables are accessible as dependencies, which should make converting existing graph building Saver uses easier/safer. This CL includes none of those things, it just moves the split_dependency tool here instead of contrib/eager. PiperOrigin-RevId: 193531292 --- tensorflow/contrib/__init__.py | 1 + tensorflow/contrib/checkpoint/README.md | 2 + tensorflow/contrib/checkpoint/__init__.py | 29 +++++++++++ tensorflow/contrib/checkpoint/python/BUILD | 29 +++++++++++ .../python/split_dependency.py} | 8 ++-- .../python/split_dependency_test.py} | 4 +- tensorflow/contrib/cmake/python_modules.txt | 2 + tensorflow/contrib/cudnn_rnn/BUILD | 2 +- .../cudnn_rnn/python/ops/cudnn_rnn_ops.py | 4 +- tensorflow/contrib/eager/python/BUILD | 48 ++----------------- tensorflow/contrib/optimizer_v2/BUILD | 1 - tensorflow/tools/pip_package/BUILD | 1 - 12 files changed, 75 insertions(+), 56 deletions(-) create mode 100644 tensorflow/contrib/checkpoint/README.md create mode 100644 tensorflow/contrib/checkpoint/__init__.py create mode 100644 tensorflow/contrib/checkpoint/python/BUILD rename tensorflow/contrib/{eager/python/checkpointable_utils.py => checkpoint/python/split_dependency.py} (95%) rename tensorflow/contrib/{eager/python/checkpointable_utils_test.py => checkpoint/python/split_dependency_test.py} (96%) diff --git a/tensorflow/contrib/__init__.py b/tensorflow/contrib/__init__.py index 36cc5144d0..0d163daa6e 100644 --- a/tensorflow/contrib/__init__.py +++ b/tensorflow/contrib/__init__.py @@ -24,6 +24,7 @@ import os # Add projects here, they will show up under tf.contrib. from tensorflow.contrib import batching from tensorflow.contrib import bayesflow +from tensorflow.contrib import checkpoint from tensorflow.contrib import cloud from tensorflow.contrib import cluster_resolver from tensorflow.contrib import coder diff --git a/tensorflow/contrib/checkpoint/README.md b/tensorflow/contrib/checkpoint/README.md new file mode 100644 index 0000000000..d35c5bae3b --- /dev/null +++ b/tensorflow/contrib/checkpoint/README.md @@ -0,0 +1,2 @@ +Tools for working with object-based checkpoints produced by +`tf.train.Checkpoint`. diff --git a/tensorflow/contrib/checkpoint/__init__.py b/tensorflow/contrib/checkpoint/__init__.py new file mode 100644 index 0000000000..70d7d2d8d7 --- /dev/null +++ b/tensorflow/contrib/checkpoint/__init__.py @@ -0,0 +1,29 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tools for working with object-based checkpoints. + + +For creating and managing dependencies: +@@split_dependency +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.checkpoint.python.split_dependency import split_dependency +from tensorflow.python.util.all_util import remove_undocumented + +remove_undocumented(module_name=__name__) diff --git a/tensorflow/contrib/checkpoint/python/BUILD b/tensorflow/contrib/checkpoint/python/BUILD new file mode 100644 index 0000000000..d57b01aab2 --- /dev/null +++ b/tensorflow/contrib/checkpoint/python/BUILD @@ -0,0 +1,29 @@ +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//tensorflow:internal"]) + +load("//tensorflow:tensorflow.bzl", "py_test") + +py_library( + name = "split_dependency", + srcs = ["split_dependency.py"], + srcs_version = "PY2AND3", + visibility = ["//tensorflow:internal"], + deps = [ + "//tensorflow/python:control_flow_ops", + "//tensorflow/python:training", + ], +) + +py_test( + name = "split_dependency_test", + srcs = ["split_dependency_test.py"], + deps = [ + ":split_dependency", + "//tensorflow/python:array_ops", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:resource_variable_ops", + "//tensorflow/python:training", + "//tensorflow/python/eager:test", + ], +) diff --git a/tensorflow/contrib/eager/python/checkpointable_utils.py b/tensorflow/contrib/checkpoint/python/split_dependency.py similarity index 95% rename from tensorflow/contrib/eager/python/checkpointable_utils.py rename to tensorflow/contrib/checkpoint/python/split_dependency.py index 30c4103c5a..3aec8c96e9 100644 --- a/tensorflow/contrib/eager/python/checkpointable_utils.py +++ b/tensorflow/contrib/checkpoint/python/split_dependency.py @@ -1,4 +1,4 @@ -"""Utilities for working with Checkpointable objects.""" +"""Utility for creating multiple dependencies with synchronized save/restore.""" # Copyright 2017 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,7 @@ from __future__ import print_function import functools from tensorflow.python.ops import control_flow_ops -from tensorflow.python.training import checkpointable as core_checkpointable +from tensorflow.python.training import checkpointable as checkpointable from tensorflow.python.training import saver as saver_lib @@ -43,7 +43,7 @@ class _CallbackSaveable(saver_lib.BaseSaverBuilder.SaveableObject): return self._restore_callback(tensor) -class _SplitDependency(core_checkpointable.CheckpointableBase): +class _SplitDependency(checkpointable.CheckpointableBase): """Looks like a regular variable while synchronizing save/restores.""" def __init__(self, save_buffer, restore_buffer, name, dtype, num_components, @@ -83,7 +83,7 @@ class _SplitDependency(core_checkpointable.CheckpointableBase): def _gather_saveables_for_checkpoint(self): """Looks to Checkpointable like a regular variable.""" return { - core_checkpointable.VARIABLE_VALUE_KEY: + checkpointable.VARIABLE_VALUE_KEY: functools.partial(_CallbackSaveable, dtype=self._dtype, save_callback=self._save, diff --git a/tensorflow/contrib/eager/python/checkpointable_utils_test.py b/tensorflow/contrib/checkpoint/python/split_dependency_test.py similarity index 96% rename from tensorflow/contrib/eager/python/checkpointable_utils_test.py rename to tensorflow/contrib/checkpoint/python/split_dependency_test.py index da04199aaa..cb964c80e9 100644 --- a/tensorflow/contrib/eager/python/checkpointable_utils_test.py +++ b/tensorflow/contrib/checkpoint/python/split_dependency_test.py @@ -18,7 +18,7 @@ from __future__ import print_function import os -from tensorflow.contrib.eager.python import checkpointable_utils as contrib_checkpointable_utils +from tensorflow.contrib.checkpoint.python import split_dependency from tensorflow.python.eager import test from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops @@ -47,7 +47,7 @@ class SaveTensorSlicesAsDeps(checkpointable.CheckpointableBase): def __init__(self): self.combined = resource_variable_ops.ResourceVariable([0., 0., 0., 0.]) - split_dependencies = contrib_checkpointable_utils.split_dependency( + split_dependencies = split_dependency.split_dependency( component_names=("first_half", "second_half"), component_dtypes=(self.combined.dtype,) * 2, fill_save_buffer_fn=_split_variable_closure( diff --git a/tensorflow/contrib/cmake/python_modules.txt b/tensorflow/contrib/cmake/python_modules.txt index 91839194c7..fbcdf7e753 100644 --- a/tensorflow/contrib/cmake/python_modules.txt +++ b/tensorflow/contrib/cmake/python_modules.txt @@ -130,6 +130,8 @@ tensorflow/contrib/boosted_trees/ops tensorflow/contrib/boosted_trees/proto tensorflow/contrib/boosted_trees/python tensorflow/contrib/boosted_trees/python/ops +tensorflow/contrib/checkpoint +tensorflow/contrib/checkpoint/python tensorflow/contrib/cloud tensorflow/contrib/cloud/kernels tensorflow/contrib/cloud/ops diff --git a/tensorflow/contrib/cudnn_rnn/BUILD b/tensorflow/contrib/cudnn_rnn/BUILD index d68015ae15..aeefa3cee6 100644 --- a/tensorflow/contrib/cudnn_rnn/BUILD +++ b/tensorflow/contrib/cudnn_rnn/BUILD @@ -25,7 +25,7 @@ tf_custom_op_py_library( srcs_version = "PY2AND3", visibility = ["//visibility:public"], deps = [ - "//tensorflow/contrib/eager/python:checkpointable_utils", + "//tensorflow/contrib/checkpoint/python:split_dependency", "//tensorflow/contrib/util:util_py", "//tensorflow/python:array_ops", "//tensorflow/python:control_flow_ops", diff --git a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py index b615824460..a1ede4471e 100644 --- a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py +++ b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py @@ -17,7 +17,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.contrib.eager.python import checkpointable_utils +from tensorflow.contrib.checkpoint.python import split_dependency from tensorflow.contrib.rnn.python.ops import lstm_ops from tensorflow.python.framework import common_shapes from tensorflow.python.framework import dtypes @@ -318,7 +318,7 @@ class CudnnOpaqueParamsSaveable(saver.BaseSaverBuilder.SaveableObject): dependencies too (typically the cuDNN `Layer`). dtype: The dtype for the canonical parameter Tensors. """ - split_dependencies = checkpointable_utils.split_dependency( + split_dependencies = split_dependency.split_dependency( component_names=self._param_names, component_dtypes=(dtype,) * len(self._param_names), fill_save_buffer_fn=self._checkpointable_save, diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD index e2744a430d..99abbae03f 100644 --- a/tensorflow/contrib/eager/python/BUILD +++ b/tensorflow/contrib/eager/python/BUILD @@ -11,7 +11,6 @@ py_library( srcs_version = "PY2AND3", visibility = ["//visibility:public"], deps = [ - ":checkpointable_utils", ":datasets", ":metrics", ":network", @@ -19,15 +18,14 @@ py_library( "//tensorflow/python:framework_ops", "//tensorflow/python:framework_test_lib", "//tensorflow/python:gradients", - "//tensorflow/python:numerics", "//tensorflow/python:resource_variable_ops", "//tensorflow/python:script_ops", "//tensorflow/python:template", + "//tensorflow/python:training", "//tensorflow/python:util", "//tensorflow/python:variable_scope", "//tensorflow/python/eager:backprop", "//tensorflow/python/eager:context", - "//tensorflow/python/eager:core", "//tensorflow/python/eager:execution_callbacks", "//tensorflow/python/eager:function", ], @@ -70,7 +68,6 @@ cuda_py_test( srcs = ["datasets_test.py"], additional_deps = [ ":datasets", - ":checkpointable_utils", "//tensorflow/contrib/data/python/ops:prefetching_ops", "//tensorflow/contrib/data/python/ops:threadpool", "//tensorflow/contrib/data/python/ops:unique", @@ -79,6 +76,7 @@ cuda_py_test( "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", "//tensorflow/python:script_ops", + "//tensorflow/python:training", "//tensorflow/python/data", "//tensorflow/python/eager:test", ], @@ -121,8 +119,8 @@ py_library( srcs_version = "PY2AND3", visibility = ["//tensorflow:internal"], deps = [ - "//tensorflow/contrib/eager/python:checkpointable_utils", "//tensorflow/python:array_ops", + "//tensorflow/python:checkpointable", "//tensorflow/python:control_flow_ops", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", @@ -225,43 +223,3 @@ py_test( "//tensorflow/python/eager:test", ], ) - -py_library( - name = "checkpointable_utils", - srcs = ["checkpointable_utils.py"], - srcs_version = "PY2AND3", - visibility = ["//tensorflow:internal"], - deps = [ - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:training", - ], -) - -cuda_py_test( - name = "checkpointable_utils_test", - srcs = ["checkpointable_utils_test.py"], - additional_deps = [ - ":checkpointable_utils", - ":network", - "@six_archive//:six", - "//tensorflow/python:constant_op", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:init_ops", - "//tensorflow/python:layers", - "//tensorflow/python:layers_base", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:state_ops", - "//tensorflow/python:training", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//tensorflow/python/eager:context", - "//tensorflow/python/eager:test", - "//tensorflow/python/keras", - ], - tags = [ - "no_windows", # TODO: needs investigation on Windows - "notsan", # b/74395663 - ], -) diff --git a/tensorflow/contrib/optimizer_v2/BUILD b/tensorflow/contrib/optimizer_v2/BUILD index 85cfce346c..5225ecc14f 100644 --- a/tensorflow/contrib/optimizer_v2/BUILD +++ b/tensorflow/contrib/optimizer_v2/BUILD @@ -115,7 +115,6 @@ cuda_py_test( additional_deps = [ ":training", "@six_archive//:six", - "//tensorflow/contrib/eager/python:checkpointable_utils", "//tensorflow/python:constant_op", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD index 2ef105755f..0ac5a5bb6d 100644 --- a/tensorflow/tools/pip_package/BUILD +++ b/tensorflow/tools/pip_package/BUILD @@ -66,7 +66,6 @@ COMMON_PIP_DEPS = [ "//tensorflow/contrib/data/python/kernel_tests:dataset_serialization_test", "//tensorflow/contrib/data/python/ops:contrib_op_loader", "//tensorflow/contrib/eager/python/examples:examples_pip", - "//tensorflow/contrib/eager/python:checkpointable_utils", "//tensorflow/contrib/eager/python:evaluator", "//tensorflow/contrib/gan:gan", "//tensorflow/contrib/graph_editor:graph_editor_pip", -- GitLab From 2273b62a769aa477f8d2ef02ca7dee253b8ea7b0 Mon Sep 17 00:00:00 2001 From: Benoit Steiner Date: Thu, 19 Apr 2018 10:05:08 -0700 Subject: [PATCH 654/791] Added support for concatenation and slicing of symbolic shapes PiperOrigin-RevId: 193532769 --- ...direct_session_with_tracking_alloc_test.cc | 4 +- tensorflow/core/framework/shape_inference.cc | 2 + tensorflow/core/framework/shape_inference.h | 12 + .../core/grappler/costs/graph_properties.cc | 236 ++++++++++++++++-- 4 files changed, 235 insertions(+), 19 deletions(-) diff --git a/tensorflow/core/common_runtime/direct_session_with_tracking_alloc_test.cc b/tensorflow/core/common_runtime/direct_session_with_tracking_alloc_test.cc index 31fb128f93..b4dd521bbc 100644 --- a/tensorflow/core/common_runtime/direct_session_with_tracking_alloc_test.cc +++ b/tensorflow/core/common_runtime/direct_session_with_tracking_alloc_test.cc @@ -102,9 +102,9 @@ TEST(DirectSessionWithTrackingAllocTest, CostModelTest) { EXPECT_EQ(2, shape.dim(0).size()); EXPECT_EQ(1, shape.dim(1).size()); if (node->name() == y->name()) { - EXPECT_EQ(3, cm->AllocationId(node, 0)); + EXPECT_EQ(7, cm->AllocationId(node, 0)); } else { - EXPECT_EQ(4, cm->AllocationId(node, 0)); + EXPECT_EQ(8, cm->AllocationId(node, 0)); } } EXPECT_LE(0, cm->MaxExecutionTime(node)); diff --git a/tensorflow/core/framework/shape_inference.cc b/tensorflow/core/framework/shape_inference.cc index 229b4a45fa..2b995e8b5e 100644 --- a/tensorflow/core/framework/shape_inference.cc +++ b/tensorflow/core/framework/shape_inference.cc @@ -157,8 +157,10 @@ InferenceContext::~InferenceContext() {} Status InferenceContext::Run( const std::function& fn) { + ForgetMerges(); Status s = fn(this); if (!s.ok()) { + ForgetMerges(); return AttachContext(s); } #ifndef NDEBUG diff --git a/tensorflow/core/framework/shape_inference.h b/tensorflow/core/framework/shape_inference.h index cdb4bd79bb..9431a62abe 100644 --- a/tensorflow/core/framework/shape_inference.h +++ b/tensorflow/core/framework/shape_inference.h @@ -285,6 +285,8 @@ class InferenceContext { return true; } + void SetInput(int idx, ShapeHandle shape) { inputs_[idx] = shape; } + ShapeHandle input(int64 idx) const { return inputs_[idx]; } Status input(StringPiece input_name, std::vector* output) const; int num_inputs() const { return inputs_.size(); } @@ -317,6 +319,10 @@ class InferenceContext { input_tensors_as_shapes_ = input_tensors_as_shapes; } + const std::vector& input_tensors_as_shapes() const { + return input_tensors_as_shapes_; + } + ShapeHandle output(int64 idx) const { return outputs_[idx]; } void set_output(int idx, ShapeHandle shape) { outputs_[idx] = shape; } Status set_output(StringPiece output_name, @@ -587,6 +593,12 @@ class InferenceContext { int idx, const std::vector& shapes_and_types) TF_MUST_USE_RESULT; + void set_input_handle_shapes_and_types( + int idx, const std::vector& shapes_and_types) { + input_handle_shapes_and_types_[idx].reset( + new std::vector(shapes_and_types)); + } + // Returns the output handle shapes and types, for the resource tensor output // at index . Returns NULL if the shape and types were never set. const std::vector* output_handle_shapes_and_types(int idx) { diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc index a9c777e551..c83ddfe90a 100644 --- a/tensorflow/core/grappler/costs/graph_properties.cc +++ b/tensorflow/core/grappler/costs/graph_properties.cc @@ -18,8 +18,9 @@ limitations under the License. #include #include #include -#include "tensorflow/core/common_runtime/shape_refiner.h" +#include "tensorflow/core/framework/common_shape_fns.h" #include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/framework/versions.pb.h" #include "tensorflow/core/graph/graph_constructor.h" #include "tensorflow/core/grappler/costs/utils.h" #include "tensorflow/core/grappler/utils.h" @@ -394,15 +395,121 @@ class TopoQueue { // unknown shape/dimension of a given node. class SymbolicShapeRefiner { public: - explicit SymbolicShapeRefiner(ShapeRefiner* shape_refiner) - : shape_refiner_(shape_refiner) {} + explicit SymbolicShapeRefiner(const GraphDef& graph) + : function_library_(OpRegistry::Global(), graph.library()) { + graph_def_version_ = graph.versions().producer(); + node_to_context_.reserve(graph.node_size()); + } InferenceContext* GetContext(const Node* node) { - return shape_refiner_->GetContext(node); + auto it = node_to_context_.find(node); + if (it == node_to_context_.end()) { + return nullptr; + } + return it->second.inference_context.get(); } Status UpdateNode(const Node* node, bool relax, bool* refined) { - return shape_refiner_->UpdateNode(node, relax, refined); + NodeContext* node_context = GetNodeContext(node); + if (node_context == nullptr) { + TF_RETURN_IF_ERROR(AddNode(node)); + node_context = CHECK_NOTNULL(GetNodeContext(node)); + *refined = true; + } + // Check if the shapes of the nodes in the fan-in of this node have changed, + // and if they have, update the node input shapes. + InferenceContext* inference_context = node_context->inference_context.get(); + std::vector const_values(node->num_inputs()); + std::vector input_tensors(node->num_inputs(), nullptr); + std::vector input_tensors_as_shapes(node->num_inputs()); + + for (const Edge* e : node->in_edges()) { + if (e->IsControlEdge()) continue; + + int dst_input = e->dst_input(); + int src_output = e->src_output(); + + Node* input = e->src(); + NodeContext* c = GetNodeContext(input); + if (c == nullptr) { + return errors::FailedPrecondition( + "Input ", dst_input, " ('", input->name(), "') for '", node->name(), + "' was not previously added to ShapeRefiner."); + } + + if (input->IsConstant()) { + // Convert constant value into tensors. + if (const_values[dst_input].FromProto( + input->def().attr().at("value").tensor())) { + input_tensors[dst_input] = &const_values[dst_input]; + // Integer tensors of rank one can also be interpreted as a shape + // provided all their values are >= -1. + if (const_values[dst_input].dims() == 1 && + (const_values[dst_input].dtype() == DT_INT32 || + const_values[dst_input].dtype() == DT_INT64)) { + ShapeHandle tensor_shape = inference_context->Vector( + const_values[dst_input].NumElements()); + ShapeHandle shp; + if (inference_context + ->MakeShapeFromTensor(input_tensors[dst_input], + tensor_shape, &shp) + .ok()) { + input_tensors_as_shapes[dst_input] = shp; + } + } + } + } + + if (c->output_tensors_as_shapes.size() > src_output) { + input_tensors_as_shapes[dst_input] = + c->output_tensors_as_shapes[src_output]; + } + + DCHECK_GE(dst_input, 0); + if (!*refined && !inference_context->input(dst_input).SameHandle( + c->inference_context->output(src_output))) { + *refined = true; + } + inference_context->SetInput(dst_input, + c->inference_context->output(src_output)); + + if (!*refined && + inference_context->requested_input_tensor_as_partial_shape( + dst_input)) { + // The input value may have changed. Since we have no way to know if + // that's indeed the case, err on the safe side. + *refined = true; + } + + // Also propagate handle shape and dtype of edges which are carrying + // resource handles. + if (e->src()->output_type(src_output) == DT_RESOURCE) { + auto* outputs = + c->inference_context->output_handle_shapes_and_types(src_output); + if (!outputs) continue; + auto* inputs = + inference_context->input_handle_shapes_and_types(dst_input); + + if (!inputs || !EquivalentShapesAndTypes(*outputs, *inputs)) { + *refined = true; + } + inference_context->set_input_handle_shapes_and_types(dst_input, + *outputs); + } + } + + if (!*refined) { + // No input shape has changed, we're done + return Status::OK(); + } + + node_context->inference_context->set_input_tensors(input_tensors); + node_context->inference_context->set_input_tensors_as_shapes( + input_tensors_as_shapes); + + // Update the shapes of the outputs. + return InferShapes(node, node_context); } + Status SetUnknownShape(const Node* node, int output_port) { shape_inference::ShapeHandle shape = GetUnknownOutputShape(node, output_port); @@ -450,7 +557,7 @@ class SymbolicShapeRefiner { if (shape1.SameHandle(shape2)) { return shape1; } - InferenceContext* ctx = shape_refiner_->GetContext(node); + InferenceContext* ctx = GetContext(node); ShapeHandle merged = shape1; if (!ctx->RankKnown(shape2) && !ctx->RankKnown(shape1)) { // Return either one since they're expected to represent the same value. @@ -495,7 +602,7 @@ class SymbolicShapeRefiner { if (shape1.SameHandle(shape2)) { return shape1; } - InferenceContext* ctx = shape_refiner_->GetContext(node); + InferenceContext* ctx = GetContext(node); ShapeHandle relaxed = shape1; const int rank = ctx->Rank(shape1); if (!ctx->RankKnown(shape2) || ctx->Rank(shape2) != rank) { @@ -569,7 +676,7 @@ class SymbolicShapeRefiner { if (it != unknown_shapes_.end()) { return it->second; } - InferenceContext* c = shape_refiner_->GetContext(node); + InferenceContext* c = GetContext(node); ShapeHandle shp = c->UnknownShape(); unknown_shapes_[id] = shp; return shp; @@ -582,16 +689,114 @@ class SymbolicShapeRefiner { if (it != unknown_dims_.end()) { return it->second; } - InferenceContext* c = shape_refiner_->GetContext(node); + InferenceContext* c = GetContext(node); DimensionHandle dim = c->UnknownDim(); unknown_dims_[id] = dim; return dim; } - ShapeRefiner* shape_refiner_; + Status AddNode(const Node* node) { + // Create the inference context for this node. + std::vector input_shapes(node->num_inputs()); + std::vector>> + input_handle_shapes_and_types(node->num_inputs()); + std::vector input_tensors(node->num_inputs(), nullptr); + std::vector input_tensors_as_shapes; + + NodeContext& node_ctx = node_to_context_[node]; + node_ctx.inference_context.reset(new InferenceContext( + graph_def_version_, &node->def(), node->op_def(), input_shapes, + input_tensors, input_tensors_as_shapes, + std::move(input_handle_shapes_and_types))); + const Status s = node_ctx.inference_context->construction_status(); + if (!s.ok()) { + node_ctx.inference_context.reset(nullptr); + } + return s; + } + + struct NodeContext { + std::unique_ptr inference_context; + std::vector output_tensors_as_shapes; + }; + + Status InferShapes(const Node* node, NodeContext* c) { + InferenceContext* ic = c->inference_context.get(); + + // Propagate shape tensors + if (node->type_string() == "Shape") { + c->output_tensors_as_shapes.resize(1); + c->output_tensors_as_shapes[0] = c->inference_context->input(0); + } else if (node->type_string() == "ShapeN") { + c->output_tensors_as_shapes.resize(c->inference_context->num_inputs()); + for (int i = 0; i < c->inference_context->num_inputs(); ++i) { + c->output_tensors_as_shapes[i] = c->inference_context->input(i); + } + } else if (node->type_string() == "ConcatV2") { + bool valid = true; + ShapeHandle result; + for (int i = 0; i < ic->num_inputs() - 1; ++i) { + ShapeHandle input = ic->input_tensors_as_shapes()[i]; + if (!ic->RankKnown(input)) { + valid = false; + break; + } else if (i == 0) { + result = input; + } else { + TF_RETURN_IF_ERROR(ic->Concatenate(result, input, &result)); + } + } + if (valid) { + c->output_tensors_as_shapes.resize(1); + c->output_tensors_as_shapes[0] = result; + } + } else if (node->type_string() == "Slice") { + ShapeHandle input = ic->input_tensors_as_shapes()[0]; + bool valid = ic->RankKnown(input); + const Tensor* slice_offset = ic->input_tensor(1); + valid &= slice_offset != nullptr && slice_offset->NumElements() == 1; + const Tensor* slice_size = ic->input_tensor(2); + valid &= slice_size != nullptr && slice_size->NumElements() == 1; + if (valid) { + int64 start = slice_offset->dtype() == DT_INT32 + ? slice_offset->flat()(0) + : slice_offset->flat()(0); + int64 end = start + (slice_size->dtype() == DT_INT32 + ? slice_size->flat()(0) + : slice_size->flat()(0)); + ShapeHandle result; + TF_RETURN_IF_ERROR(ic->Subshape(input, start, end, &result)); + c->output_tensors_as_shapes.resize(1); + c->output_tensors_as_shapes[0] = result; + } + } + + // Infer the shapes of output tensors. + const OpRegistrationData* op_reg_data; + Status s = function_library_.default_registry()->LookUp(node->type_string(), + &op_reg_data); + if (!s.ok() || op_reg_data->shape_inference_fn == nullptr) { + // There is nothing more we can infer, annotate outputs with unknown + // shapes + return c->inference_context->Run(shape_inference::UnknownShape); + } + + return c->inference_context->Run(op_reg_data->shape_inference_fn); + } + + NodeContext* GetNodeContext(const Node* node) { + auto it = node_to_context_.find(node); + if (it == node_to_context_.end()) { + return nullptr; + } + return &it->second; + } + int graph_def_version_; + std::unordered_map node_to_context_; std::unordered_map unknown_shapes_; std::unordered_map unknown_dims_; + FunctionLibraryDefinition function_library_; }; // Keep track of shapes and dimensions in a graph. @@ -977,9 +1182,6 @@ Status GraphProperties::InferStatically(bool assume_valid_feeds) { item_.graph.library()); Graph graph(function_library); graph_ = &graph; - ShapeRefiner shape_refiner(graph.versions(), graph.op_registry()); - shape_refiner.set_require_shape_inference_fns(false); - shape_refiner.set_disable_constant_propagation(true); ImportGraphDefOptions options; // Graph optimization happens at the late stage of graph execution, // when colocation constraints are already validated previously and @@ -987,7 +1189,7 @@ Status GraphProperties::InferStatically(bool assume_valid_feeds) { // is no need to validate colocation constraints again. options.validate_colocation_constraints = false; options.validate_shape = false; - Status s = ImportGraphDef(options, item_.graph, &graph, &shape_refiner); + Status s = ImportGraphDef(options, item_.graph, &graph, nullptr); TF_RETURN_IF_ERROR(s); std::unordered_map> fed_ports; @@ -1041,7 +1243,7 @@ Status GraphProperties::InferStatically(bool assume_valid_feeds) { } } - SymbolicShapeRefiner refiner(&shape_refiner); + SymbolicShapeRefiner refiner(item_.graph); // We propagate shapes through the graph in two phases. In the first phase, we // exclusively merge shapes but we do not propagate shapes through the @@ -1073,7 +1275,7 @@ Status GraphProperties::InferStatically(bool assume_valid_feeds) { SymbolicShapeManager shape_manager; bool found_error = false; for (const Node* const node : graph.nodes()) { - auto node_ctx = shape_refiner.GetContext(node); + auto node_ctx = refiner.GetContext(node); if (!node_ctx) { continue; } @@ -1105,7 +1307,7 @@ Status GraphProperties::InferStatically(bool assume_valid_feeds) { for (const Node* const node : graph.nodes()) { VLOG(3) << "Filling in graph properties for node: " << node->name(); - auto ctx = shape_refiner.GetContext(node); + auto ctx = refiner.GetContext(node); if (!ctx) { continue; } -- GitLab From bdcca449fc22cf1d8a1d6a2c01c3b67706d6023b Mon Sep 17 00:00:00 2001 From: Jiri Simsa Date: Thu, 19 Apr 2018 10:14:09 -0700 Subject: [PATCH 655/791] Prototype for tf.data writer API. PiperOrigin-RevId: 193534333 --- .../contrib/data/python/kernel_tests/BUILD | 20 +++ .../python/kernel_tests/writer_ops_test.py | 117 ++++++++++++++++++ tensorflow/contrib/data/python/ops/BUILD | 13 ++ tensorflow/contrib/data/python/ops/writers.py | 58 +++++++++ .../base_api/api_def_DatasetToTFRecord.pbtxt | 24 ++++ tensorflow/core/framework/dataset.h | 4 +- tensorflow/core/kernels/data/BUILD | 14 +++ tensorflow/core/kernels/data/writer_ops.cc | 113 +++++++++++++++++ tensorflow/core/ops/dataset_ops.cc | 6 + 9 files changed, 367 insertions(+), 2 deletions(-) create mode 100644 tensorflow/contrib/data/python/kernel_tests/writer_ops_test.py create mode 100644 tensorflow/contrib/data/python/ops/writers.py create mode 100644 tensorflow/core/api_def/base_api/api_def_DatasetToTFRecord.pbtxt create mode 100644 tensorflow/core/kernels/data/writer_ops.cc diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD index c554607960..83daa04efc 100644 --- a/tensorflow/contrib/data/python/kernel_tests/BUILD +++ b/tensorflow/contrib/data/python/kernel_tests/BUILD @@ -516,3 +516,23 @@ tf_py_test( "//third_party/py/numpy", ], ) + +tf_py_test( + name = "writer_ops_test", + size = "small", + srcs = ["writer_ops_test.py"], + additional_deps = [ + "//tensorflow/contrib/data/python/ops:writers", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:dataset_ops_gen", + "//tensorflow/python:dtypes", + "//tensorflow/python:errors", + "//tensorflow/python:framework_ops", + "//tensorflow/python:io_ops", + "//tensorflow/python:lib", + "//tensorflow/python:tensor_shape", + "//tensorflow/python:util", + "//tensorflow/python/data/ops:readers", + ], +) diff --git a/tensorflow/contrib/data/python/kernel_tests/writer_ops_test.py b/tensorflow/contrib/data/python/kernel_tests/writer_ops_test.py new file mode 100644 index 0000000000..c603ecc5ab --- /dev/null +++ b/tensorflow/contrib/data/python/kernel_tests/writer_ops_test.py @@ -0,0 +1,117 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for the experimental input pipeline ops.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +from tensorflow.contrib.data.python.ops import writers +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.data.ops import readers +from tensorflow.python.framework import dtypes +from tensorflow.python.lib.io import python_io +from tensorflow.python.lib.io import tf_record +from tensorflow.python.ops import array_ops +from tensorflow.python.platform import test +from tensorflow.python.util import compat + + +class TFRecordWriterTest(test.TestCase): + + def setUp(self): + super(TFRecordWriterTest, self).setUp() + self._num_records = 7 + self.filename = array_ops.placeholder(dtypes.string, shape=[]) + self.compression_type = array_ops.placeholder_with_default("", shape=[]) + + input_dataset = readers.TFRecordDataset([self.filename], + self.compression_type) + self.writer = writers.TFRecordWriter( + self._outputFilename(), self.compression_type).write(input_dataset) + + def _record(self, i): + return compat.as_bytes("Record %d" % (i)) + + def _createFile(self, options=None): + filename = self._inputFilename() + writer = python_io.TFRecordWriter(filename, options) + for i in range(self._num_records): + writer.write(self._record(i)) + writer.close() + return filename + + def _inputFilename(self): + return os.path.join(self.get_temp_dir(), "tf_record.in.txt") + + def _outputFilename(self): + return os.path.join(self.get_temp_dir(), "tf_record.out.txt") + + def testWrite(self): + with self.test_session() as sess: + sess.run( + self.writer, feed_dict={ + self.filename: self._createFile(), + }) + for i, r in enumerate(tf_record.tf_record_iterator(self._outputFilename())): + self.assertAllEqual(self._record(i), r) + + def testWriteZLIB(self): + options = tf_record.TFRecordOptions(tf_record.TFRecordCompressionType.ZLIB) + with self.test_session() as sess: + sess.run( + self.writer, + feed_dict={ + self.filename: self._createFile(options), + self.compression_type: "ZLIB", + }) + for i, r in enumerate( + tf_record.tf_record_iterator(self._outputFilename(), options=options)): + self.assertAllEqual(self._record(i), r) + + def testWriteGZIP(self): + options = tf_record.TFRecordOptions(tf_record.TFRecordCompressionType.GZIP) + with self.test_session() as sess: + sess.run( + self.writer, + feed_dict={ + self.filename: self._createFile(options), + self.compression_type: "GZIP", + }) + for i, r in enumerate( + tf_record.tf_record_iterator(self._outputFilename(), options=options)): + self.assertAllEqual(self._record(i), r) + + def testFailDataset(self): + with self.assertRaises(TypeError): + writers.TFRecordWriter(self._outputFilename(), + self.compression_type).write("whoops") + + def testFailDType(self): + input_dataset = dataset_ops.Dataset.from_tensors(10) + with self.assertRaises(TypeError): + writers.TFRecordWriter(self._outputFilename(), + self.compression_type).write(input_dataset) + + def testFailShape(self): + input_dataset = dataset_ops.Dataset.from_tensors([["hello"], ["world"]]) + with self.assertRaises(TypeError): + writers.TFRecordWriter(self._outputFilename(), + self.compression_type).write(input_dataset) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/contrib/data/python/ops/BUILD b/tensorflow/contrib/data/python/ops/BUILD index e00f2304cc..5b04c5316c 100644 --- a/tensorflow/contrib/data/python/ops/BUILD +++ b/tensorflow/contrib/data/python/ops/BUILD @@ -280,6 +280,18 @@ py_library( ], ) +py_library( + name = "writers", + srcs = [ + "writers.py", + ], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:dtypes", + "//tensorflow/python/data/ops:dataset_ops", + ], +) + tf_gen_op_wrapper_py( name = "gen_dataset_ops", out = "gen_dataset_ops.py", @@ -342,6 +354,7 @@ py_library( ":stats_ops", ":threadpool", ":unique", + ":writers", "//tensorflow/python:dataset_ops_gen", "//tensorflow/python:util", "//tensorflow/python/data/ops:dataset_ops", diff --git a/tensorflow/contrib/data/python/ops/writers.py b/tensorflow/contrib/data/python/ops/writers.py new file mode 100644 index 0000000000..f53bd3f738 --- /dev/null +++ b/tensorflow/contrib/data/python/ops/writers.py @@ -0,0 +1,58 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Python wrappers for tf.data writers.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.data.util import convert +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_shape +from tensorflow.python.ops import gen_dataset_ops + + +class TFRecordWriter(object): + """Writes data to a TFRecord file.""" + + def __init__(self, filename, compression_type=None): + self._filename = ops.convert_to_tensor( + filename, dtypes.string, name="filename") + self._compression_type = convert.optional_param_to_tensor( + "compression_type", + compression_type, + argument_default="", + argument_dtype=dtypes.string) + + def write(self, dataset): + """Returns a @{tf.Operation} to write a dataset to a file. + + Args: + dataset: a @{tf.data.Dataset} whose elements are to be written to a file + + Returns: + A @{tf.Operation} that, when run, writes contents of `dataset` to a file. + """ + if not isinstance(dataset, dataset_ops.Dataset): + raise TypeError("`dataset` must be a `tf.data.Dataset` object.") + if (dataset.output_types != dtypes.string or + dataset.output_shapes != tensor_shape.scalar()): + raise TypeError( + "`dataset` must produce scalar `DT_STRING` tensors whereas it " + "produces shape {0} and types {1}".format(dataset.output_shapes, + dataset.output_types)) + return gen_dataset_ops.dataset_to_tf_record( + dataset._as_variant_tensor(), self._filename, self._compression_type) # pylint: disable=protected-access diff --git a/tensorflow/core/api_def/base_api/api_def_DatasetToTFRecord.pbtxt b/tensorflow/core/api_def/base_api/api_def_DatasetToTFRecord.pbtxt new file mode 100644 index 0000000000..e1b8a9abdd --- /dev/null +++ b/tensorflow/core/api_def/base_api/api_def_DatasetToTFRecord.pbtxt @@ -0,0 +1,24 @@ +op { + graph_op_name: "DatasetToTFRecord" + visibility: HIDDEN + in_arg { + name: "input_dataset" + description: <& parent) { return parent->SaveInternal(writer); @@ -372,7 +372,7 @@ class IteratorBase { // This is needed so that sub-classes of IteratorBase can call // `RestoreInternal` on their parent iterators, e.g., in - // `RepeatDataasetOp::Dataset`. + // `RepeatDatasetOp::Dataset`. Status RestoreParent(IteratorContext* ctx, IteratorStateReader* reader, const std::unique_ptr& parent) { return parent->RestoreInternal(ctx, reader); diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD index 1e96eb6421..667a6967a8 100644 --- a/tensorflow/core/kernels/data/BUILD +++ b/tensorflow/core/kernels/data/BUILD @@ -576,6 +576,20 @@ tf_kernel_library( ":tensor_queue_dataset_op", ":tensor_slice_dataset_op", ":unbatch_dataset_op", + ":writer_ops", ":zip_dataset_op", ], ) + +tf_kernel_library( + name = "writer_ops", + srcs = ["writer_ops.cc"], + deps = [ + ":dataset", + "//tensorflow/core:core_cpu_internal", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + "//tensorflow/core/kernels:ops_util", + ], +) diff --git a/tensorflow/core/kernels/data/writer_ops.cc b/tensorflow/core/kernels/data/writer_ops.cc new file mode 100644 index 0000000000..46821fd7b3 --- /dev/null +++ b/tensorflow/core/kernels/data/writer_ops.cc @@ -0,0 +1,113 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/common_runtime/device.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/kernels/data/dataset.h" +#include "tensorflow/core/kernels/ops_util.h" +#include "tensorflow/core/lib/core/threadpool.h" +#include "tensorflow/core/lib/io/record_writer.h" +#include "tensorflow/core/platform/file_system.h" + +namespace tensorflow { + +namespace { + +class ToTFRecordOp : public AsyncOpKernel { + public: + explicit ToTFRecordOp(OpKernelConstruction* ctx) + : AsyncOpKernel(ctx), + thread_pool_(new thread::ThreadPool( + ctx->env(), ThreadOptions(), + strings::StrCat("to_tf_record__op_", SanitizeThreadSuffix(name())), + 1 /* num_threads */, false /* low_latency_hint */)) {} + + template + Status ParseScalarArgument(OpKernelContext* ctx, + const StringPiece& argument_name, T* output) { + const Tensor* argument_t; + TF_RETURN_IF_ERROR(ctx->input(argument_name, &argument_t)); + if (!TensorShapeUtils::IsScalar(argument_t->shape())) { + return errors::InvalidArgument(argument_name, " must be a scalar"); + } + *output = argument_t->scalar()(); + return Status::OK(); + } + + void ComputeAsync(OpKernelContext* ctx, DoneCallback done) override { + // The call to `iterator->GetNext()` may block and depend on an + // inter-op thread pool thread, so we issue the call from the + // owned thread pool. + thread_pool_->Schedule([this, ctx, done]() { + string filename; + OP_REQUIRES_OK_ASYNC( + ctx, ParseScalarArgument(ctx, "filename", &filename), done); + string compression_type; + OP_REQUIRES_OK_ASYNC(ctx, + ParseScalarArgument(ctx, "compression_type", + &compression_type), + done); + std::unique_ptr file; + OP_REQUIRES_OK_ASYNC(ctx, ctx->env()->NewWritableFile(filename, &file), + done); + std::unique_ptr writer; + writer.reset(new io::RecordWriter( + file.get(), io::RecordWriterOptions::CreateRecordWriterOptions( + compression_type))); + + DatasetBase* dataset; + OP_REQUIRES_OK_ASYNC( + ctx, GetDatasetFromVariantTensor(ctx->input(0), &dataset), done); + auto iterator = dataset->MakeIterator("ToTFRecordOpIterator"); + + IteratorContext::Params params; // TODO(b/78245447) + params.env = ctx->env(); + params.runner = *(ctx->runner()); + params.lib = ctx->function_library(); + DeviceBase* device = ctx->function_library()->device(); + params.allocator_getter = [device](AllocatorAttributes attrs) { + return device->GetAllocator(attrs); + }; + + IteratorContext iter_ctx(std::move(params)); + + std::vector components; + components.reserve(dataset->output_dtypes().size()); + bool end_of_sequence; + + do { + OP_REQUIRES_OK_ASYNC( + ctx, iterator->GetNext(&iter_ctx, &components, &end_of_sequence), + done); + + if (!end_of_sequence) { + OP_REQUIRES_OK_ASYNC( + ctx, writer->WriteRecord(components[0].scalar()()), done); + } + components.clear(); + } while (!end_of_sequence); + done(); + }); + } + + private: + std::unique_ptr thread_pool_; +}; + +REGISTER_KERNEL_BUILDER(Name("DatasetToTFRecord").Device(DEVICE_CPU), + ToTFRecordOp); + +} // namespace +} // namespace tensorflow diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc index 8be569b315..67c6c58fe2 100644 --- a/tensorflow/core/ops/dataset_ops.cc +++ b/tensorflow/core/ops/dataset_ops.cc @@ -551,4 +551,10 @@ REGISTER_OP("EnqueueInQueueDataset") // reading from queue handle (is that even possible?). .SetShapeFn(shape_inference::NoOutputs); +REGISTER_OP("DatasetToTFRecord") + .Input("input_dataset: variant") + .Input("filename: string") + .Input("compression_type: string") + .SetShapeFn(shape_inference::NoOutputs); + } // namespace tensorflow -- GitLab From 5fbd21e3bbd4f89dd2c6eed8a63b66ee2eff40a0 Mon Sep 17 00:00:00 2001 From: Ian Langmore Date: Thu, 19 Apr 2018 10:20:43 -0700 Subject: [PATCH 656/791] distribution_util moved into its own BUILD target, so linear_operator can depend on it. PiperOrigin-RevId: 193535400 --- tensorflow/python/ops/distributions/BUILD | 26 ++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/ops/distributions/BUILD b/tensorflow/python/ops/distributions/BUILD index 9d9ede7ad7..e7ad028376 100644 --- a/tensorflow/python/ops/distributions/BUILD +++ b/tensorflow/python/ops/distributions/BUILD @@ -8,9 +8,13 @@ licenses(["notice"]) # Apache 2.0 py_library( name = "distributions", - srcs = glob(["*.py"]), + srcs = glob( + ["*.py"], + exclude = ["util.py"], + ), srcs_version = "PY2AND3", deps = [ + ":util", "//tensorflow/python:array_ops", "//tensorflow/python:check_ops", "//tensorflow/python:control_flow_ops", @@ -26,3 +30,23 @@ py_library( "@six_archive//:six", ], ) + +py_library( + name = "util", + srcs = ["util.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:array_ops", + "//tensorflow/python:check_ops", + "//tensorflow/python:control_flow_ops", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:math_ops", + "//tensorflow/python:nn", + "//tensorflow/python:nn_ops", + "//tensorflow/python:random_ops", + "//tensorflow/python:special_math_ops", + "//tensorflow/python:tensor_util", + "//third_party/py/numpy", + "@six_archive//:six", + ], +) -- GitLab From 72240a9b5e67e315f6c037bb4579df9709335e35 Mon Sep 17 00:00:00 2001 From: imsheridan Date: Fri, 20 Apr 2018 01:23:54 +0800 Subject: [PATCH 657/791] fix single paragraph format and also arrow like format --- tensorflow/contrib/optimizer_v2/adam.py | 16 ++++++++-------- .../api_def/base_api/api_def_ApplyAdam.pbtxt | 8 ++++---- .../base_api/api_def_ResourceApplyAdam.pbtxt | 8 ++++---- tensorflow/python/training/adam.py | 16 ++++++++-------- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/tensorflow/contrib/optimizer_v2/adam.py b/tensorflow/contrib/optimizer_v2/adam.py index a38c98f471..76a867039a 100644 --- a/tensorflow/contrib/optimizer_v2/adam.py +++ b/tensorflow/contrib/optimizer_v2/adam.py @@ -40,19 +40,19 @@ class AdamOptimizer(optimizer_v2.OptimizerV2): Initialization: - \\(m_0 <- 0\\) (Initialize initial 1st moment vector) - \\(v_0 <- 0\\) (Initialize initial 2nd moment vector) - \\(t <- 0\\) (Initialize timestep) + $$m_0 \Leftarrow 0 (Initialize initial 1st moment vector)$$ + $$v_0 \Leftarrow 0 (Initialize initial 2nd moment vector)$$ + $$t \Leftarrow 0 (Initialize timestep)$$ The update rule for `variable` with gradient `g` uses an optimization described at the end of section2 of the paper: - $$t <- t + 1$$ - $$lr_t <- \text{learning_rate} * \sqrt{(1 - beta_2^t) / (1 - beta_1^t)}$$ + $$t \Leftarrow t + 1$$ + $$lr_t \Leftarrow \text{learning_rate} * \sqrt{(1 - beta_2^t) / (1 - beta_1^t)}$$ - $$m_t <- beta_1 * m_{t-1} + (1 - beta_1) * g$$ - $$v_t <- beta_2 * v_{t-1} + (1 - beta_2) * g * g$$ - $$variable <- variable - lr_t * m_t / (\sqrt{v_t} + \epsilon)$$ + $$m_t \Leftarrow beta_1 * m_{t-1} + (1 - beta_1) * g$$ + $$v_t \Leftarrow beta_2 * v_{t-1} + (1 - beta_2) * g * g$$ + $$variable \Leftarrow variable - lr_t * m_t / (\sqrt{v_t} + \epsilon)$$ The default value of 1e-8 for epsilon might not be a good default in general. For example, when training an Inception network on ImageNet a diff --git a/tensorflow/core/api_def/base_api/api_def_ApplyAdam.pbtxt b/tensorflow/core/api_def/base_api/api_def_ApplyAdam.pbtxt index fc2cb09471..fca8ba2530 100644 --- a/tensorflow/core/api_def/base_api/api_def_ApplyAdam.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_ApplyAdam.pbtxt @@ -82,9 +82,9 @@ END } summary: "Update \'*var\' according to the Adam algorithm." description: < Date: Thu, 19 Apr 2018 10:26:26 -0700 Subject: [PATCH 658/791] Fix doc gen error Mismatch after the fix in #17815 --- tensorflow/contrib/tensor_forest/ops/stats_ops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/tensor_forest/ops/stats_ops.cc b/tensorflow/contrib/tensor_forest/ops/stats_ops.cc index be0a11546d..5be581aaec 100644 --- a/tensorflow/contrib/tensor_forest/ops/stats_ops.cc +++ b/tensorflow/contrib/tensor_forest/ops/stats_ops.cc @@ -75,7 +75,7 @@ REGISTER_OP("GrowTreeV4") .Attr("params: string") .Input("tree_handle: resource") .Input("stats_handle: resource") - .Input("finshed_nodes: int32") + .Input("finished_nodes: int32") .SetShapeFn(tensorflow::shape_inference::NoOutputs) .Doc(R"doc( Grows the tree for finished nodes and allocates waiting nodes. -- GitLab From ba3bc495bbf1140e9375e1ec03c3ff788b8ebc6e Mon Sep 17 00:00:00 2001 From: Anjali Sridhar Date: Thu, 19 Apr 2018 10:26:54 -0700 Subject: [PATCH 659/791] Add metric names to model.metrics_names in compile for keras models run in eager execution. This prevents us from dropping metrics when we run model.evaluate. PiperOrigin-RevId: 193536341 --- .../keras/_impl/keras/engine/training.py | 29 ++------- .../_impl/keras/engine/training_eager.py | 39 ++++-------- .../_impl/keras/engine/training_eager_test.py | 12 ++-- .../keras/_impl/keras/engine/training_test.py | 26 ++++++++ .../_impl/keras/engine/training_utils.py | 62 +++++++++++++++++++ 5 files changed, 109 insertions(+), 59 deletions(-) diff --git a/tensorflow/python/keras/_impl/keras/engine/training.py b/tensorflow/python/keras/_impl/keras/engine/training.py index 7c46743814..012d9ceea4 100644 --- a/tensorflow/python/keras/_impl/keras/engine/training.py +++ b/tensorflow/python/keras/_impl/keras/engine/training.py @@ -276,6 +276,8 @@ class Model(Network): self.metrics_names.append(self.output_names[i] + '_loss') self.nested_metrics = training_utils.collect_metrics(metrics, self.output_names) + with K.name_scope('metrics'): + training_utils.populate_metric_names(self) self._feed_sample_weight_modes = [] for i in range(len(self.outputs)): self._feed_sample_weight_modes.append(None) @@ -462,7 +464,6 @@ class Model(Network): output_weighted_metrics = nested_weighted_metrics[i] def handle_metrics(metrics, weights=None): - metric_name_prefix = 'weighted_' if weights is not None else '' for metric in metrics: if metric in ('accuracy', 'acc', 'crossentropy', 'ce'): @@ -489,39 +490,19 @@ class Model(Network): metric_fn = metrics_module.categorical_accuracy elif metric in ('crossentropy', 'ce'): metric_fn = metrics_module.categorical_crossentropy - if metric in ('accuracy', 'acc'): - suffix = 'acc' - elif metric in ('crossentropy', 'ce'): - suffix = 'ce' weighted_metric_fn = training_utils.weighted_masked_objective( metric_fn) - metric_name = metric_name_prefix + suffix else: metric_fn = metrics_module.get(metric) weighted_metric_fn = training_utils.weighted_masked_objective( metric_fn) - # Get metric name as string - if hasattr(metric_fn, 'name'): - metric_name = metric_fn.name - else: - metric_name = metric_fn.__name__ - metric_name = metric_name_prefix + metric_name - + metric_name = training_utils.get_base_metric_name( + metric, weighted=weights is not None) with K.name_scope(metric_name): metric_result = weighted_metric_fn( y_true, y_pred, weights=weights, mask=masks[i]) - # Append to self.metrics_names, self.metric_tensors, - # self.stateful_metric_names - if len(self.output_names) > 1: - metric_name = '%s_%s' % (self.output_names[i], metric_name) - # Dedupe name - j = 1 - base_metric_name = metric_name - while metric_name in self.metrics_names: - metric_name = '%s_%d' % (base_metric_name, j) - j += 1 - self.metrics_names.append(metric_name) + training_utils.add_metric_name(self, metric_name, i) self.metrics_tensors.append(metric_result) # Keep track of state updates created by diff --git a/tensorflow/python/keras/_impl/keras/engine/training_eager.py b/tensorflow/python/keras/_impl/keras/engine/training_eager.py index 695669d9ee..ad239d6151 100644 --- a/tensorflow/python/keras/_impl/keras/engine/training_eager.py +++ b/tensorflow/python/keras/_impl/keras/engine/training_eager.py @@ -100,7 +100,7 @@ def _eager_metrics_fn(model, outputs, targets): metric_names.append(metric_name) metric_results.append(backend.mean(metric_result)) - return metric_names, metric_results + return metric_results def _model_loss(model, inputs, targets, sample_weights=None, training=False): @@ -151,7 +151,12 @@ def _model_loss(model, inputs, targets, sample_weights=None, training=False): with backend.name_scope(model.output_names[i] + '_loss'): output_loss = weighted_masked_fn( targets[i], outs[i], weights, mask=mask) - loss_metrics.append(backend.mean(output_loss)) + # If the number of outputs is 1 then we don't append the loss metric + # associated with each model output. When there are multiple outputs + # associated with a model, each output's loss is calculated and returned + # as part of the loss_metrics. + if len(model.outputs) > 1: + loss_metrics.append(backend.mean(output_loss)) loss_weight = model.loss_weights_list[i] if total_loss is None: @@ -274,7 +279,7 @@ def train_on_batch(model, inputs, targets, sample_weights=None): model, inputs, targets, sample_weights=sample_weights, training=True) if not isinstance(outs, list): outs = [outs] - _, metrics_results = _eager_metrics_fn( + metrics_results = _eager_metrics_fn( model, outs, targets) if not isinstance(loss, list): loss = [loss] @@ -304,7 +309,7 @@ def test_on_batch(model, inputs, targets, sample_weights=None): model, inputs, targets, sample_weights=sample_weights, training=False) if not isinstance(outs, list): outs = [outs] - _, metrics_results = _eager_metrics_fn( + metrics_results = _eager_metrics_fn( model, outs, targets) if not isinstance(loss, list): loss = [loss] @@ -498,34 +503,12 @@ def fit_loop( for l, o in zip(out_labels, outs): batch_logs[l] = o # Required for Eager mode - metrics_names, metrics_results = _eager_metrics_fn( - model, outs, targets_batch) + metrics_results = _eager_metrics_fn(model, outs, targets_batch) batch_logs['loss'] = tensor_util.constant_value(backend.mean(loss)) - # TODO(anjalisridhar): Move this to compile to avoid duplicate code. - # In graph mode we set the metric names in compile. However in - # Eager mode we calculate the metrics for each batch in fit_loop. - # We could calculate the metric names and functions in compile. - # This would avoid setting the callback parameters separately. - # We need to do this for the first iteration alone - for m in metrics_names: - if m not in callback_metrics: - callback_metrics.append(m) - - callbacks.set_params({ - 'batch_size': batch_size, - 'epochs': epochs, - 'steps': steps_per_epoch, - 'samples': num_train_samples, - 'verbose': verbose, - 'do_validation': do_validation, - 'metrics': callback_metrics or [], - }) - for k, v in zip(model.metrics_names, [backend.mean(loss)] + loss_metrics + metrics_results): batch_logs[k] = tensor_util.constant_value(v) - callbacks.on_batch_end(batch_index, batch_logs) if callback_model.stop_training: break @@ -611,7 +594,7 @@ def test_loop(model, inputs, targets, targets_batch, sample_weights=sample_weights_batch, training=False) - _, metrics_results = _eager_metrics_fn(model, loss_outs, targets_batch) + metrics_results = _eager_metrics_fn(model, loss_outs, targets_batch) batch_outs = [] for _, v in zip(model.metrics_names, [backend.mean(loss)] + loss_metrics + metrics_results): diff --git a/tensorflow/python/keras/_impl/keras/engine/training_eager_test.py b/tensorflow/python/keras/_impl/keras/engine/training_eager_test.py index ed0f91ee1e..deaf1d1306 100644 --- a/tensorflow/python/keras/_impl/keras/engine/training_eager_test.py +++ b/tensorflow/python/keras/_impl/keras/engine/training_eager_test.py @@ -212,7 +212,7 @@ class TrainingTest(test.TestCase): optimizer = RMSPropOptimizer(learning_rate=0.001) loss = 'mse' loss_weights = [1., 0.5] - metrics = ['mae'] + metrics = ['acc', 'mae'] model.compile( optimizer, loss, @@ -231,20 +231,20 @@ class TrainingTest(test.TestCase): [input_a_np, input_b_np], [output_d_np, output_e_np], batch_size=5, verbose=0) - self.assertEqual(len(out), 5) + self.assertEqual(len(out), 7) out = model.evaluate( [input_a_np, input_b_np], [output_d_np, output_e_np], batch_size=5, verbose=1) - self.assertEqual(len(out), 5) + self.assertEqual(len(out), 7) out = model.evaluate( [input_a_np, input_b_np], [output_d_np, output_e_np], batch_size=5, verbose=2) - self.assertEqual(len(out), 5) + self.assertEqual(len(out), 7) out = model.test_on_batch([input_a_np, input_b_np], [output_d_np, output_e_np]) - self.assertEqual(len(out), 5) + self.assertEqual(len(out), 7) # Test evaluate with dictionary inputs model.evaluate( @@ -625,7 +625,6 @@ class LossWeightingTest(test.TestCase): bad_w_np = np.random.random((10, 2, 2)) model.fit(x_np, [y_np, y_np], epochs=1, sample_weight={'1': bad_w_np}) - class CorrectnessTest(test.TestCase): @tf_test_util.run_in_graph_and_eager_modes() @@ -649,7 +648,6 @@ class CorrectnessTest(test.TestCase): self.assertEqual( np.around(history.history['loss'][-1], decimals=4), 0.6173) - if __name__ == '__main__': ops.enable_eager_execution() test.main() diff --git a/tensorflow/python/keras/_impl/keras/engine/training_test.py b/tensorflow/python/keras/_impl/keras/engine/training_test.py index 6699fd5212..d9281436de 100644 --- a/tensorflow/python/keras/_impl/keras/engine/training_test.py +++ b/tensorflow/python/keras/_impl/keras/engine/training_test.py @@ -24,12 +24,15 @@ import unittest import numpy as np from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util as tf_test_util from tensorflow.python.keras._impl import keras from tensorflow.python.keras._impl.keras import testing_utils from tensorflow.python.keras._impl.keras.engine.training_utils import weighted_masked_objective from tensorflow.python.keras._impl.keras.utils.generic_utils import slice_arrays from tensorflow.python.ops import array_ops from tensorflow.python.platform import test +from tensorflow.python.training.rmsprop import RMSPropOptimizer + try: import scipy.sparse as scipy_sparse # pylint: disable=g-import-not-at-top @@ -1684,6 +1687,29 @@ class TestTrainingWithDataTensors(test.TestCase): model.train_on_batch([input_a_np, input_b_np], [output_a_np, output_b_np]) + @tf_test_util.run_in_graph_and_eager_modes() + def test_metric_names_are_identical_in_graph_and_eager(self): + a = keras.layers.Input(shape=(3,), name='input_a') + b = keras.layers.Input(shape=(3,), name='input_b') + + dense = keras.layers.Dense(4, name='dense') + c = dense(a) + d = dense(b) + e = keras.layers.Dropout(0.5, name='dropout')(c) + + model = keras.models.Model([a, b], [d, e]) + + optimizer = RMSPropOptimizer(learning_rate=0.001) + loss = 'mse' + loss_weights = [1., 0.5] + metrics = ['mae', 'acc'] + model.compile(optimizer, loss, metrics=metrics, loss_weights=loss_weights) + reference_metric_names = ['loss', 'dense_loss', 'dropout_loss', + 'dense_mean_absolute_error', + 'dense_acc', + 'dropout_mean_absolute_error', + 'dropout_acc'] + self.assertEqual(reference_metric_names, model.metrics_names) if __name__ == '__main__': # Bazel sets these environment variables to very long paths. diff --git a/tensorflow/python/keras/_impl/keras/engine/training_utils.py b/tensorflow/python/keras/_impl/keras/engine/training_utils.py index 48afe48e6c..662938f421 100644 --- a/tensorflow/python/keras/_impl/keras/engine/training_utils.py +++ b/tensorflow/python/keras/_impl/keras/engine/training_utils.py @@ -26,6 +26,7 @@ from tensorflow.python.eager import context from tensorflow.python.framework import tensor_util from tensorflow.python.keras._impl.keras import backend as K from tensorflow.python.keras._impl.keras import losses +from tensorflow.python.keras._impl.keras import metrics as metrics_module from tensorflow.python.ops import math_ops @@ -552,3 +553,64 @@ def standardize_weights(y, def has_symbolic_tensors(ls): return (any(tensor_util.is_tensor(v) for v in ls) and not context.executing_eagerly()) + + +def populate_metric_names(model): + for i in range(len(model.outputs)): + metrics = model.nested_metrics[i] + for metric in metrics: + base_metric_name = get_base_metric_name(metric) + add_metric_name(model, base_metric_name, i) + + +def get_base_metric_name(metric, weighted=False): + """Returns the metric name given the metric function. + + Arguments: + metric: Metric function name or reference. + weighted: Boolean indicating if the metric for which we are adding + names is weighted. + + Returns: + a metric name. + """ + metric_name_prefix = 'weighted_' if weighted else '' + if metric in ('accuracy', 'acc', 'crossentropy', 'ce'): + if metric in ('accuracy', 'acc'): + suffix = 'acc' + elif metric in ('crossentropy', 'ce'): + suffix = 'ce' + metric_name = metric_name_prefix + suffix + else: + metric_fn = metrics_module.get(metric) + # Get metric name as string + if hasattr(metric_fn, 'name'): + metric_name = metric_fn.name + else: + metric_name = metric_fn.__name__ + metric_name = metric_name_prefix + metric_name + + return metric_name + + +def add_metric_name(model, metric_name, index): + """Makes the metric name unique and adds it to the model's metric name list. + + If there are multiple outputs for which the metrics are calculated, the + metric names have to be made unique by appending an integer. + + Arguments: + model: Model to which we are adding metric names. + metric_name: Metric name that corresponds to the metric specified by the + user. For example: 'acc' + index: The index of the model output for which the metric name is being + added. + """ + if len(model.output_names) > 1: + metric_name = '%s_%s' % (model.output_names[index], metric_name) + j = 1 + base_metric_name = metric_name + while metric_name in model.metrics_names: + metric_name = '%s_%d' % (base_metric_name, j) + j += 1 + model.metrics_names.append(metric_name) -- GitLab From 6a7779f3384e48012d3e27ae0f48d410f5174d06 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 10:33:42 -0700 Subject: [PATCH 660/791] Fix undefined signed integer overflow by performing addition more carefully. PiperOrigin-RevId: 193537461 --- .../core/lib/random/random_distributions.h | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/lib/random/random_distributions.h b/tensorflow/core/lib/random/random_distributions.h index 4cf3a999f6..e963511f5c 100644 --- a/tensorflow/core/lib/random/random_distributions.h +++ b/tensorflow/core/lib/random/random_distributions.h @@ -23,6 +23,7 @@ limitations under the License. #include #include +#include #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/lib/bfloat16/bfloat16.h" @@ -40,6 +41,20 @@ PHILOX_DEVICE_INLINE float Uint32ToFloat(uint32 x); // Helper function to convert two 32-bit integers to a double between [0..1). PHILOX_DEVICE_INLINE double Uint64ToDouble(uint32 x0, uint32 x1); +// Computes a + b. Requires that the result is representable in the destination +// type and that b is not maximal (i.e. b + 1 is not 0). Notably, the addend b +// need *not* be representable in that type. (The condition on b excludes the +// extremal case INT_MIN + UINT_MAX = INT_MAX, which this function cannot +// compute.) +template +PHILOX_DEVICE_INLINE Int SignedAdd(Int a, + typename std::make_unsigned::type b) { + // Implementation note: both b_div_2 and b - b_div_2 are positive and + // representatble as Int. + auto b_div_2 = b >> 1; + return a + static_cast(b_div_2) + static_cast(b - b_div_2); +} + // A class that generates uniform distribution random numbers from the // underlying random integer generator. // Arguments: @@ -172,7 +187,7 @@ class UniformDistribution { typename Generator::ResultType sample = (*gen)(); ResultType result; for (int i = 0; i < kResultElementCount; ++i) { - result[i] = lo_ + static_cast(sample[i] % range_); + result[i] = SignedAdd(lo_, sample[i] % range_); } return result; } @@ -208,7 +223,7 @@ class UniformDistribution { ResultType result; for (int i = 0; i < kResultElementCount; ++i) { auto bits = sample[2 * i] | static_cast(sample[2 * i + 1]) << 32; - result[i] = lo_ + static_cast(bits % range_); + result[i] = SignedAdd(lo_, bits % range_); } return result; } -- GitLab From 430230b4b966cade863ea5b660862734ede1cc56 Mon Sep 17 00:00:00 2001 From: imsheridan Date: Fri, 20 Apr 2018 01:37:03 +0800 Subject: [PATCH 661/791] Fix minor pylint issue --- tensorflow/contrib/losses/python/losses/loss_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/losses/python/losses/loss_ops.py b/tensorflow/contrib/losses/python/losses/loss_ops.py index 5af1f21b11..bdad34a665 100644 --- a/tensorflow/contrib/losses/python/losses/loss_ops.py +++ b/tensorflow/contrib/losses/python/losses/loss_ops.py @@ -652,7 +652,7 @@ def cosine_distance(predictions, ValueError: If `predictions` shape doesn't match `labels` shape, or `weights` is `None`. """ - axis = deprecation.deprecated_argument_lookup( + axis = deprecated_argument_lookup( "axis", axis, "dim", dim) if axis is None: raise ValueError("You must specify 'axis'.") -- GitLab From f196351cd4e21ed6c17dcf544e0fa6cfa3030b4e Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Thu, 19 Apr 2018 10:57:55 -0700 Subject: [PATCH 662/791] Allow non-isolated worker sessions to borrow `WorkerEnv::device_mgr`. Without this change, a shared resource (e.g. an Iterator) could not be created in one session `s1`, and used in a later session `s2` after `s1` was closed, because the iterator might indirectly capture devices from the previous session, and use them after they are freed when the `WorkerSession` was deleted. The current change only affects the singleton "legacy" WorkerSession, which is never deleted, but this is necessary to switch all sessions to use separate WorkerSession objects. PiperOrigin-RevId: 193541426 --- tensorflow/contrib/gdr/gdr_rendezvous_mgr.cc | 2 +- tensorflow/core/distributed_runtime/BUILD | 1 + .../base_rendezvous_mgr.cc | 4 +- .../rpc/rpc_rendezvous_mgr.cc | 2 +- .../core/distributed_runtime/session_mgr.cc | 40 +++++++++++++------ .../core/distributed_runtime/session_mgr.h | 2 +- .../distributed_runtime/session_mgr_test.cc | 23 ++++++----- .../distributed_runtime/worker_session.cc | 38 +++++++++++++++++- .../core/distributed_runtime/worker_session.h | 28 +++++++++++-- 9 files changed, 105 insertions(+), 35 deletions(-) diff --git a/tensorflow/contrib/gdr/gdr_rendezvous_mgr.cc b/tensorflow/contrib/gdr/gdr_rendezvous_mgr.cc index 28f68cec8c..94f522c04e 100644 --- a/tensorflow/contrib/gdr/gdr_rendezvous_mgr.cc +++ b/tensorflow/contrib/gdr/gdr_rendezvous_mgr.cc @@ -155,7 +155,7 @@ class GdrRemoteRendezvous : public BaseRemoteRendezvous { } Device* dst_device; - Status s = sess->device_mgr->LookupDevice(parsed.dst_device, &dst_device); + Status s = sess->device_mgr()->LookupDevice(parsed.dst_device, &dst_device); if (!s.ok()) { sess->worker_cache->ReleaseWorker(src_worker, rwi); done(s, Args(), recv_args, Tensor{}, false); diff --git a/tensorflow/core/distributed_runtime/BUILD b/tensorflow/core/distributed_runtime/BUILD index b07cb8cdcb..d564727da5 100644 --- a/tensorflow/core/distributed_runtime/BUILD +++ b/tensorflow/core/distributed_runtime/BUILD @@ -133,6 +133,7 @@ cc_library( "//tensorflow/core:core_cpu_internal", "//tensorflow/core:lib", "//tensorflow/core:protos_all_cc", + "//tensorflow/core:ptr_util", "//tensorflow/core:worker_proto_cc", ], ) diff --git a/tensorflow/core/distributed_runtime/base_rendezvous_mgr.cc b/tensorflow/core/distributed_runtime/base_rendezvous_mgr.cc index bafd9bfc68..5f6931e008 100644 --- a/tensorflow/core/distributed_runtime/base_rendezvous_mgr.cc +++ b/tensorflow/core/distributed_runtime/base_rendezvous_mgr.cc @@ -253,13 +253,13 @@ void BaseRemoteRendezvous::SameWorkerRecvDone( WorkerSession* sess = session(); Device* src_device; - Status s = sess->device_mgr->LookupDevice(parsed.src_device, &src_device); + Status s = sess->device_mgr()->LookupDevice(parsed.src_device, &src_device); if (!s.ok()) { done(s); return; } Device* dst_device; - s = sess->device_mgr->LookupDevice(parsed.dst_device, &dst_device); + s = sess->device_mgr()->LookupDevice(parsed.dst_device, &dst_device); if (!s.ok()) { done(s); return; diff --git a/tensorflow/core/distributed_runtime/rpc/rpc_rendezvous_mgr.cc b/tensorflow/core/distributed_runtime/rpc/rpc_rendezvous_mgr.cc index 067dc5dff5..b8cb538503 100644 --- a/tensorflow/core/distributed_runtime/rpc/rpc_rendezvous_mgr.cc +++ b/tensorflow/core/distributed_runtime/rpc/rpc_rendezvous_mgr.cc @@ -227,7 +227,7 @@ void RpcRemoteRendezvous::RecvFromRemoteAsync( Device* dst_device; if (s.ok()) { - s = sess->device_mgr->LookupDevice(parsed.dst_device, &dst_device); + s = sess->device_mgr()->LookupDevice(parsed.dst_device, &dst_device); } if (!s.ok()) { if (rwi != nullptr) { diff --git a/tensorflow/core/distributed_runtime/session_mgr.cc b/tensorflow/core/distributed_runtime/session_mgr.cc index e51d63cf2b..357e9f8930 100644 --- a/tensorflow/core/distributed_runtime/session_mgr.cc +++ b/tensorflow/core/distributed_runtime/session_mgr.cc @@ -24,6 +24,7 @@ limitations under the License. #include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/protobuf/cluster.pb.h" #include "tensorflow/core/protobuf/tensorflow_server.pb.h" +#include "tensorflow/core/util/ptr_util.h" namespace tensorflow { @@ -33,11 +34,11 @@ SessionMgr::SessionMgr( WorkerCacheFactory worker_cache_factory) : worker_env_(worker_env), default_worker_cache_(std::move(default_worker_cache)), - legacy_session_(new WorkerSession( + legacy_session_(WorkerSession::CreateWithBorrowedDeviceMgr( "", default_worker_name, std::unique_ptr( new WorkerCacheWrapper(default_worker_cache_.get())), - std::unique_ptr(worker_env->device_mgr), + worker_env->device_mgr, std::unique_ptr( new GraphMgr(worker_env, worker_env->device_mgr)))), worker_cache_factory_(std::move(worker_cache_factory)) {} @@ -71,19 +72,32 @@ Status SessionMgr::CreateSession(const string& session, CHECK(!worker_env_->local_devices.empty()) << "The WorkerEnv must have at least one device in `local_devices`."; - std::vector renamed_devices; - for (Device* d : worker_env_->local_devices) { - renamed_devices.push_back(RenamedDevice::NewRenamedDevice( - worker_name, d, false, isolate_session_state)); - } - std::unique_ptr device_mgr(new DeviceMgr(renamed_devices)); + std::shared_ptr worker_session; - std::unique_ptr graph_mgr( - new GraphMgr(worker_env_, device_mgr.get())); + if (isolate_session_state) { + // Create a private copy of the DeviceMgr for the WorkerSession. + std::vector renamed_devices; + for (Device* d : worker_env_->local_devices) { + renamed_devices.push_back(RenamedDevice::NewRenamedDevice( + worker_name, d, false, isolate_session_state)); + } - std::shared_ptr worker_session(new WorkerSession( - session, worker_name, std::unique_ptr(worker_cache), - std::move(device_mgr), std::move(graph_mgr))); + auto device_mgr = MakeUnique(renamed_devices); + auto graph_mgr = MakeUnique(worker_env_, device_mgr.get()); + worker_session.reset( + new WorkerSession(session, worker_name, + std::unique_ptr(worker_cache), + std::move(device_mgr), std::move(graph_mgr))); + } else { + // Borrown the WorkerEnv's DeviceMgr for the WorkerSession, so + // that resources using it can use its devices after the + // WorkerSession has been deleted. + auto graph_mgr = MakeUnique(worker_env_, worker_env_->device_mgr); + worker_session = WorkerSession::CreateWithBorrowedDeviceMgr( + session, worker_name, + std::unique_ptr(worker_cache), + worker_env_->device_mgr, std::move(graph_mgr)); + } sessions_.insert(std::make_pair(session, std::move(worker_session))); return Status::OK(); diff --git a/tensorflow/core/distributed_runtime/session_mgr.h b/tensorflow/core/distributed_runtime/session_mgr.h index 0a10fe240f..04d1d61409 100644 --- a/tensorflow/core/distributed_runtime/session_mgr.h +++ b/tensorflow/core/distributed_runtime/session_mgr.h @@ -65,7 +65,7 @@ class SessionMgr { void ClearLogs(); private: - const WorkerEnv* const worker_env_; // Not owned. + WorkerEnv* const worker_env_; // Not owned. // A note about destruction: // We must delete graph_mgr before device_mgr, due to shared diff --git a/tensorflow/core/distributed_runtime/session_mgr_test.cc b/tensorflow/core/distributed_runtime/session_mgr_test.cc index 858e636e08..0da333833a 100644 --- a/tensorflow/core/distributed_runtime/session_mgr_test.cc +++ b/tensorflow/core/distributed_runtime/session_mgr_test.cc @@ -43,15 +43,17 @@ class FakeDevice : public Device { class SessionMgrTest : public ::testing::Test { protected: SessionMgrTest() - : device_(FakeDevice::MakeCPU( - "/job:mnist/replica:0/task:0/device:fakecpu:0")), - mgr_(&env_, "/job:mnist/replica:0/task:0", + : mgr_(&env_, "/job:mnist/replica:0/task:0", std::unique_ptr(), factory_) { - TF_CHECK_OK(mgr_.WorkerSessionForSession("", &legacy_session_)); - env_.local_devices = {device_.get()}; + Device* device = + FakeDevice::MakeCPU("/job:mnist/replica:0/task:0/device:fakecpu:0") + .release(); + env_.local_devices = {device}; + device_mgr_.reset(new DeviceMgr(env_.local_devices)); + env_.device_mgr = device_mgr_.get(); } - std::unique_ptr device_; + std::unique_ptr device_mgr_; WorkerEnv env_; SessionMgr::WorkerCacheFactory factory_ = [](const ServerDef& server_def, WorkerCacheInterface** worker_cache) { @@ -59,7 +61,6 @@ class SessionMgrTest : public ::testing::Test { return Status::OK(); }; SessionMgr mgr_; - std::shared_ptr legacy_session_; }; TEST_F(SessionMgrTest, CreateSessionSimple) { @@ -84,25 +85,25 @@ TEST_F(SessionMgrTest, CreateSessionIsolateSessionState) { TF_EXPECT_OK(mgr_.CreateSession("handle_1", server_def, false)); std::shared_ptr session_1; TF_EXPECT_OK(mgr_.WorkerSessionForSession("handle_1", &session_1)); - std::vector devices_1 = session_1->device_mgr->ListDevices(); + std::vector devices_1 = session_1->device_mgr()->ListDevices(); EXPECT_EQ(1, devices_1.size()); TF_EXPECT_OK(mgr_.CreateSession("handle_2", server_def, false)); std::shared_ptr session_2; TF_EXPECT_OK(mgr_.WorkerSessionForSession("handle_2", &session_2)); - std::vector devices_2 = session_2->device_mgr->ListDevices(); + std::vector devices_2 = session_2->device_mgr()->ListDevices(); EXPECT_EQ(1, devices_2.size()); TF_EXPECT_OK(mgr_.CreateSession("handle_3", server_def, true)); std::shared_ptr session_3; TF_EXPECT_OK(mgr_.WorkerSessionForSession("handle_3", &session_3)); - std::vector devices_3 = session_3->device_mgr->ListDevices(); + std::vector devices_3 = session_3->device_mgr()->ListDevices(); EXPECT_EQ(1, devices_3.size()); TF_EXPECT_OK(mgr_.CreateSession("handle_4", server_def, true)); std::shared_ptr session_4; TF_EXPECT_OK(mgr_.WorkerSessionForSession("handle_4", &session_4)); - std::vector devices_4 = session_4->device_mgr->ListDevices(); + std::vector devices_4 = session_4->device_mgr()->ListDevices(); EXPECT_EQ(1, devices_4.size()); EXPECT_EQ(devices_1[0]->resource_manager(), devices_2[0]->resource_manager()); diff --git a/tensorflow/core/distributed_runtime/worker_session.cc b/tensorflow/core/distributed_runtime/worker_session.cc index 18886babd5..ca6dc1b1de 100644 --- a/tensorflow/core/distributed_runtime/worker_session.cc +++ b/tensorflow/core/distributed_runtime/worker_session.cc @@ -95,9 +95,43 @@ WorkerSession::WorkerSession(const string& session_name, : session_name(session_name), worker_name(worker_name), worker_cache(new WorkerFreeListCache(std::move(worker_cache))), - device_mgr(std::move(device_mgr)), graph_mgr(std::move(graph_mgr)), cluster_flr( - new ClusterFunctionLibraryRuntime(this, !session_name.empty())) {} + new ClusterFunctionLibraryRuntime(this, !session_name.empty())), + device_mgr_(std::move(device_mgr)), + borrowed_device_mgr_(nullptr) {} + +/* static */ +std::shared_ptr WorkerSession::CreateWithBorrowedDeviceMgr( + const string& session_name, const string& worker_name, + std::unique_ptr worker_cache, + DeviceMgr* borrowed_device_mgr, std::unique_ptr graph_mgr) { + return std::shared_ptr( + new WorkerSession(session_name, worker_name, std::move(worker_cache), + borrowed_device_mgr, std::move(graph_mgr))); +} + +WorkerSession::WorkerSession(const string& session_name, + const string& worker_name, + std::unique_ptr worker_cache, + DeviceMgr* borrowed_device_mgr, + std::unique_ptr graph_mgr) + : session_name(session_name), + worker_name(worker_name), + worker_cache(new WorkerFreeListCache(std::move(worker_cache))), + graph_mgr(std::move(graph_mgr)), + cluster_flr( + new ClusterFunctionLibraryRuntime(this, !session_name.empty())), + device_mgr_(nullptr), + borrowed_device_mgr_(borrowed_device_mgr) {} + +WorkerSession::~WorkerSession() { + if (graph_mgr) { + Status s = graph_mgr->DeregisterAll(); + if (!s.ok()) { + LOG(WARNING) << "Error during worker session deletion: " << s; + } + } +} } // namespace tensorflow diff --git a/tensorflow/core/distributed_runtime/worker_session.h b/tensorflow/core/distributed_runtime/worker_session.h index 0fd19ac27f..f1faf49364 100644 --- a/tensorflow/core/distributed_runtime/worker_session.h +++ b/tensorflow/core/distributed_runtime/worker_session.h @@ -40,10 +40,14 @@ struct WorkerSession { // Object from which WorkerInterface instances can be obtained. const std::unique_ptr worker_cache; - // Collection of local devices. These devices are typically RenamedDevices - // in all except the SessionMgr.legacy_session_. legacy_session_.device_mgr - // == worker_env_.device_mgr, which holds the true devices. - const std::unique_ptr device_mgr; + // Collection of local devices. These devices are typically + // RenamedDevices in all except the SessionMgr.legacy_session_ and + // sessions created with `isolate_session_state == false`. In the + // those cases, this method returns a pointer to a borrowed + // DeviceMgr (typically the `worker_env.device_mgr`). + DeviceMgr* device_mgr() { + return device_mgr_ ? device_mgr_.get() : borrowed_device_mgr_; + } // graph_mgr keeps track of the registered graphs of this session. // @@ -57,6 +61,22 @@ struct WorkerSession { std::unique_ptr worker_cache, std::unique_ptr device_mgr, std::unique_ptr graph_mgr); + + static std::shared_ptr CreateWithBorrowedDeviceMgr( + const string& session_name, const string& worker_name, + std::unique_ptr worker_cache, + DeviceMgr* borrowed_device_mgr, std::unique_ptr graph_mgr); + + ~WorkerSession(); + + private: + WorkerSession(const string& session_name, const string& worker_name, + std::unique_ptr worker_cache, + DeviceMgr* borrowed_device_mgr, + std::unique_ptr graph_mgr); + + const std::unique_ptr device_mgr_; + DeviceMgr* const borrowed_device_mgr_; // Not owned. }; } // namespace tensorflow -- GitLab From e77bb988e470d35aca3ea1e27a4f335409f1f4d2 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 10:59:08 -0700 Subject: [PATCH 663/791] Fix open source BUILD bugs for cloud profiler. Increment version for releasing cloud_tpu_profiler 1.6 with pod profiling support. PiperOrigin-RevId: 193541692 --- .../tpu/profiler/capture_tpu_profile.cc | 12 +++++----- .../pip_package/cloud_tpu_profiler/main.py | 23 +++++++++++++++++-- .../contrib/tpu/profiler/pip_package/setup.py | 2 +- tensorflow/contrib/tpu/profiler/version.h | 2 +- 4 files changed, 29 insertions(+), 10 deletions(-) diff --git a/tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc b/tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc index a535884263..816897499b 100644 --- a/tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc +++ b/tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc @@ -41,7 +41,7 @@ namespace tensorflow { namespace tpu { namespace { -using ::tensorflow::grpc::TPUProfileAnalysis; +using ::tensorflow::TPUProfileAnalysis; using ::tensorflow::TPUProfiler; constexpr uint64 kMaxEvents = 1000000; @@ -137,9 +137,9 @@ bool NewSession(const string& service_addr, PopulateProfileRequest(duration_ms, repository_root, session_id, opts); new_session_request.set_repository_root(repository_root); new_session_request.set_session_id(session_id); - std::copy( - hostnames.begin(), hostnames.end(), - proto2::RepeatedFieldBackInserter(new_session_request.mutable_hosts())); + for (const auto& hostname : hostnames) { + new_session_request.add_hosts(hostname); + } ::grpc::ClientContext context; ::grpc::ChannelArguments channel_args; @@ -159,8 +159,8 @@ bool NewSession(const string& service_addr, TF_QCHECK_OK(FromGrpcStatus( stub->NewSession(&context, new_session_request, &new_session_response))); - std::cout << "Profile session succeed for hosts:" - << str_util::Join(hostnames, ","); + std::cout << "Profile session succeed for host(s):" + << str_util::Join(hostnames, ",") << std::endl; return new_session_response.empty_trace(); } diff --git a/tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py b/tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py index 0b78cf8695..508c7a842f 100644 --- a/tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py +++ b/tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py @@ -37,12 +37,17 @@ flags.DEFINE_string( 'will attempt to automatically detect the GCE project from metadata.') flags.DEFINE_string('tpu_name', None, 'Name of the Cloud TPU for Cluster Resolvers. You must ' - 'specify either this flag or --master.') + 'specify either this flag or --service_addr.') # Tool specific parameters flags.DEFINE_string( 'service_addr', None, 'Address of TPU profiler service e.g. ' 'localhost:8466, you must specify either this flag or --tpu_name.') +flags.DEFINE_string( + 'workers_list', None, 'The list of worker TPUs that we are about to profile' + ' e.g. 10.0.1.2, 10.0.1.3. You can specify this flag with --tpu_name or ' + '--service_addr to profile a subset of tpu nodes. You can also use only' + '--tpu_name and leave this flag unspecified to profile all the tpus.') flags.DEFINE_string('logdir', None, 'Path of TensorBoard log directory e.g. /tmp/tb_log, ' 'gs://tb_bucket') @@ -56,18 +61,25 @@ flags.DEFINE_boolean('include_dataset_ops', True, FLAGS = flags.FLAGS EXECUTABLE = 'data/capture_tpu_profile' +JOB_NAME = 'worker' +def get_workers_list(cluster_resolver): + cluster_spec = cluster_resolver.cluster_spec() + task_indices = cluster_spec.task_indices(JOB_NAME) + workers_list = [cluster_spec.task_address(JOB_NAME, i).split(':')[0] + for i in task_indices] + return ','.join(workers_list) def run_main(): tf.app.run(main) - def main(unused_argv=None): tf.logging.set_verbosity(tf.logging.INFO) if FLAGS.service_addr is None and FLAGS.tpu_name is None: sys.exit('You must specify either --service_addr or --tpu_name.') + tpu_cluster_resolver = None if FLAGS.service_addr is not None: if FLAGS.tpu_name is not None: tf.logging.warn('Both --service_addr and --tpu_name are set. Ignoring ' @@ -82,6 +94,12 @@ def main(unused_argv=None): service_addr = tpu_cluster_resolver.get_master() service_addr = service_addr.replace('grpc://', '').replace(':8470', ':8466') + workers_list = "" + if FLAGS.workers_list is not None: + workers_list = FLAGS.workers_list + elif tpu_cluster_resolver is not None: + workers_list = get_workers_list(tpu_cluster_resolver) + if not FLAGS.logdir: sys.exit('logdir must be provided.') executable_path = os.path.join(os.path.dirname(__file__), EXECUTABLE) @@ -89,6 +107,7 @@ def main(unused_argv=None): cmd = [executable_path] cmd.append('--logdir=' + logdir) cmd.append('--service_addr=' + service_addr) + cmd.append('--workers_list=' + workers_list) cmd.append('--duration_ms=' + str(FLAGS.duration_ms)) cmd.append('--num_tracing_attempts=' + str(FLAGS.num_tracing_attempts)) cmd.append('--include_dataset_ops=' + str(FLAGS.include_dataset_ops).lower()) diff --git a/tensorflow/contrib/tpu/profiler/pip_package/setup.py b/tensorflow/contrib/tpu/profiler/pip_package/setup.py index 8d99835b64..ebd478fd02 100644 --- a/tensorflow/contrib/tpu/profiler/pip_package/setup.py +++ b/tensorflow/contrib/tpu/profiler/pip_package/setup.py @@ -20,7 +20,7 @@ from __future__ import print_function from setuptools import setup -_VERSION = '1.6.0-rc1' +_VERSION = '1.6.0' CONSOLE_SCRIPTS = [ 'capture_tpu_profile=cloud_tpu_profiler.main:run_main', diff --git a/tensorflow/contrib/tpu/profiler/version.h b/tensorflow/contrib/tpu/profiler/version.h index dc6a934891..618479e1a6 100644 --- a/tensorflow/contrib/tpu/profiler/version.h +++ b/tensorflow/contrib/tpu/profiler/version.h @@ -16,6 +16,6 @@ limitations under the License. #ifndef TENSORFLOW_CONTRIB_TPU_PROFILER_VERSION_H_ #define TENSORFLOW_CONTRIB_TPU_PROFILER_VERSION_H_ -#define TPU_PROFILER_VERSION "1.5.0" +#define TPU_PROFILER_VERSION "1.6.0" #endif // TENSORFLOW_CONTRIB_TPU_PROFILER_VERSION_H_ -- GitLab From 62c3b7dece92a3ad1a39e7c4eb0894411e435258 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 11:08:08 -0700 Subject: [PATCH 664/791] Updating tests in constant_folding_test.cc so that they Evaluate the optimized and original graph and check if their outputs are the same. PiperOrigin-RevId: 193543478 --- .../optimizers/constant_folding_test.cc | 52 +++++++++++++++++-- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc index 36625b68b7..1acce05909 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc +++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc @@ -689,8 +689,7 @@ TEST_F(ConstantFoldingTest, ControlDependencies) { GrapplerItem item; item.fetch.push_back("e"); TF_CHECK_OK(scope.ToGraphDef(&item.graph)); - auto tensors_expected = EvaluateNodes(item.graph, item.fetch); - EXPECT_EQ(1, tensors_expected.size()); + ConstantFolding optimizer(nullptr /* cpu_device */); GraphDef output; Status status = optimizer.Optimize(nullptr, item, &output); @@ -717,9 +716,6 @@ TEST_F(ConstantFoldingTest, ControlDependencies) { } } EXPECT_EQ(1, found); - auto tensors = EvaluateNodes(output, item.fetch); - EXPECT_EQ(1, tensors.size()); - test::ExpectTensorEqual(tensors_expected[0], tensors[0]); } TEST_F(ConstantFoldingTest, ControlDependenciesEmptyFetch) { @@ -995,6 +991,18 @@ TEST_F(ConstantFoldingTest, ShapeMaterializationEmptyFetch) { } } EXPECT_EQ(3, found); + + auto v1_t = GenerateRandomTensor(TensorShape({3})); + auto v2_t = GenerateRandomTensor(TensorShape({5, 7})); + auto v3_t = GenerateRandomTensor(TensorShape({11, 13})); + std::vector fetch_nodes = {"p2"}; + auto tensors_expected = EvaluateNodes( + item.graph, fetch_nodes, {{"v1", v1_t}, {"v2", v2_t}, {"v3", v3_t}}); + EXPECT_EQ(1, tensors_expected.size()); + auto tensors = EvaluateNodes(output, fetch_nodes, + {{"v1", v1_t}, {"v2", v2_t}, {"v3", v3_t}}); + EXPECT_EQ(1, tensors.size()); + test::ExpectTensorEqual(tensors_expected[0], tensors[0]); } TEST_F(ConstantFoldingTest, ShapeMaterializationShapeN) { @@ -1192,6 +1200,30 @@ TEST_F(ConstantFoldingTest, SwitchNodesEmptyFetch) { } } EXPECT_EQ(4, found); + + auto v_in_t = GenerateRandomTensor(TensorShape({3})); + Tensor v_ctrl_t(DT_BOOL, TensorShape({})); + + v_ctrl_t.flat()(0) = true; + std::vector fetch_nodes = {"m", "m2"}; + auto tensors_expected = EvaluateNodes( + item.graph, fetch_nodes, {{"v_in", v_in_t}, {"v_ctrl", v_ctrl_t}}); + EXPECT_EQ(2, tensors_expected.size()); + auto tensors = EvaluateNodes(output, fetch_nodes, + {{"v_in", v_in_t}, {"v_ctrl", v_ctrl_t}}); + EXPECT_EQ(2, tensors.size()); + test::ExpectTensorEqual(tensors_expected[0], tensors[0]); + test::ExpectTensorNear(tensors_expected[1], tensors[1], 1e-5); + + v_ctrl_t.flat()(0) = false; + tensors_expected = EvaluateNodes(item.graph, fetch_nodes, + {{"v_in", v_in_t}, {"v_ctrl", v_ctrl_t}}); + EXPECT_EQ(2, tensors_expected.size()); + tensors = EvaluateNodes(output, fetch_nodes, + {{"v_in", v_in_t}, {"v_ctrl", v_ctrl_t}}); + EXPECT_EQ(2, tensors.size()); + test::ExpectTensorEqual(tensors_expected[0], tensors[0]); + test::ExpectTensorNear(tensors_expected[1], tensors[1], 1e-5); } TEST_F(ConstantFoldingTest, SwitchNodes) { @@ -1268,6 +1300,16 @@ TEST_F(ConstantFoldingTest, SwitchNodes) { EXPECT_EQ(2, tensors.size()); test::ExpectTensorEqual(tensors_expected[0], tensors[0]); test::ExpectTensorNear(tensors_expected[1], tensors[1], 1e-5); + + v_ctrl_t.flat()(0) = false; + tensors_expected = EvaluateNodes(item.graph, item.fetch, + {{"v_in", v_in_t}, {"v_ctrl", v_ctrl_t}}); + EXPECT_EQ(2, tensors_expected.size()); + tensors = EvaluateNodes(output, item.fetch, + {{"v_in", v_in_t}, {"v_ctrl", v_ctrl_t}}); + EXPECT_EQ(2, tensors.size()); + test::ExpectTensorEqual(tensors_expected[0], tensors[0]); + test::ExpectTensorNear(tensors_expected[1], tensors[1], 1e-5); } TEST_F(ConstantFoldingTest, MergeNodes) { -- GitLab From 9b496c9134529f6d85f0e9757099104cf506cbd6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 11:21:21 -0700 Subject: [PATCH 665/791] Update ops-related pbtxt files. PiperOrigin-RevId: 193546050 --- tensorflow/core/ops/compat/ops_history.v1.pbtxt | 15 +++++++++++++++ tensorflow/core/ops/ops.pbtxt | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt index 9bc11cf0fe..dbd6f859c4 100644 --- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt +++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt @@ -15829,6 +15829,21 @@ op { minimum: 1 } } +op { + name: "DatasetToTFRecord" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "filename" + type: DT_STRING + } + input_arg { + name: "compression_type" + type: DT_STRING + } +} op { name: "DebugGradientIdentity" input_arg { diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index 9b665190ce..46afe357f0 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -7051,6 +7051,21 @@ op { minimum: 1 } } +op { + name: "DatasetToTFRecord" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "filename" + type: DT_STRING + } + input_arg { + name: "compression_type" + type: DT_STRING + } +} op { name: "DebugGradientIdentity" input_arg { -- GitLab From 87229e4fc3bc23c7a92bfdf40e5834ac65a00d34 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 11:47:28 -0700 Subject: [PATCH 666/791] Go: Update generated wrapper functions for TensorFlow ops. PiperOrigin-RevId: 193550428 --- tensorflow/go/op/wrappers.go | 72 ++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index 35ad1eff0f..3b3dff0573 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -3105,6 +3105,42 @@ func Betainc(scope *Scope, a tf.Output, b tf.Output, x tf.Output) (z tf.Output) return op.Output(0) } +// Return a tensor with the same shape and contents as the input tensor or value. +func Identity(scope *Scope, input tf.Output) (output tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "Identity", + Input: []tf.Input{ + input, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Computes arctangent of `y/x` element-wise, respecting signs of the arguments. +// +// This is the angle \( \theta \in [-\pi, \pi] \) such that +// \[ x = r \cos(\theta) \] +// and +// \[ y = r \sin(\theta) \] +// where \(r = \sqrt(x^2 + y^2) \). +func Atan2(scope *Scope, y tf.Output, x tf.Output) (z tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "Atan2", + Input: []tf.Input{ + y, x, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Creates a dataset that passes a sliding window over `input_dataset`. // // Arguments: @@ -25383,42 +25419,6 @@ func IteratorFromStringHandle(scope *Scope, string_handle tf.Output, optional .. return op.Output(0) } -// Computes arctangent of `y/x` element-wise, respecting signs of the arguments. -// -// This is the angle \( \theta \in [-\pi, \pi] \) such that -// \[ x = r \cos(\theta) \] -// and -// \[ y = r \sin(\theta) \] -// where \(r = \sqrt(x^2 + y^2) \). -func Atan2(scope *Scope, y tf.Output, x tf.Output) (z tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "Atan2", - Input: []tf.Input{ - y, x, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// Return a tensor with the same shape and contents as the input tensor or value. -func Identity(scope *Scope, input tf.Output) (output tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "Identity", - Input: []tf.Input{ - input, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // Gather slices from `params` axis `axis` according to `indices`. // // `indices` must be an integer tensor of any dimension (usually 0-D or 1-D). -- GitLab From 78db5136edf30667090988c703f98f4f8c4c4269 Mon Sep 17 00:00:00 2001 From: Rohan Jain Date: Thu, 19 Apr 2018 11:52:10 -0700 Subject: [PATCH 667/791] Implements linear_model using _LinearModel. Added support for cols_to_vars in _LinearModel in order to make this possible. Also, made some fixes so that variable names come out the same as before. PiperOrigin-RevId: 193551353 --- .../python/feature_column/feature_column.py | 106 ++++++++-------- .../feature_column/feature_column_test.py | 117 ++++++++++++------ .../training/warm_starting_util_test.py | 16 +-- 3 files changed, 138 insertions(+), 101 deletions(-) diff --git a/tensorflow/python/feature_column/feature_column.py b/tensorflow/python/feature_column/feature_column.py index 0ad8131599..87a52f8441 100644 --- a/tensorflow/python/feature_column/feature_column.py +++ b/tensorflow/python/feature_column/feature_column.py @@ -409,58 +409,19 @@ def linear_model(features, ValueError: if an item in `feature_columns` is neither a `_DenseColumn` nor `_CategoricalColumn`. """ - feature_columns = _clean_feature_columns(feature_columns) - for column in feature_columns: - if not isinstance(column, (_DenseColumn, _CategoricalColumn)): - raise ValueError('Items of feature_columns must be either a _DenseColumn ' - 'or _CategoricalColumn. Given: {}'.format(column)) - weight_collections = list(weight_collections or []) - if ops.GraphKeys.GLOBAL_VARIABLES not in weight_collections: - weight_collections.append(ops.GraphKeys.GLOBAL_VARIABLES) - if ops.GraphKeys.MODEL_VARIABLES not in weight_collections: - weight_collections.append(ops.GraphKeys.MODEL_VARIABLES) - with variable_scope.variable_scope( - None, default_name='linear_model', values=features.values()): - weighted_sums = [] - ordered_columns = [] - builder = _LazyBuilder(features) - for column in sorted(feature_columns, key=lambda x: x.name): - with variable_scope.variable_scope( - None, default_name=column._var_scope_name): # pylint: disable=protected-access - ordered_columns.append(column) - weighted_sum = _create_weighted_sum( - column=column, - builder=builder, - units=units, - sparse_combiner=sparse_combiner, - weight_collections=weight_collections, - trainable=trainable) - weighted_sums.append(weighted_sum) - if cols_to_vars is not None: - # Retrieve the variables created. - cols_to_vars[column] = ops.get_collection( - ops.GraphKeys.GLOBAL_VARIABLES, - scope=variable_scope.get_variable_scope().name) - _verify_static_batch_size_equality(weighted_sums, ordered_columns) - predictions_no_bias = math_ops.add_n( - weighted_sums, name='weighted_sum_no_bias') - bias = variable_scope.get_variable( - 'bias_weights', - shape=[units], - initializer=init_ops.zeros_initializer(), - trainable=trainable, - collections=weight_collections) - predictions = nn_ops.bias_add( - predictions_no_bias, bias, name='weighted_sum') - if cols_to_vars is not None: - # Add the bias to cols_to_vars as well, converting the Variable or - # PartitionedVariable to a list of Variable's. - if (isinstance(bias, variables.Variable) or - resource_variable_ops.is_resource_variable(bias)): - cols_to_vars['bias'] = [bias] - else: # Must be a PartitionedVariable. - cols_to_vars['bias'] = list(bias) - return predictions + linear_model_layer = _LinearModel( + feature_columns=feature_columns, + units=units, + sparse_combiner=sparse_combiner, + weight_collections=weight_collections, + trainable=trainable, + name='linear_model') + retval = linear_model_layer(features) # pylint: disable=not-callable + if cols_to_vars is None: + return retval + for k, v in linear_model_layer.cols_to_vars().items(): + cols_to_vars[k] = v + return retval def _add_to_collections(var, weight_collections): @@ -551,8 +512,22 @@ class _BiasLayer(base.Layer): return self._bias_variable +def _get_expanded_variable_list(variable): + if (isinstance(variable, variables.Variable) or + resource_variable_ops.is_resource_variable(variable)): + return [variable] # Single variable case. + else: # Must be a PartitionedVariable, so convert into a list. + return list(variable) + + +def _strip_leading_slashes(name): + return name.rsplit('/', 1)[-1] + + class _LinearModel(training.Model): """Creates a linear model using feature columns. + + See `linear_model` for details. """ def __init__(self, @@ -573,7 +548,10 @@ class _LinearModel(training.Model): for column in sorted(self._feature_columns, key=lambda x: x.name): with variable_scope.variable_scope( None, default_name=column._var_scope_name) as vs: # pylint: disable=protected-access - column_name = vs.name + # Having the fully expressed variable scope name ends up doubly + # expressing the outer scope (scope with which this method was called) + # in the name of the variable that would get created. + column_name = _strip_leading_slashes(vs.name) column_layer = _FCLinearWrapper(column, units, sparse_combiner, self._weight_collections, trainable, column_name, **kwargs) @@ -585,6 +563,15 @@ class _LinearModel(training.Model): weight_collections=self._weight_collections, name='bias_layer', **kwargs) + self._cols_to_vars = {} + + def cols_to_vars(self): + """Returns a dict mapping _FeatureColumns to variables. + + See `linear_model` for more information. + This is not populated till `call` is called i.e. layer is built. + """ + return self._cols_to_vars def call(self, features): with variable_scope.variable_scope(self.name): @@ -597,15 +584,24 @@ class _LinearModel(training.Model): ordered_columns = [] builder = _LazyBuilder(features) for layer in sorted(self._column_layers.values(), key=lambda x: x.name): - ordered_columns.append(layer._feature_column) # pylint: disable=protected-access + column = layer._feature_column # pylint: disable=protected-access + ordered_columns.append(column) weighted_sum = layer(builder) weighted_sums.append(weighted_sum) + self._cols_to_vars[column] = ops.get_collection( + ops.GraphKeys.GLOBAL_VARIABLES, scope=layer.scope_name) _verify_static_batch_size_equality(weighted_sums, ordered_columns) predictions_no_bias = math_ops.add_n( weighted_sums, name='weighted_sum_no_bias') predictions = nn_ops.bias_add( - predictions_no_bias, self._bias_layer(builder), name='weighted_sum') # pylint: disable=not-callable + predictions_no_bias, + self._bias_layer( # pylint: disable=not-callable + builder, + scope=variable_scope.get_variable_scope()), # pylint: disable=not-callable + name='weighted_sum') + bias = self._bias_layer.variables[0] + self._cols_to_vars['bias'] = _get_expanded_variable_list(bias) return predictions def _add_layers(self, layers): diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py index 46404abadc..49e06b8245 100644 --- a/tensorflow/python/feature_column/feature_column_test.py +++ b/tensorflow/python/feature_column/feature_column_test.py @@ -345,7 +345,7 @@ class NumericColumnTest(test.TestCase): with ops.Graph().as_default(): features = {'price': [[1.], [5.]]} predictions = get_keras_linear_model_predictions(features, [price]) - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() price_var = get_linear_model_column_var(price) with _initialized_session() as sess: self.assertAllClose([0.], bias.eval()) @@ -584,7 +584,7 @@ class BucketizedColumnTest(test.TestCase): features = {'price': [[-1.], [1.], [5.], [6.]]} predictions = get_keras_linear_model_predictions(features, [bucketized_price]) - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() bucketized_price_var = get_linear_model_column_var(bucketized_price) with _initialized_session() as sess: self.assertAllClose([0.], bias.eval()) @@ -610,7 +610,7 @@ class BucketizedColumnTest(test.TestCase): features = {'price': [[-1., 1.], [5., 6.]]} predictions = get_keras_linear_model_predictions(features, [bucketized_price]) - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() bucketized_price_var = get_linear_model_column_var(bucketized_price) with _initialized_session() as sess: self.assertAllClose([0.], bias.eval()) @@ -849,7 +849,7 @@ class HashedCategoricalColumnTest(test.TestCase): values=('marlo', 'skywalker', 'omar'), dense_shape=(2, 2)) }, (wire_column,)) - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() wire_var = get_linear_model_column_var(wire_column) with _initialized_session(): self.assertAllClose((0.,), bias.eval()) @@ -1171,7 +1171,7 @@ class CrossedColumnTest(test.TestCase): values=['cA', 'cB', 'cC'], dense_shape=(2, 2)), }, (crossed,)) - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() crossed_var = get_linear_model_column_var(crossed) with _initialized_session() as sess: self.assertAllClose((0.,), bias.eval()) @@ -1254,18 +1254,13 @@ def get_linear_model_column_var(column): 'linear_model/' + column.name)[0] -def get_keras_linear_model_bias(): - with variable_scope.variable_scope('linear_model', reuse=True): - with variable_scope.variable_scope('bias_layer', reuse=True): - return variable_scope.get_variable('bias_weights') - - def get_keras_linear_model_predictions(features, feature_columns, units=1, sparse_combiner='sum', weight_collections=None, - trainable=True): + trainable=True, + cols_to_vars=None): keras_linear_model = _LinearModel( feature_columns, units, @@ -1273,7 +1268,12 @@ def get_keras_linear_model_predictions(features, weight_collections, trainable, name='linear_model') - return keras_linear_model(features) # pylint: disable=not-callable + retval = keras_linear_model(features) # pylint: disable=not-callable + if cols_to_vars is None: + return retval + for k, v in keras_linear_model.cols_to_vars().items(): + cols_to_vars[k] = v + return retval @test_util.with_c_api @@ -1977,7 +1977,7 @@ class _LinearModelTest(test.TestCase): with ops.Graph().as_default(): features = {'price': [[1.], [5.]]} predictions = get_keras_linear_model_predictions(features, [price]) - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() price_var = get_linear_model_column_var(price) with _initialized_session() as sess: self.assertAllClose([0.], bias.eval()) @@ -1994,7 +1994,7 @@ class _LinearModelTest(test.TestCase): dense_shape=[2, 2]) features = {'wire_cast': wire_tensor} predictions = get_keras_linear_model_predictions(features, [wire_cast]) - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() wire_cast_var = get_linear_model_column_var(wire_cast) with _initialized_session() as sess: self.assertAllClose([0.], bias.eval()) @@ -2014,7 +2014,7 @@ class _LinearModelTest(test.TestCase): features = {'wire_cast': wire_tensor, 'price': [[1.], [5.]]} predictions = get_keras_linear_model_predictions(features, [wire_cast, price]) - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() wire_cast_var = get_linear_model_column_var(wire_cast) price_var = get_linear_model_column_var(price) with _initialized_session() as sess: @@ -2072,7 +2072,7 @@ class _LinearModelTest(test.TestCase): features = {dense_and_sparse_column.name: sp_tensor} predictions = get_keras_linear_model_predictions( features, [dense_and_sparse_column]) - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() dense_and_sparse_column_var = get_linear_model_column_var( dense_and_sparse_column) with _initialized_session() as sess: @@ -2088,7 +2088,7 @@ class _LinearModelTest(test.TestCase): features = {'price': [[1.], [5.]]} predictions = get_keras_linear_model_predictions( features, [price], units=3) - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() price_var = get_linear_model_column_var(price) with _initialized_session() as sess: self.assertAllClose(np.zeros((3,)), bias.eval()) @@ -2108,7 +2108,7 @@ class _LinearModelTest(test.TestCase): features = {'wire_cast': wire_tensor} predictions = get_keras_linear_model_predictions( features, [wire_cast], units=3) - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() wire_cast_var = get_linear_model_column_var(wire_cast) with _initialized_session() as sess: self.assertAllClose(np.zeros((3,)), bias.eval()) @@ -2163,7 +2163,7 @@ class _LinearModelTest(test.TestCase): features = {'wire_cast': wire_tensor} predictions = get_keras_linear_model_predictions( features, [wire_cast], sparse_combiner='mean') - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() wire_cast_var = get_linear_model_column_var(wire_cast) with _initialized_session() as sess: sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]])) @@ -2176,7 +2176,7 @@ class _LinearModelTest(test.TestCase): features = {'price': [[1., 2.], [5., 6.]]} predictions = get_keras_linear_model_predictions( features, [price], units=3) - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() price_var = get_linear_model_column_var(price) with _initialized_session() as sess: self.assertAllClose(np.zeros((3,)), bias.eval()) @@ -2206,7 +2206,7 @@ class _LinearModelTest(test.TestCase): with ops.Graph().as_default(): features = {'price': [[[1., 2.]], [[5., 6.]]]} predictions = get_keras_linear_model_predictions(features, [price]) - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() price_var = get_linear_model_column_var(price) with _initialized_session() as sess: self.assertAllClose([0.], bias.eval()) @@ -2222,7 +2222,7 @@ class _LinearModelTest(test.TestCase): features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]} predictions = get_keras_linear_model_predictions(features, [price1, price2]) - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() price1_var = get_linear_model_column_var(price1) price2_var = get_linear_model_column_var(price2) with _initialized_session() as sess: @@ -2235,6 +2235,45 @@ class _LinearModelTest(test.TestCase): sess.run(bias.assign([7.])) self.assertAllClose([[3217.], [4657.]], predictions.eval()) + def test_fills_cols_to_vars(self): + price1 = fc.numeric_column('price1', shape=2) + price2 = fc.numeric_column('price2') + with ops.Graph().as_default(): + features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]} + cols_to_vars = {} + get_keras_linear_model_predictions( + features, [price1, price2], cols_to_vars=cols_to_vars) + bias = get_linear_model_bias() + price1_var = get_linear_model_column_var(price1) + price2_var = get_linear_model_column_var(price2) + self.assertAllEqual(cols_to_vars['bias'], [bias]) + self.assertAllEqual(cols_to_vars[price1], [price1_var]) + self.assertAllEqual(cols_to_vars[price2], [price2_var]) + + def test_fills_cols_to_vars_partitioned_variables(self): + price1 = fc.numeric_column('price1', shape=2) + price2 = fc.numeric_column('price2', shape=3) + with ops.Graph().as_default(): + features = { + 'price1': [[1., 2.], [6., 7.]], + 'price2': [[3., 4., 5.], [8., 9., 10.]] + } + cols_to_vars = {} + with variable_scope.variable_scope( + 'linear', + partitioner=partitioned_variables.fixed_size_partitioner(2, axis=0)): + get_keras_linear_model_predictions( + features, [price1, price2], cols_to_vars=cols_to_vars) + with _initialized_session(): + self.assertEqual([0.], cols_to_vars['bias'][0].eval()) + # Partitioning shards the [2, 1] price1 var into 2 [1, 1] Variables. + self.assertAllEqual([[0.]], cols_to_vars[price1][0].eval()) + self.assertAllEqual([[0.]], cols_to_vars[price1][1].eval()) + # Partitioning shards the [3, 1] price2 var into a [2, 1] Variable and + # a [1, 1] Variable. + self.assertAllEqual([[0.], [0.]], cols_to_vars[price2][0].eval()) + self.assertAllEqual([[0.]], cols_to_vars[price2][1].eval()) + def test_dense_collection(self): price = fc.numeric_column('price') with ops.Graph().as_default() as g: @@ -2242,7 +2281,7 @@ class _LinearModelTest(test.TestCase): get_keras_linear_model_predictions( features, [price], weight_collections=['my-vars']) my_vars = g.get_collection('my-vars') - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() price_var = get_linear_model_column_var(price) self.assertIn(bias, my_vars) self.assertIn(price_var, my_vars) @@ -2256,7 +2295,7 @@ class _LinearModelTest(test.TestCase): get_keras_linear_model_predictions( features, [wire_cast], weight_collections=['my-vars']) my_vars = g.get_collection('my-vars') - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() wire_cast_var = get_linear_model_column_var(wire_cast) self.assertIn(bias, my_vars) self.assertIn(wire_cast_var, my_vars) @@ -2266,7 +2305,7 @@ class _LinearModelTest(test.TestCase): with ops.Graph().as_default() as g: features = {'price': [[1.], [5.]]} get_keras_linear_model_predictions(features, [price]) - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() price_var = get_linear_model_column_var(price) trainable_vars = g.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES) self.assertIn(bias, trainable_vars) @@ -2280,7 +2319,7 @@ class _LinearModelTest(test.TestCase): features = {'wire_cast': wire_tensor} get_keras_linear_model_predictions(features, [wire_cast]) trainable_vars = g.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES) - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() wire_cast_var = get_linear_model_column_var(wire_cast) self.assertIn(bias, trainable_vars) self.assertIn(wire_cast_var, trainable_vars) @@ -2427,7 +2466,7 @@ class _LinearModelTest(test.TestCase): coord = coordinator.Coordinator() threads = queue_runner_impl.start_queue_runners(sess, coord=coord) - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() price_buckets_var = get_linear_model_column_var(price_buckets) body_style_var = get_linear_model_column_var(body_style) @@ -2470,7 +2509,7 @@ class _LinearModelTest(test.TestCase): net = get_keras_linear_model_predictions(features, [price_buckets, body_style]) with _initialized_session() as sess: - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() price_buckets_var = get_linear_model_column_var(price_buckets) body_style_var = get_linear_model_column_var(body_style) @@ -2509,7 +2548,7 @@ class _LinearModelTest(test.TestCase): net = get_keras_linear_model_predictions( features, [price_buckets, body_style, country]) - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() price_buckets_var = get_linear_model_column_var(price_buckets) body_style_var = get_linear_model_column_var(body_style) with _initialized_session() as sess: @@ -3688,7 +3727,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): values=('marlo', 'skywalker', 'omar'), dense_shape=(2, 2)) }, (wire_column,)) - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() wire_var = get_linear_model_column_var(wire_column) with _initialized_session(): self.assertAllClose((0.,), bias.eval()) @@ -4080,7 +4119,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase): values=('marlo', 'skywalker', 'omar'), dense_shape=(2, 2)) }, (wire_column,)) - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() wire_var = get_linear_model_column_var(wire_column) with _initialized_session(): self.assertAllClose((0.,), bias.eval()) @@ -4326,7 +4365,7 @@ class IdentityCategoricalColumnTest(test.TestCase): values=(0, 2, 1), dense_shape=(2, 2)) }, (column,)) - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() weight_var = get_linear_model_column_var(column) with _initialized_session(): self.assertAllClose((0.,), bias.eval()) @@ -5108,7 +5147,7 @@ class EmbeddingColumnTest(test.TestCase): categorical_column.name: sparse_input }, (embedding_column,)) expected_var_names = ( - 'linear_model/bias_layer/bias_weights:0', + 'linear_model/bias_weights:0', 'linear_model/aaa_embedding/weights:0', 'linear_model/aaa_embedding/embedding_weights:0', ) @@ -5120,7 +5159,7 @@ class EmbeddingColumnTest(test.TestCase): for v in ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES) } self.assertItemsEqual(expected_var_names, trainable_vars.keys()) - bias = trainable_vars['linear_model/bias_layer/bias_weights:0'] + bias = trainable_vars['linear_model/bias_weights:0'] embedding_weights = trainable_vars[ 'linear_model/aaa_embedding/embedding_weights:0'] linear_weights = trainable_vars['linear_model/aaa_embedding/weights:0'] @@ -5757,7 +5796,7 @@ class SharedEmbeddingColumnTest(test.TestCase): # Linear weights do not follow the column name. But this is a rare use # case, and fixing it would add too much complexity to the code. expected_var_names = ( - 'linear_model/bias_layer/bias_weights:0', + 'linear_model/bias_weights:0', 'linear_model/aaa_bbb_shared_embedding/weights:0', 'linear_model/aaa_bbb_shared_embedding/embedding_weights:0', 'linear_model/aaa_bbb_shared_embedding_1/weights:0', @@ -5770,7 +5809,7 @@ class SharedEmbeddingColumnTest(test.TestCase): for v in ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES) } self.assertItemsEqual(expected_var_names, trainable_vars.keys()) - bias = trainable_vars['linear_model/bias_layer/bias_weights:0'] + bias = trainable_vars['linear_model/bias_weights:0'] embedding_weights = trainable_vars[ 'linear_model/aaa_bbb_shared_embedding/embedding_weights:0'] linear_weights_a = trainable_vars[ @@ -6105,7 +6144,7 @@ class WeightedCategoricalColumnTest(test.TestCase): values=(.5, 1., .1), dense_shape=(2, 2)) }, (column,)) - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() weight_var = get_linear_model_column_var(column) with _initialized_session(): self.assertAllClose((0.,), bias.eval()) @@ -6172,7 +6211,7 @@ class WeightedCategoricalColumnTest(test.TestCase): dense_shape=(2, 2)), 'values': ((.5,), (1.,), (.1,)) }, (column,)) - bias = get_keras_linear_model_bias() + bias = get_linear_model_bias() weight_var = get_linear_model_column_var(column) with _initialized_session(): self.assertAllClose((0.,), bias.eval()) diff --git a/tensorflow/python/training/warm_starting_util_test.py b/tensorflow/python/training/warm_starting_util_test.py index 6e445d8bd1..7e8cbd6bae 100644 --- a/tensorflow/python/training/warm_starting_util_test.py +++ b/tensorflow/python/training/warm_starting_util_test.py @@ -946,18 +946,20 @@ class WarmStartingUtilTest(test.TestCase): # emb_vocab should be correctly warm-started after vocab remapping. # Missing values are filled in with the EmbeddingColumn's initializer. self._assert_cols_to_vars( - cols_to_vars, { + cols_to_vars, + { emb_vocab: [ - # embedding_weights part 0. - np.array([[3., 3.3], [2., 2.2], [1., 1.1]]), - # embedding_weights part 1. - np.array([[0.5, 0.4], [0.42, 0.42], [0.42, 0.42]]), # linear weights part 0. np.array([[0.69]]), # linear weights part 1. - np.array([[0.71]]) + np.array([[0.71]]), + # embedding_weights part 0. + np.array([[3., 3.3], [2., 2.2], [1., 1.1]]), + # embedding_weights part 1. + np.array([[0.5, 0.4], [0.42, 0.42], [0.42, 0.42]]) ] - }, sess) + }, + sess) def testErrorConditions(self): x = variable_scope.get_variable( -- GitLab From 173aadc6b62dd95691257c2d9f158dd9044bb4ef Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 11:55:46 -0700 Subject: [PATCH 668/791] Change estimator to only log non-binary eval metrics, because logging binary metrics such as images will lead to crash. PiperOrigin-RevId: 193551927 --- tensorflow/python/estimator/estimator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py index a42b6cfee8..9862fdecdb 100644 --- a/tensorflow/python/estimator/estimator.py +++ b/tensorflow/python/estimator/estimator.py @@ -1256,7 +1256,8 @@ def _dict_to_str(dictionary): A `str` representing the `dictionary`. """ return ', '.join('%s = %s' % (k, v) - for k, v in sorted(six.iteritems(dictionary))) + for k, v in sorted(six.iteritems(dictionary)) + if not isinstance(v, six.binary_type)) def _write_dict_to_summary(output_dir, -- GitLab From fb02b02689b0e126c93cbcb8462e8417e1d954cc Mon Sep 17 00:00:00 2001 From: Benoit Steiner Date: Thu, 19 Apr 2018 11:57:36 -0700 Subject: [PATCH 669/791] Avoid looking up the shape functions multiple times Improved the handling of fed nodes PiperOrigin-RevId: 193552210 --- .../core/grappler/costs/graph_properties.cc | 155 +++++++++--------- .../core/grappler/costs/graph_properties.h | 7 - 2 files changed, 78 insertions(+), 84 deletions(-) diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc index c83ddfe90a..dd2d53dfdf 100644 --- a/tensorflow/core/grappler/costs/graph_properties.cc +++ b/tensorflow/core/grappler/costs/graph_properties.cc @@ -395,8 +395,11 @@ class TopoQueue { // unknown shape/dimension of a given node. class SymbolicShapeRefiner { public: - explicit SymbolicShapeRefiner(const GraphDef& graph) - : function_library_(OpRegistry::Global(), graph.library()) { + explicit SymbolicShapeRefiner( + const GraphDef& graph, + const std::unordered_map>& fed_ports) + : function_library_(OpRegistry::Global(), graph.library()), + fed_ports_(fed_ports) { graph_def_version_ = graph.versions().producer(); node_to_context_.reserve(graph.node_size()); } @@ -704,6 +707,9 @@ class SymbolicShapeRefiner { std::vector input_tensors_as_shapes; NodeContext& node_ctx = node_to_context_[node]; + TF_RETURN_IF_ERROR( + function_library_.LookUp(node->type_string(), &node_ctx.op_data)); + node_ctx.inference_context.reset(new InferenceContext( graph_def_version_, &node->def(), node->op_def(), input_shapes, input_tensors, input_tensors_as_shapes, @@ -716,6 +722,7 @@ class SymbolicShapeRefiner { } struct NodeContext { + const OpRegistrationData* op_data; std::unique_ptr inference_context; std::vector output_tensors_as_shapes; }; @@ -723,65 +730,80 @@ class SymbolicShapeRefiner { Status InferShapes(const Node* node, NodeContext* c) { InferenceContext* ic = c->inference_context.get(); - // Propagate shape tensors - if (node->type_string() == "Shape") { - c->output_tensors_as_shapes.resize(1); - c->output_tensors_as_shapes[0] = c->inference_context->input(0); - } else if (node->type_string() == "ShapeN") { - c->output_tensors_as_shapes.resize(c->inference_context->num_inputs()); - for (int i = 0; i < c->inference_context->num_inputs(); ++i) { - c->output_tensors_as_shapes[i] = c->inference_context->input(i); - } - } else if (node->type_string() == "ConcatV2") { - bool valid = true; - ShapeHandle result; - for (int i = 0; i < ic->num_inputs() - 1; ++i) { - ShapeHandle input = ic->input_tensors_as_shapes()[i]; - if (!ic->RankKnown(input)) { - valid = false; - break; - } else if (i == 0) { - result = input; - } else { - TF_RETURN_IF_ERROR(ic->Concatenate(result, input, &result)); - } - } - if (valid) { + auto it = fed_ports_.find(node->name()); + const bool is_fed = it != fed_ports_.end(); + + // Propagate shape tensors unless the node is fed. + // TODO(bsteiner) We should still propagate the shapes to the ports that + // aren't fed in the case of a ShapeN node. + if (!is_fed) { + if (node->type_string() == "Shape") { c->output_tensors_as_shapes.resize(1); - c->output_tensors_as_shapes[0] = result; - } - } else if (node->type_string() == "Slice") { - ShapeHandle input = ic->input_tensors_as_shapes()[0]; - bool valid = ic->RankKnown(input); - const Tensor* slice_offset = ic->input_tensor(1); - valid &= slice_offset != nullptr && slice_offset->NumElements() == 1; - const Tensor* slice_size = ic->input_tensor(2); - valid &= slice_size != nullptr && slice_size->NumElements() == 1; - if (valid) { - int64 start = slice_offset->dtype() == DT_INT32 - ? slice_offset->flat()(0) - : slice_offset->flat()(0); - int64 end = start + (slice_size->dtype() == DT_INT32 - ? slice_size->flat()(0) - : slice_size->flat()(0)); + c->output_tensors_as_shapes[0] = c->inference_context->input(0); + } else if (node->type_string() == "ShapeN") { + c->output_tensors_as_shapes.resize(c->inference_context->num_inputs()); + for (int i = 0; i < c->inference_context->num_inputs(); ++i) { + c->output_tensors_as_shapes[i] = c->inference_context->input(i); + } + } else if (node->type_string() == "ConcatV2") { + bool valid = true; ShapeHandle result; - TF_RETURN_IF_ERROR(ic->Subshape(input, start, end, &result)); - c->output_tensors_as_shapes.resize(1); - c->output_tensors_as_shapes[0] = result; + for (int i = 0; i < ic->num_inputs() - 1; ++i) { + ShapeHandle input = ic->input_tensors_as_shapes()[i]; + if (!ic->RankKnown(input)) { + valid = false; + break; + } else if (i == 0) { + result = input; + } else { + TF_RETURN_IF_ERROR(ic->Concatenate(result, input, &result)); + } + } + if (valid) { + c->output_tensors_as_shapes.resize(1); + c->output_tensors_as_shapes[0] = result; + } + } else if (node->type_string() == "Slice") { + ShapeHandle input = ic->input_tensors_as_shapes()[0]; + bool valid = ic->RankKnown(input); + const Tensor* slice_offset = ic->input_tensor(1); + valid &= slice_offset != nullptr && slice_offset->NumElements() == 1; + const Tensor* slice_size = ic->input_tensor(2); + valid &= slice_size != nullptr && slice_size->NumElements() == 1; + if (valid) { + int64 start = slice_offset->dtype() == DT_INT32 + ? slice_offset->flat()(0) + : slice_offset->flat()(0); + int64 end = start + (slice_size->dtype() == DT_INT32 + ? slice_size->flat()(0) + : slice_size->flat()(0)); + ShapeHandle result; + TF_RETURN_IF_ERROR(ic->Subshape(input, start, end, &result)); + c->output_tensors_as_shapes.resize(1); + c->output_tensors_as_shapes[0] = result; + } } } // Infer the shapes of output tensors. - const OpRegistrationData* op_reg_data; - Status s = function_library_.default_registry()->LookUp(node->type_string(), - &op_reg_data); - if (!s.ok() || op_reg_data->shape_inference_fn == nullptr) { + if (!c->op_data || c->op_data->shape_inference_fn == nullptr) { // There is nothing more we can infer, annotate outputs with unknown // shapes return c->inference_context->Run(shape_inference::UnknownShape); } - return c->inference_context->Run(op_reg_data->shape_inference_fn); + TF_RETURN_IF_ERROR( + c->inference_context->Run(c->op_data->shape_inference_fn)); + + Status status = Status::OK(); + if (is_fed) { + // It is possible to feed node output ports with tensors of any shape: as + // a result, the shape of a fed port is completely unknown. + for (const int output_port : it->second) { + status.Update(SetUnknownShape(node, output_port)); + } + } + return status; } NodeContext* GetNodeContext(const Node* node) { @@ -797,6 +819,7 @@ class SymbolicShapeRefiner { std::unordered_map unknown_shapes_; std::unordered_map unknown_dims_; FunctionLibraryDefinition function_library_; + const std::unordered_map>& fed_ports_; }; // Keep track of shapes and dimensions in a graph. @@ -983,23 +1006,6 @@ Status GraphProperties::UpdateMergeNode(SymbolicShapeRefiner* shape_refiner, return Status::OK(); } -Status GraphProperties::OverwriteFedPorts( - SymbolicShapeRefiner* shape_refiner, - const std::unordered_map>& fed_ports, - const Node* node, bool* new_shapes) const { - auto it = fed_ports.find(node->name()); - Status status; - if (it != fed_ports.end()) { - // It is possible to feed node output ports with tensors of any shape: as a - // result, the shape of a fed port is completely unknown. - for (const int output_port : it->second) { - status.Update(shape_refiner->SetUnknownShape(node, output_port)); - } - *new_shapes = true; - } - return status; -} - // Manually propagate the input shape for Enter nodes and update any Merge node // outputs. Status GraphProperties::UpdateEnter(SymbolicShapeRefiner* shape_refiner, @@ -1032,7 +1038,6 @@ Status GraphProperties::UpdateEnter(SymbolicShapeRefiner* shape_refiner, Status GraphProperties::UpdateShapes( SymbolicShapeRefiner* shape_refiner, bool relax, - const std::unordered_map>& fed_ports, const Node* n, bool* new_shapes) const { if (n->IsEnter()) { // The Enter shape function always forwards an UnknownShape, so do the right @@ -1053,9 +1058,7 @@ Status GraphProperties::UpdateShapes( } } } - // Nodes can be fed with any shape. The TensorFlow shape inference code can't - // handle this properly, so overwrite its behavior here. - return OverwriteFedPorts(shape_refiner, fed_ports, n, new_shapes); + return Status::OK(); } // Propagates the shapes in the transitive fan-out of . @@ -1063,7 +1066,6 @@ Status GraphProperties::PropagateShapes( SymbolicShapeRefiner* shape_refiner, bool relax, TopoQueue* new_shapes, const std::unordered_map>& resources, - const std::unordered_map>& fed_ports, int num_loops) const { // Limit the number of iterations to prevent infinite loops in the presence of // incorrect shape functions. The algoritm should converge in at most @@ -1087,8 +1089,7 @@ Status GraphProperties::PropagateShapes( num_loop_iterations++ < max_loop_iterations) { const Node* n = new_shapes->pop(); bool updated = false; - TF_RETURN_IF_ERROR( - UpdateShapes(shape_refiner, relax, fed_ports, n, &updated)); + TF_RETURN_IF_ERROR(UpdateShapes(shape_refiner, relax, n, &updated)); if (updated) { for (const Edge* e : n->out_edges()) { if (!e->IsControlEdge()) { @@ -1243,7 +1244,7 @@ Status GraphProperties::InferStatically(bool assume_valid_feeds) { } } - SymbolicShapeRefiner refiner(item_.graph); + SymbolicShapeRefiner refiner(item_.graph, fed_ports); // We propagate shapes through the graph in two phases. In the first phase, we // exclusively merge shapes but we do not propagate shapes through the @@ -1267,8 +1268,8 @@ Status GraphProperties::InferStatically(bool assume_valid_feeds) { new_shapes.push(node); } // Propagate shapes normally. - TF_RETURN_IF_ERROR(PropagateShapes(&refiner, relax, &new_shapes, resources, - fed_ports, num_loops)); + TF_RETURN_IF_ERROR( + PropagateShapes(&refiner, relax, &new_shapes, resources, num_loops)); } // Track shapes globally across the graph. diff --git a/tensorflow/core/grappler/costs/graph_properties.h b/tensorflow/core/grappler/costs/graph_properties.h index 30351f58fd..4c3f3f5f53 100644 --- a/tensorflow/core/grappler/costs/graph_properties.h +++ b/tensorflow/core/grappler/costs/graph_properties.h @@ -102,16 +102,10 @@ class GraphProperties { // Process the Enter node, and enqueue its fanout in new_shapes if needed. static Status UpdateEnter(SymbolicShapeRefiner* shape_refiner, const Node* node, bool relax, bool* new_shapes); - // Process a node that is used to feed the model. - Status OverwriteFedPorts( - SymbolicShapeRefiner* shape_refiner, - const std::unordered_map>& fed_ports, - const Node* node, bool* new_shapes) const; // Update the shapes for node 'n'. If output shapes for n have changed, // enqueue its fanout in 'new_shapes'. Status UpdateShapes( SymbolicShapeRefiner* shape_refiner, bool relax, - const std::unordered_map>& fed_ports, const Node* n, bool* new_shapes) const; // Propagate the shapes for the nodes enqueued in new_shapes and their // transitive fanout until a fixed point is reached. @@ -119,7 +113,6 @@ class GraphProperties { SymbolicShapeRefiner* shape_refiner, bool relax, TopoQueue* new_shapes, const std::unordered_map>& resources, - const std::unordered_map>& fed_ports, int num_loops) const; // Data members -- GitLab From 0ea0049fa500078c132ed29b60beb8831de26dbb Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 11:57:48 -0700 Subject: [PATCH 670/791] Internal cleanup. PiperOrigin-RevId: 193552240 --- .../java/org/tensorflow/lite/DataType.java | 12 ++- .../java/org/tensorflow/lite/Interpreter.java | 19 +++-- .../lite/NativeInterpreterWrapper.java | 21 +++--- .../main/java/org/tensorflow/lite/Tensor.java | 7 +- .../java/src/main/native/exception_jni.cc | 3 +- .../native/nativeinterpreterwrapper_jni.cc | 74 +++++++++++-------- .../lite/java/src/main/native/tensor_jni.cc | 35 +++++---- .../lite/NativeInterpreterWrapperTest.java | 6 +- 8 files changed, 102 insertions(+), 75 deletions(-) diff --git a/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/DataType.java b/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/DataType.java index fc16488a64..75334cd96e 100644 --- a/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/DataType.java +++ b/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/DataType.java @@ -51,7 +51,11 @@ enum DataType { } } throw new IllegalArgumentException( - "DataType " + c + " is not recognized in Java (version " + TensorFlowLite.version() + ")"); + "DataType error: DataType " + + c + + " is not recognized in Java (version " + + TensorFlowLite.version() + + ")"); } /** Returns byte size of the type. */ @@ -68,7 +72,8 @@ enum DataType { case BYTEBUFFER: return 1; } - throw new IllegalArgumentException("DataType " + this + " is not supported yet"); + throw new IllegalArgumentException( + "DataType error: DataType " + this + " is not supported yet"); } /** Gets string names of the data type. */ @@ -85,7 +90,8 @@ enum DataType { case BYTEBUFFER: return "ByteBuffer"; } - throw new IllegalArgumentException("DataType " + this + " is not supported yet"); + throw new IllegalArgumentException( + "DataType error: DataType " + this + " is not supported yet"); } // Cached to avoid copying it diff --git a/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java b/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java index a33959dca4..e915e65aa1 100644 --- a/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java +++ b/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java @@ -137,17 +137,19 @@ public final class Interpreter implements AutoCloseable { public void runForMultipleInputsOutputs( @NonNull Object[] inputs, @NonNull Map outputs) { if (wrapper == null) { - throw new IllegalStateException("The Interpreter has already been closed."); + throw new IllegalStateException("Internal error: The Interpreter has already been closed."); } Tensor[] tensors = wrapper.run(inputs); if (outputs == null || tensors == null || outputs.size() > tensors.length) { - throw new IllegalArgumentException("Outputs do not match with model outputs."); + throw new IllegalArgumentException("Output error: Outputs do not match with model outputs."); } final int size = tensors.length; for (Integer idx : outputs.keySet()) { if (idx == null || idx < 0 || idx >= size) { throw new IllegalArgumentException( - String.format("Invalid index of output %d (should be in range [0, %d))", idx, size)); + String.format( + "Output error: Invalid index of output %d (should be in range [0, %d))", + idx, size)); } tensors[idx].copyTo(outputs.get(idx)); } @@ -160,7 +162,7 @@ public final class Interpreter implements AutoCloseable { */ public void resizeInput(int idx, @NonNull int[] dims) { if (wrapper == null) { - throw new IllegalStateException("The Interpreter has already been closed."); + throw new IllegalStateException("Internal error: The Interpreter has already been closed."); } wrapper.resizeInput(idx, dims); } @@ -173,7 +175,7 @@ public final class Interpreter implements AutoCloseable { */ public int getInputIndex(String opName) { if (wrapper == null) { - throw new IllegalStateException("The Interpreter has already been closed."); + throw new IllegalStateException("Internal error: The Interpreter has already been closed."); } return wrapper.getInputIndex(opName); } @@ -186,7 +188,7 @@ public final class Interpreter implements AutoCloseable { */ public int getOutputIndex(String opName) { if (wrapper == null) { - throw new IllegalStateException("The Interpreter has already been closed."); + throw new IllegalStateException("Internal error: The Interpreter has already been closed."); } return wrapper.getOutputIndex(opName); } @@ -198,7 +200,7 @@ public final class Interpreter implements AutoCloseable { */ public Long getLastNativeInferenceDurationNanoseconds() { if (wrapper == null) { - throw new IllegalStateException("The interpreter has already been closed."); + throw new IllegalStateException("Internal error: The interpreter has already been closed."); } return wrapper.getLastNativeInferenceDurationNanoseconds(); } @@ -208,7 +210,8 @@ public final class Interpreter implements AutoCloseable { if (wrapper != null) { wrapper.setUseNNAPI(useNNAPI); } else { - throw new IllegalStateException("NativeInterpreterWrapper has already been closed."); + throw new IllegalStateException( + "Internal error: NativeInterpreterWrapper has already been closed."); } } diff --git a/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/NativeInterpreterWrapper.java b/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/NativeInterpreterWrapper.java index fc8187acfe..dfc8ac111a 100644 --- a/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/NativeInterpreterWrapper.java +++ b/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/NativeInterpreterWrapper.java @@ -80,7 +80,7 @@ final class NativeInterpreterWrapper implements AutoCloseable { /** Sets inputs, runs model inference and returns outputs. */ Tensor[] run(Object[] inputs) { if (inputs == null || inputs.length == 0) { - throw new IllegalArgumentException("Invalid inputs. Inputs should not be null or empty."); + throw new IllegalArgumentException("Input error: Inputs should not be null or empty."); } int[] dataTypes = new int[inputs.length]; Object[] sizes = new Object[inputs.length]; @@ -92,7 +92,7 @@ final class NativeInterpreterWrapper implements AutoCloseable { ByteBuffer buffer = (ByteBuffer) inputs[i]; if (buffer.order() != ByteOrder.nativeOrder()) { throw new IllegalArgumentException( - "Invalid ByteBuffer. It shoud use ByteOrder.nativeOrder()."); + "Input error: ByteBuffer shoud use ByteOrder.nativeOrder()."); } numsOfBytes[i] = buffer.limit(); sizes[i] = getInputDims(interpreterHandle, i, numsOfBytes[i]); @@ -103,7 +103,7 @@ final class NativeInterpreterWrapper implements AutoCloseable { } else { throw new IllegalArgumentException( String.format( - "%d-th element of the %d inputs is not an array or a ByteBuffer.", + "Input error: %d-th element of the %d inputs is not an array or a ByteBuffer.", i, inputs.length)); } } @@ -119,7 +119,7 @@ final class NativeInterpreterWrapper implements AutoCloseable { this, isMemoryAllocated); if (outputsHandles == null || outputsHandles.length == 0) { - throw new IllegalStateException("Interpreter has no outputs."); + throw new IllegalStateException("Internal error: Interpreter has no outputs."); } isMemoryAllocated = true; Tensor[] outputs = new Tensor[outputsHandles.length]; @@ -169,7 +169,8 @@ final class NativeInterpreterWrapper implements AutoCloseable { } else { throw new IllegalArgumentException( String.format( - "%s is not a valid name for any input. The indexes of the inputs are %s", + "Input error: %s is not a valid name for any input. " + + "The indexes of the inputs are %s", name, inputsIndexes.toString())); } } @@ -190,7 +191,8 @@ final class NativeInterpreterWrapper implements AutoCloseable { } else { throw new IllegalArgumentException( String.format( - "%s is not a valid name for any output. The indexes of the outputs are %s", + "Input error: %s is not a valid name for any output. " + + "The indexes of the outputs are %s", name, outputsIndexes.toString())); } } @@ -229,7 +231,8 @@ final class NativeInterpreterWrapper implements AutoCloseable { return DataType.BYTEBUFFER; } } - throw new IllegalArgumentException("cannot resolve DataType of " + o.getClass().getName()); + throw new IllegalArgumentException( + "DataType error: cannot resolve DataType of " + o.getClass().getName()); } /** Returns the shape of an object as an int array. */ @@ -245,7 +248,7 @@ final class NativeInterpreterWrapper implements AutoCloseable { return 0; } if (Array.getLength(o) == 0) { - throw new IllegalArgumentException("array lengths cannot be 0."); + throw new IllegalArgumentException("Array lengths cannot be 0."); } return 1 + numDimensions(Array.get(o, 0)); } @@ -259,7 +262,7 @@ final class NativeInterpreterWrapper implements AutoCloseable { shape[dim] = len; } else if (shape[dim] != len) { throw new IllegalArgumentException( - String.format("mismatched lengths (%d and %d) in dimension %d", shape[dim], len, dim)); + String.format("Mismatched lengths (%d and %d) in dimension %d", shape[dim], len, dim)); } for (int i = 0; i < len; ++i) { fillShape(Array.get(o, i), dim + 1, shape); diff --git a/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/Tensor.java b/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/Tensor.java index 54ace6c63c..09e887aae3 100644 --- a/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/Tensor.java +++ b/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/Tensor.java @@ -34,15 +34,16 @@ final class Tensor { if (NativeInterpreterWrapper.dataTypeOf(dst) != dtype) { throw new IllegalArgumentException( String.format( - "Cannot convert an TensorFlowLite tensor with type %s to a Java object of " - + "type %s (which is compatible with the TensorFlowLite type %s)", + "Output error: Cannot convert an TensorFlowLite tensor with type %s to a Java " + + "object of type %s (which is compatible with the TensorFlowLite type %s)", dtype, dst.getClass().getName(), NativeInterpreterWrapper.dataTypeOf(dst))); } int[] dstShape = NativeInterpreterWrapper.shapeOf(dst); if (!Arrays.equals(dstShape, shapeCopy)) { throw new IllegalArgumentException( String.format( - "Shape of output target %s does not match with the shape of the Tensor %s.", + "Output error: Shape of output target %s does not match with the shape of the " + + "Tensor %s.", Arrays.toString(dstShape), Arrays.toString(shapeCopy))); } readMultiDimensionalArray(nativeHandle, dst); diff --git a/tensorflow/contrib/lite/java/src/main/native/exception_jni.cc b/tensorflow/contrib/lite/java/src/main/native/exception_jni.cc index 1578c9e3dd..34d91be04c 100644 --- a/tensorflow/contrib/lite/java/src/main/native/exception_jni.cc +++ b/tensorflow/contrib/lite/java/src/main/native/exception_jni.cc @@ -44,7 +44,8 @@ BufferErrorReporter::BufferErrorReporter(JNIEnv* env, int limit) { buffer_ = new char[limit]; if (!buffer_) { throwException(env, kNullPointerException, - "Malloc of BufferErrorReporter to hold %d char failed.", + "Internal error: Malloc of BufferErrorReporter to hold %d " + "char failed.", limit); return; } diff --git a/tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.cc b/tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.cc index 844226203b..ccfdfd829b 100644 --- a/tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.cc +++ b/tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.cc @@ -22,7 +22,7 @@ const int kBufferSize = 256; tflite::Interpreter* convertLongToInterpreter(JNIEnv* env, jlong handle) { if (handle == 0) { throwException(env, kIllegalArgumentException, - "Invalid handle to Interpreter."); + "Internal error: Invalid handle to Interpreter."); return nullptr; } return reinterpret_cast(handle); @@ -30,7 +30,8 @@ tflite::Interpreter* convertLongToInterpreter(JNIEnv* env, jlong handle) { tflite::FlatBufferModel* convertLongToModel(JNIEnv* env, jlong handle) { if (handle == 0) { - throwException(env, kIllegalArgumentException, "Invalid handle to model."); + throwException(env, kIllegalArgumentException, + "Internal error: Invalid handle to model."); return nullptr; } return reinterpret_cast(handle); @@ -39,7 +40,7 @@ tflite::FlatBufferModel* convertLongToModel(JNIEnv* env, jlong handle) { BufferErrorReporter* convertLongToErrorReporter(JNIEnv* env, jlong handle) { if (handle == 0) { throwException(env, kIllegalArgumentException, - "Invalid handle to ErrorReporter."); + "Internal error: Invalid handle to ErrorReporter."); return nullptr; } return reinterpret_cast(handle); @@ -51,7 +52,7 @@ std::vector convertJIntArrayToVector(JNIEnv* env, jintArray inputs) { jint* ptr = env->GetIntArrayElements(inputs, nullptr); if (ptr == nullptr) { throwException(env, kIllegalArgumentException, - "Empty dimensions of input array."); + "Array has empty dimensions."); return {}; } for (int i = 0; i < size; ++i) { @@ -113,7 +114,7 @@ TfLiteStatus checkInputs(JNIEnv* env, tflite::Interpreter* interpreter, jobjectArray sizes) { if (input_size != interpreter->inputs().size()) { throwException(env, kIllegalArgumentException, - "Expected num of inputs is %d but got %d", + "Input error: Expected num of inputs is %d but got %d", interpreter->inputs().size(), input_size); return kTfLiteError; } @@ -121,8 +122,9 @@ TfLiteStatus checkInputs(JNIEnv* env, tflite::Interpreter* interpreter, input_size != env->GetArrayLength(nums_of_bytes) || input_size != env->GetArrayLength(values)) { throwException(env, kIllegalArgumentException, - "Arrays in arguments should be of the same length, but got " - "%d sizes, %d data_types, %d nums_of_bytes, and %d values", + "Internal error: Arrays in arguments should be of the same " + "length, but got %d sizes, %d data_types, %d nums_of_bytes, " + "and %d values", input_size, env->GetArrayLength(data_types), env->GetArrayLength(nums_of_bytes), env->GetArrayLength(values)); @@ -136,8 +138,8 @@ TfLiteStatus checkInputs(JNIEnv* env, tflite::Interpreter* interpreter, int num_dims = static_cast(env->GetArrayLength(dims)); if (target->dims->size != num_dims) { throwException(env, kIllegalArgumentException, - "%d-th input should have %d dimensions, but found %d " - "dimensions", + "Input error: %d-th input should have %d dimensions, but " + "found %d dimensions", i, target->dims->size, num_dims); return kTfLiteError; } @@ -150,7 +152,8 @@ TfLiteStatus checkInputs(JNIEnv* env, tflite::Interpreter* interpreter, num_dims); printDims(obtained_dims.get(), kBufferSize, ptr, num_dims); throwException(env, kIllegalArgumentException, - "%d-th input dimension should be [%s], but found [%s]", + "Input error: %d-th input dimension should be [%s], but " + "found [%s]", i, expected_dims.get(), obtained_dims.get()); env->ReleaseIntArrayElements(dims, ptr, JNI_ABORT); return kTfLiteError; @@ -236,8 +239,8 @@ TfLiteStatus setInputs(JNIEnv* env, tflite::Interpreter* interpreter, TfLiteType type = resolveDataType(data_type[i]); if (type != target->type) { throwException(env, kIllegalArgumentException, - "DataType (%d) of input data does not match with the " - "DataType (%d) of model inputs.", + "Input error: DataType (%d) of input data does not " + "match with the DataType (%d) of model inputs.", type, target->type); return kTfLiteError; } @@ -270,7 +273,8 @@ Java_org_tensorflow_lite_NativeInterpreterWrapper_getInputNames(JNIEnv* env, jclass string_class = env->FindClass("java/lang/String"); if (string_class == nullptr) { throwException(env, kUnsupportedOperationException, - "Can not find java/lang/String class to get input names."); + "Internal error: Can not find java/lang/String class to get " + "input names."); return nullptr; } size_t size = interpreter->inputs().size(); @@ -292,7 +296,8 @@ Java_org_tensorflow_lite_NativeInterpreterWrapper_getOutputNames(JNIEnv* env, jclass string_class = env->FindClass("java/lang/String"); if (string_class == nullptr) { throwException(env, kUnsupportedOperationException, - "Can not find java/lang/String class to get output names."); + "Internal error: Can not find java/lang/String class to get " + "output names."); return nullptr; } size_t size = interpreter->outputs().size(); @@ -351,8 +356,8 @@ Java_org_tensorflow_lite_NativeInterpreterWrapper_createModel( path, verifier.get(), error_reporter); if (!model) { throwException(env, kIllegalArgumentException, - "Contents of %s does not encode a valid TensorFlowLite " - "model: %s", + "Contents of %s does not encode a valid " + "TensorFlowLite model: %s", path, error_reporter->CachedErrorMessage()); env->ReleaseStringUTFChars(model_file, path); return 0; @@ -380,8 +385,8 @@ Java_org_tensorflow_lite_NativeInterpreterWrapper_createModelWithBuffer( buf, static_cast(capacity), error_reporter); if (!model) { throwException(env, kIllegalArgumentException, - "MappedByteBuffer does not encode a valid TensorFlowLite " - "model: %s", + "MappedByteBuffer does not encode a valid " + "TensorFlowLite model: %s", error_reporter->CachedErrorMessage()); return 0; } @@ -403,7 +408,7 @@ Java_org_tensorflow_lite_NativeInterpreterWrapper_createInterpreter( &interpreter, static_cast(num_threads)); if (status != kTfLiteOk) { throwException(env, kIllegalArgumentException, - "Cannot create interpreter: %s", + "Internal error: Cannot create interpreter: %s", error_reporter->CachedErrorMessage()); return 0; } @@ -411,7 +416,7 @@ Java_org_tensorflow_lite_NativeInterpreterWrapper_createInterpreter( status = interpreter->AllocateTensors(); if (status != kTfLiteOk) { throwException(env, kNullPointerException, - "Can not allocate memory for the interpreter", + "Internal error: Cannot allocate memory for the interpreter", error_reporter->CachedErrorMessage()); return 0; } @@ -440,7 +445,8 @@ Java_org_tensorflow_lite_NativeInterpreterWrapper_run( // resizes inputs status = resizeInputs(env, interpreter, input_size, sizes); if (status != kTfLiteOk) { - throwException(env, kNullPointerException, "Can not resize the input: %s", + throwException(env, kNullPointerException, + "Internal error: Can not resize the input: %s", error_reporter->CachedErrorMessage()); return nullptr; } @@ -448,7 +454,8 @@ Java_org_tensorflow_lite_NativeInterpreterWrapper_run( status = interpreter->AllocateTensors(); if (status != kTfLiteOk) { throwException(env, kNullPointerException, - "Can not allocate memory for the given inputs: %s", + "Internal error: Can not allocate memory for the given " + "inputs: %s", error_reporter->CachedErrorMessage()); return nullptr; } @@ -461,7 +468,7 @@ Java_org_tensorflow_lite_NativeInterpreterWrapper_run( // runs inference if (interpreter->Invoke() != kTfLiteOk) { throwException(env, kIllegalArgumentException, - "Failed to run on the given Interpreter: %s", + "Internal error: Failed to run on the given Interpreter: %s", error_reporter->CachedErrorMessage()); return nullptr; } @@ -479,8 +486,9 @@ Java_org_tensorflow_lite_NativeInterpreterWrapper_run( // returns outputs const std::vector& results = interpreter->outputs(); if (results.empty()) { - throwException(env, kIllegalArgumentException, - "The Interpreter does not have any outputs."); + throwException( + env, kIllegalArgumentException, + "Internal error: The Interpreter does not have any outputs."); return nullptr; } jlongArray outputs = env->NewLongArray(results.size()); @@ -501,7 +509,8 @@ Java_org_tensorflow_lite_NativeInterpreterWrapper_getInputDims( const int idx = static_cast(input_idx); if (input_idx < 0 || input_idx >= interpreter->inputs().size()) { throwException(env, kIllegalArgumentException, - "Out of range: Failed to get %d-th input out of %d inputs", + "Input error: Out of range: Failed to get %d-th input out of" + " %d inputs", input_idx, interpreter->inputs().size()); return nullptr; } @@ -514,8 +523,8 @@ Java_org_tensorflow_lite_NativeInterpreterWrapper_getInputDims( } if (num_bytes != expected_num_bytes) { throwException(env, kIllegalArgumentException, - "Failed to get input dimensions. %d-th input should have" - " %d bytes, but found %d bytes.", + "Input error: Failed to get input dimensions. %d-th input " + "should have %d bytes, but found %d bytes.", idx, expected_num_bytes, num_bytes); return nullptr; } @@ -533,8 +542,8 @@ Java_org_tensorflow_lite_NativeInterpreterWrapper_getOutputDataType( const int idx = static_cast(output_idx); if (output_idx < 0 || output_idx >= interpreter->outputs().size()) { throwException(env, kIllegalArgumentException, - "Out of range: Failed to get %d-th output out of %d outputs", - output_idx, interpreter->outputs().size()); + "Failed to get %d-th output out of %d outputs", output_idx, + interpreter->outputs().size()); return -1; } TfLiteTensor* target = interpreter->tensor(interpreter->outputs()[idx]); @@ -555,7 +564,8 @@ Java_org_tensorflow_lite_NativeInterpreterWrapper_resizeInput( const int idx = static_cast(input_idx); if (idx < 0 || idx >= interpreter->inputs().size()) { throwException(env, kIllegalArgumentException, - "Can not resize %d-th input for a model having %d inputs.", + "Input error: Can not resize %d-th input for a model having " + "%d inputs.", idx, interpreter->inputs().size()); return JNI_FALSE; } @@ -567,7 +577,7 @@ Java_org_tensorflow_lite_NativeInterpreterWrapper_resizeInput( interpreter->inputs()[idx], convertJIntArrayToVector(env, dims)); if (status != kTfLiteOk) { throwException(env, kIllegalArgumentException, - "Failed to resize %d-th input: %s", idx, + "Internal error: Failed to resize %d-th input: %s", idx, error_reporter->CachedErrorMessage()); return JNI_FALSE; } diff --git a/tensorflow/contrib/lite/java/src/main/native/tensor_jni.cc b/tensorflow/contrib/lite/java/src/main/native/tensor_jni.cc index 65126e78a3..17f4be09c6 100644 --- a/tensorflow/contrib/lite/java/src/main/native/tensor_jni.cc +++ b/tensorflow/contrib/lite/java/src/main/native/tensor_jni.cc @@ -23,7 +23,7 @@ namespace { TfLiteTensor* convertLongToTensor(JNIEnv* env, jlong handle) { if (handle == 0) { throwException(env, kIllegalArgumentException, - "Invalid handle to TfLiteTensor."); + "Internal error: Invalid handle to TfLiteTensor."); return nullptr; } return reinterpret_cast(handle); @@ -36,7 +36,8 @@ size_t writeOneDimensionalArray(JNIEnv* env, jobject object, TfLiteType type, size_t to_copy = num_elements * elementByteSize(type); if (to_copy > dst_size) { throwException(env, kIllegalStateException, - "cannot write Java array of %d bytes to Tensor of %d bytes", + "Internal error: cannot write Java array of %d bytes to " + "Tensor of %d bytes", to_copy, dst_size); return 0; } @@ -71,10 +72,10 @@ size_t writeOneDimensionalArray(JNIEnv* env, jobject object, TfLiteType type, } default: { throwException(env, kUnsupportedOperationException, - "TensorFlowLite currently supports float (32 bits), " - "int (32 bits), byte (8 bits), and long (64 bits), " - "support for other types (DataType %d in this case) will " - "be added in the future", + "DataType error: TensorFlowLite currently supports float " + "(32 bits), int (32 bits), byte (8 bits), and long " + "(64 bits), support for other types (DataType %d in this " + "case) will be added in the future", kTfLiteFloat32, type); return 0; } @@ -88,8 +89,9 @@ size_t readOneDimensionalArray(JNIEnv* env, TfLiteType data_type, if (size > src_size) { throwException( env, kIllegalStateException, - "cannot fill a Java array of %d bytes with a Tensor of %d bytes", size, - src_size); + "Internal error: cannot fill a Java array of %d bytes with a Tensor of " + "%d bytes", + size, src_size); return 0; } switch (data_type) { @@ -117,8 +119,8 @@ size_t readOneDimensionalArray(JNIEnv* env, TfLiteType data_type, return size; } default: { - throwException(env, kIllegalStateException, "invalid DataType(%d)", - data_type); + throwException(env, kIllegalStateException, + "DataType error: invalid DataType(%d)", data_type); } } return 0; @@ -152,19 +154,22 @@ size_t elementByteSize(TfLiteType data_type) { switch (data_type) { case kTfLiteFloat32: static_assert(sizeof(jfloat) == 4, - "Java float not compatible with kTfLiteFloat"); + "Interal error: Java float not compatible with " + "kTfLiteFloat"); return 4; case kTfLiteInt32: static_assert(sizeof(jint) == 4, - "Java int not compatible with kTfLiteInt"); + "Interal error: Java int not compatible with kTfLiteInt"); return 4; case kTfLiteUInt8: static_assert(sizeof(jbyte) == 1, - "Java byte not compatible with kTfLiteUInt8"); + "Interal error: Java byte not compatible with " + "kTfLiteUInt8"); return 1; case kTfLiteInt64: static_assert(sizeof(jlong) == 8, - "Java long not compatible with kTfLiteInt64"); + "Interal error: Java long not compatible with " + "kTfLiteInt64"); return 8; default: return 0; @@ -212,7 +217,7 @@ Java_org_tensorflow_lite_Tensor_readMultiDimensionalArray(JNIEnv* env, int num_dims = tensor->dims->size; if (num_dims == 0) { throwException(env, kIllegalArgumentException, - "copyTo() is not meant for scalar Tensors."); + "Internal error: Cannot copy empty/scalar Tensors."); return; } readMultiDimensionalArray(env, tensor->type, tensor->data.raw, tensor->bytes, diff --git a/tensorflow/contrib/lite/java/src/test/java/org/tensorflow/lite/NativeInterpreterWrapperTest.java b/tensorflow/contrib/lite/java/src/test/java/org/tensorflow/lite/NativeInterpreterWrapperTest.java index dbe45e5a05..7c00d3196f 100644 --- a/tensorflow/contrib/lite/java/src/test/java/org/tensorflow/lite/NativeInterpreterWrapperTest.java +++ b/tensorflow/contrib/lite/java/src/test/java/org/tensorflow/lite/NativeInterpreterWrapperTest.java @@ -321,9 +321,7 @@ public final class NativeInterpreterWrapperTest { wrapper.run(inputs); fail(); } catch (IllegalArgumentException e) { - assertThat(e) - .hasMessageThat() - .contains("Invalid inputs. Inputs should not be null or empty."); + assertThat(e).hasMessageThat().contains("Inputs should not be null or empty."); } wrapper.close(); } @@ -440,7 +438,7 @@ public final class NativeInterpreterWrapperTest { NativeInterpreterWrapper.numDimensions(emptyArray); fail(); } catch (IllegalArgumentException e) { - assertThat(e).hasMessageThat().contains("array lengths cannot be 0."); + assertThat(e).hasMessageThat().contains("Array lengths cannot be 0."); } } -- GitLab From 16d25e8c8a9ebb6500d3b3418ca8c2bb80c3e42e Mon Sep 17 00:00:00 2001 From: Francois Chollet Date: Thu, 19 Apr 2018 11:58:04 -0700 Subject: [PATCH 671/791] Add support for Dataset Iterators in Model training/eval methods in graph mode. PiperOrigin-RevId: 193552275 --- tensorflow/python/keras/BUILD | 1 + .../keras/_impl/keras/engine/training.py | 195 ++++++++++++------ .../_impl/keras/engine/training_arrays.py | 12 +- .../keras/_impl/keras/engine/training_test.py | 84 +++++++- .../api/golden/tensorflow.keras.-model.pbtxt | 4 +- .../golden/tensorflow.keras.-sequential.pbtxt | 4 +- .../tensorflow.keras.models.-model.pbtxt | 4 +- .../tensorflow.keras.models.-sequential.pbtxt | 4 +- 8 files changed, 223 insertions(+), 85 deletions(-) diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD index ca7686b1d1..70040b7e74 100755 --- a/tensorflow/python/keras/BUILD +++ b/tensorflow/python/keras/BUILD @@ -175,6 +175,7 @@ py_library( srcs_version = "PY2AND3", deps = [ ":backend", + "//tensorflow/python/data", "@six_archive//:six", ], ) diff --git a/tensorflow/python/keras/_impl/keras/engine/training.py b/tensorflow/python/keras/_impl/keras/engine/training.py index 012d9ceea4..146e8fdac9 100644 --- a/tensorflow/python/keras/_impl/keras/engine/training.py +++ b/tensorflow/python/keras/_impl/keras/engine/training.py @@ -20,6 +20,8 @@ from __future__ import print_function import numpy as np +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.data.ops import iterator_ops from tensorflow.python.eager import context from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_util @@ -634,12 +636,20 @@ class Model(Network): This is a purely internal method, subject to refactoring at any time. Args: - x: An array or list of arrays, to be used as input data. If the model - has known, named inputs, this could also be a dict mapping input names - to the corresponding array. - y: An array or list of arrays, to be used as target data. If the model - has known, named outputs, this could also be a dict mapping output names - to the corresponding array. + x: Input data. It could be: + - A Numpy array (or array-like), or a list of arrays + (in case the model has multiple inputs). + - A TensorFlow tensor, or a list of tensors + (in case the model has multiple inputs). + - A dict mapping input names to the corresponding array/tensors, + if the model has named inputs. + - A `tf.data` dataset iterator. + y: Target data. Like the input data `x`, + it could be either Numpy array(s) or TensorFlow tensor(s). + It should be consistent with `x` (you cannot have Numpy inputs and + tensor targets, or inversely). If `x` is a dataset iterator, + `y` should not be specified + (since targets will be obtained from the iterator). sample_weight: An optional sample-weight array passed by the user to weight the importance of each sample in `x`. class_weight: An optional class-weight array by the user to @@ -659,6 +669,31 @@ class Model(Network): RuntimeError: If the model was never compiled. """ # First, we build/compile the model on the fly if necessary. + if isinstance(x, dataset_ops.Dataset): + raise ValueError('You passed a `Dataset` instance to your model (%s), ' + 'which is not supported. Instead, pass an `Iterator`, ' + 'which you can obtain e.g. via ' + '`dataset.make_one_shot_iterator()` (the exact method ' + 'to use will depend on your specific dataset).' % x) + if isinstance(x, iterator_ops.Iterator): + if y is not None: + raise ValueError('You passed a dataset iterator (%s) as input `x` to ' + 'your model. In that case, you should not specify ' + 'a target (`y`) argument, since the dataset iterator ' + 'generates both input data and target data. ' + 'Received: %s' % (x, y)) + if not context.executing_eagerly(): + x, y = x.get_next() + # TODO(fchollet): handle case of `get_next` not returning 2 tensors? + else: + # TODO(psv): implement this. The way to support it will be to typecheck + # for `iterator` before `_standardize_user_data` is called and redirect + # to new training/eval functions in `training_eager.py`. The model + # may need to get built using the specs of the data from the first batch + # drawn from the iterator. + raise ValueError('Dataset iterators are not supported ' + 'with eager execution yet.') + all_inputs = [] if not self.built: # We need to use `x` to set the model inputs. @@ -1016,22 +1051,26 @@ class Model(Network): """Trains the model for a fixed number of epochs (iterations on a dataset). Arguments: - x: Numpy array of training data (if the model has a single input), - or list of Numpy arrays (if the model has multiple inputs). - If input layers in the model are named, you can also pass a - dictionary mapping input names to Numpy arrays. - `x` can be `None` (default) if feeding from - TensorFlow data tensors. - y: Numpy array of target (label) data - (if the model has a single output), - or list of Numpy arrays (if the model has multiple outputs). - If output layers in the model are named, you can also pass a - dictionary mapping output names to Numpy arrays. - `y` can be `None` (default) if feeding from - TensorFlow data tensors. + x: Input data. It could be: + - A Numpy array (or array-like), or a list of arrays + (in case the model has multiple inputs). + - A TensorFlow tensor, or a list of tensors + (in case the model has multiple inputs). + - A dict mapping input names to the corresponding array/tensors, + if the model has named inputs. + - A `tf.data` dataset iterator. + y: Target data. Like the input data `x`, + it could be either Numpy array(s) or TensorFlow tensor(s). + It should be consistent with `x` (you cannot have Numpy inputs and + tensor targets, or inversely). If `x` is a dataset iterator, + `y` should not be specified + (since targets will be obtained from the iterator). batch_size: Integer or `None`. Number of samples per gradient update. If unspecified, `batch_size` will default to 32. + Do not specify the `batch_size` is your data is in the + form of symbolic tensors or dataset iterators (since they generate + batches). epochs: Integer. Number of epochs to train the model. An epoch is an iteration over the entire `x` and `y` data provided. @@ -1053,11 +1092,14 @@ class Model(Network): on this data at the end of each epoch. The validation data is selected from the last samples in the `x` and `y` data provided, before shuffling. - validation_data: tuple `(x_val, y_val)` or tuple - `(x_val, y_val, val_sample_weights)` on which to evaluate + validation_data: Data on which to evaluate the loss and any model metrics at the end of each epoch. The model will not be trained on this data. `validation_data` will override `validation_split`. + `validation_data` could be: + - tuple `(x_val, y_val)` of Numpy arrays or tensors + - tuple `(x_val, y_val, val_sample_weights)` of Numpy arrays + - dataset iterator shuffle: Boolean (whether to shuffle the training data before each epoch) or str (for 'batch'). 'batch' is a special option for dealing with the @@ -1134,17 +1176,22 @@ class Model(Network): batch_size=batch_size) # Prepare validation data. if validation_data: - if len(validation_data) == 2: + if isinstance(validation_data, iterator_ops.Iterator): + val_x = validation_data + val_y = None + val_sample_weight = None + elif len(validation_data) == 2: val_x, val_y = validation_data # pylint: disable=unpacking-non-sequence val_sample_weight = None elif len(validation_data) == 3: val_x, val_y, val_sample_weight = validation_data # pylint: disable=unpacking-non-sequence else: raise ValueError( - 'When passing validation_data, ' - 'it must contain 2 (x_val, y_val) ' - 'or 3 (x_val, y_val, val_sample_weights) ' - 'items, however it contains %d items' % len(validation_data)) + 'When passing a `validation_data` argument, ' + 'it must contain either 2 items (x_val, y_val), ' + 'or 3 items (x_val, y_val, val_sample_weights), ' + 'or alternatively it could be a dataset iterator. However we ' + 'received `validation_data=%s`' % validation_data) val_x, val_y, val_sample_weights = self._standardize_user_data( val_x, @@ -1218,22 +1265,26 @@ class Model(Network): Computation is done in batches. Arguments: - x: Numpy array of test data (if the model has a single input), - or list of Numpy arrays (if the model has multiple inputs). - If input layers in the model are named, you can also pass a - dictionary mapping input names to Numpy arrays. - `x` can be `None` (default) if feeding from - TensorFlow data tensors. - y: Numpy array of target (label) data - (if the model has a single output), - or list of Numpy arrays (if the model has multiple outputs). - If output layers in the model are named, you can also pass a - dictionary mapping output names to Numpy arrays. - `y` can be `None` (default) if feeding from - TensorFlow data tensors. + x: Input data. It could be: + - A Numpy array (or array-like), or a list of arrays + (in case the model has multiple inputs). + - A TensorFlow tensor, or a list of tensors + (in case the model has multiple inputs). + - A dict mapping input names to the corresponding array/tensors, + if the model has named inputs. + - A `tf.data` dataset iterator. + y: Target data. Like the input data `x`, + it could be either Numpy array(s) or TensorFlow tensor(s). + It should be consistent with `x` (you cannot have Numpy inputs and + tensor targets, or inversely). If `x` is a dataset iterator, + `y` should not be specified + (since targets will be obtained from the iterator). batch_size: Integer or `None`. - Number of samples per evaluation step. + Number of samples per gradient update. If unspecified, `batch_size` will default to 32. + Do not specify the `batch_size` is your data is in the + form of symbolic tensors or dataset iterators (since they generate + batches). verbose: 0 or 1. Verbosity mode. 0 = silent, 1 = progress bar. sample_weight: Optional Numpy array of weights for @@ -1291,9 +1342,13 @@ class Model(Network): Computation is done in batches. Arguments: - x: The input data, as a Numpy array - (or list of Numpy arrays if the model has multiple outputs). - batch_size: Integer. If unspecified, it will default to 32. + x: Input samples, as Numpy array(s) or tensor(s). + batch_size: Integer or `None`. + Number of samples per gradient update. + If unspecified, `batch_size` will default to 32. + Do not specify the `batch_size` is your data is in the + form of symbolic tensors or dataset iterators (since they generate + batches). verbose: Verbosity mode, 0 or 1. steps: Total number of steps (batches of samples) before declaring the prediction round finished. @@ -1324,20 +1379,24 @@ class Model(Network): return training_arrays.predict_loop( self, x, batch_size=batch_size, verbose=verbose, steps=steps) - def train_on_batch(self, x, y, sample_weight=None, class_weight=None): + def train_on_batch(self, x, y=None, sample_weight=None, class_weight=None): """Runs a single gradient update on a single batch of data. Arguments: - x: Numpy array of training data, - or list of Numpy arrays if the model has multiple inputs. - If all inputs in the model are named, - you can also pass a dictionary - mapping input names to Numpy arrays. - y: Numpy array of target data, - or list of Numpy arrays if the model has multiple outputs. - If all outputs in the model are named, - you can also pass a dictionary - mapping output names to Numpy arrays. + x: Input data. It could be: + - A Numpy array (or array-like), or a list of arrays + (in case the model has multiple inputs). + - A TensorFlow tensor, or a list of tensors + (in case the model has multiple inputs). + - A dict mapping input names to the corresponding array/tensors, + if the model has named inputs. + - A `tf.data` dataset iterator. + y: Target data. Like the input data `x`, + it could be either Numpy array(s) or TensorFlow tensor(s). + It should be consistent with `x` (you cannot have Numpy inputs and + tensor targets, or inversely). If `x` is a dataset iterator, + `y` should not be specified + (since targets will be obtained from the iterator). sample_weight: Optional array of the same length as x, containing weights to apply to the model's loss for each sample. In the case of temporal data, you can pass a 2D array @@ -1384,20 +1443,24 @@ class Model(Network): return outputs[0] return outputs - def test_on_batch(self, x, y, sample_weight=None): + def test_on_batch(self, x, y=None, sample_weight=None): """Test the model on a single batch of samples. Arguments: - x: Numpy array of test data, - or list of Numpy arrays if the model has multiple inputs. - If all inputs in the model are named, - you can also pass a dictionary - mapping input names to Numpy arrays. - y: Numpy array of target data, - or list of Numpy arrays if the model has multiple outputs. - If all outputs in the model are named, - you can also pass a dictionary - mapping output names to Numpy arrays. + x: Input data. It could be: + - A Numpy array (or array-like), or a list of arrays + (in case the model has multiple inputs). + - A TensorFlow tensor, or a list of tensors + (in case the model has multiple inputs). + - A dict mapping input names to the corresponding array/tensors, + if the model has named inputs. + - A `tf.data` dataset iterator. + y: Target data. Like the input data `x`, + it could be either Numpy array(s) or TensorFlow tensor(s). + It should be consistent with `x` (you cannot have Numpy inputs and + tensor targets, or inversely). If `x` is a dataset iterator, + `y` should not be specified + (since targets will be obtained from the iterator). sample_weight: Optional array of the same length as x, containing weights to apply to the model's loss for each sample. In the case of temporal data, you can pass a 2D array @@ -1437,7 +1500,7 @@ class Model(Network): """Returns predictions for a single batch of samples. Arguments: - x: Input samples, as a Numpy array. + x: Input samples, as Numpy array(s) or tensor(s). Returns: Numpy array(s) of predictions. diff --git a/tensorflow/python/keras/_impl/keras/engine/training_arrays.py b/tensorflow/python/keras/_impl/keras/engine/training_arrays.py index 18116e3a14..4164cae864 100644 --- a/tensorflow/python/keras/_impl/keras/engine/training_arrays.py +++ b/tensorflow/python/keras/_impl/keras/engine/training_arrays.py @@ -23,6 +23,7 @@ import copy import numpy as np +from tensorflow.python.framework import errors from tensorflow.python.keras._impl.keras import backend as K from tensorflow.python.keras._impl.keras import callbacks as cbks from tensorflow.python.keras._impl.keras.engine import training_utils @@ -30,6 +31,7 @@ from tensorflow.python.keras._impl.keras.engine.base_layer import Layer from tensorflow.python.keras._impl.keras.utils.generic_utils import make_batches from tensorflow.python.keras._impl.keras.utils.generic_utils import Progbar from tensorflow.python.keras._impl.keras.utils.generic_utils import slice_arrays +from tensorflow.python.platform import tf_logging as logging try: from scipy.sparse import issparse # pylint: disable=g-import-not-at-top @@ -190,7 +192,15 @@ def fit_loop(model, batch_logs['batch'] = step_index batch_logs['size'] = 1 callbacks.on_batch_begin(step_index, batch_logs) - outs = f(ins) + try: + outs = f(ins) + except errors.OutOfRangeError: + logging.warning('Your dataset iterator ran out of data; ' + 'interrupting training. Make sure that your dataset ' + 'can generate at least `steps_per_epoch * epochs` ' + 'batches (in this case, %d batches).' % + steps_per_epoch * epochs) + break if not isinstance(outs, list): outs = [outs] diff --git a/tensorflow/python/keras/_impl/keras/engine/training_test.py b/tensorflow/python/keras/_impl/keras/engine/training_test.py index d9281436de..58011a1412 100644 --- a/tensorflow/python/keras/_impl/keras/engine/training_test.py +++ b/tensorflow/python/keras/_impl/keras/engine/training_test.py @@ -23,6 +23,7 @@ import unittest import numpy as np +from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import test_util as tf_test_util from tensorflow.python.keras._impl import keras @@ -31,9 +32,9 @@ from tensorflow.python.keras._impl.keras.engine.training_utils import weighted_m from tensorflow.python.keras._impl.keras.utils.generic_utils import slice_arrays from tensorflow.python.ops import array_ops from tensorflow.python.platform import test +from tensorflow.python.platform import tf_logging as logging from tensorflow.python.training.rmsprop import RMSPropOptimizer - try: import scipy.sparse as scipy_sparse # pylint: disable=g-import-not-at-top except ImportError: @@ -1711,14 +1712,77 @@ class TestTrainingWithDataTensors(test.TestCase): 'dropout_acc'] self.assertEqual(reference_metric_names, model.metrics_names) -if __name__ == '__main__': - # Bazel sets these environment variables to very long paths. - # Tempfile uses them to create long paths, and in turn multiprocessing - # library tries to create sockets named after paths. Delete whatever bazel - # writes to these to avoid tests failing due to socket addresses being too - # long. - for var in ('TMPDIR', 'TMP', 'TEMP'): - if var in os.environ: - del os.environ[var] +class TestTrainingWithDatasetIterators(test.TestCase): + + def test_training_and_eval_methods_on_iterators_single_io(self): + with self.test_session(): + x = keras.layers.Input(shape=(3,), name='input') + y = keras.layers.Dense(4, name='dense')(x) + model = keras.Model(x, y) + + optimizer = 'rmsprop' + loss = 'mse' + metrics = ['mae'] + model.compile(optimizer, loss, metrics=metrics) + + inputs = np.zeros((10, 3)) + targets = np.zeros((10, 4)) + dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) + dataset = dataset.repeat(100) + dataset = dataset.batch(10) + iterator = dataset.make_one_shot_iterator() + + model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=0) + model.evaluate(iterator, steps=2, verbose=0) + model.predict(iterator, steps=2) + model.train_on_batch(iterator) + model.test_on_batch(iterator) + # Test with validation data + model.fit(iterator, + epochs=1, steps_per_epoch=2, verbose=0, + validation_data=iterator, validation_steps=2) + # Test with validation split + with self.assertRaisesRegexp(ValueError, + 'you cannot use `validation_split`'): + model.fit(iterator, + epochs=1, steps_per_epoch=2, verbose=0, + validation_split=0.5, validation_steps=2) + + # Test invalid usage + with self.assertRaisesRegexp(ValueError, + 'Instead, pass an `Iterator`'): + model.fit(dataset, + epochs=1, steps_per_epoch=2, verbose=0) + with self.assertRaisesRegexp(ValueError, + 'you should not specify a target'): + model.fit(iterator, iterator, + epochs=1, steps_per_epoch=2, verbose=0) + + def test_iterators_running_out_of_data(self): + with self.test_session(): + x = keras.layers.Input(shape=(3,), name='input') + y = keras.layers.Dense(4, name='dense')(x) + model = keras.Model(x, y) + + optimizer = 'rmsprop' + loss = 'mse' + metrics = ['mae'] + model.compile(optimizer, loss, metrics=metrics) + + inputs = np.zeros((10, 3)) + targets = np.zeros((10, 4)) + dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) + dataset = dataset.repeat(2) + dataset = dataset.batch(10) + iterator = dataset.make_one_shot_iterator() + + with test.mock.patch.object(logging, 'warning') as mock_log: + model.fit(iterator, epochs=1, steps_per_epoch=3, verbose=0) + self.assertRegexpMatches( + str(mock_log.call_args), + 'dataset iterator ran out of data') + + +if __name__ == '__main__': test.main() diff --git a/tensorflow/tools/api/golden/tensorflow.keras.-model.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.-model.pbtxt index 7713d78b8a..cdf2da712f 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.-model.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.-model.pbtxt @@ -251,7 +251,7 @@ tf_class { } member_method { name: "test_on_batch" - argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "to_json" @@ -263,6 +263,6 @@ tf_class { } member_method { name: "train_on_batch" - argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } } diff --git a/tensorflow/tools/api/golden/tensorflow.keras.-sequential.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.-sequential.pbtxt index 69b81f75fa..5c2c29e60f 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.-sequential.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.-sequential.pbtxt @@ -268,7 +268,7 @@ tf_class { } member_method { name: "test_on_batch" - argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "to_json" @@ -280,6 +280,6 @@ tf_class { } member_method { name: "train_on_batch" - argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } } diff --git a/tensorflow/tools/api/golden/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.models.-model.pbtxt index 3ac285681f..b3f3f16922 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.models.-model.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.models.-model.pbtxt @@ -251,7 +251,7 @@ tf_class { } member_method { name: "test_on_batch" - argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "to_json" @@ -263,6 +263,6 @@ tf_class { } member_method { name: "train_on_batch" - argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } } diff --git a/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt index 51ba0c5043..4ac6811bac 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt @@ -268,7 +268,7 @@ tf_class { } member_method { name: "test_on_batch" - argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "to_json" @@ -280,6 +280,6 @@ tf_class { } member_method { name: "train_on_batch" - argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } } -- GitLab From a186c4c093fce7e3fcc8cd59ca0e968324311f09 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 12:32:52 -0700 Subject: [PATCH 672/791] Fix bug in ring_reducer.cc abort handling. PiperOrigin-RevId: 193557334 --- .../core/common_runtime/ring_reducer.cc | 20 ++++++++++--------- .../core/common_runtime/ring_reducer_test.cc | 12 +++++------ 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/tensorflow/core/common_runtime/ring_reducer.cc b/tensorflow/core/common_runtime/ring_reducer.cc index 79d03a24ce..a1cd762505 100644 --- a/tensorflow/core/common_runtime/ring_reducer.cc +++ b/tensorflow/core/common_runtime/ring_reducer.cc @@ -426,17 +426,20 @@ bool RingReducer::RunAsyncParts() { // is done. bool dispatched = false; // true if async action was initiated do { - if (aborted) break; + if (aborted) { + // Requeue this RingField to be counted off below. + ready_queue.Enqueue(rf); + break; + } switch (rf->action) { case RF_INIT: if (rf->do_recv) { rf->action = RF_RECV; auto requeue = [this, rf, &ready_queue, &aborted](Status s) { - if (!s.ok()) { - aborted = true; - StartAbort(s); - } + const bool bad_status = !s.ok(); + if (bad_status) aborted = true; ready_queue.Enqueue(rf); + if (bad_status) StartAbort(s); }; DispatchRecv(rf, requeue); dispatched = true; @@ -481,11 +484,10 @@ bool RingReducer::RunAsyncParts() { if (rf->do_send) { rf->action = RF_SEND; auto send_complete = [this, rf, &ready_queue, &aborted](Status s) { - if (!s.ok()) { - aborted = true; - StartAbort(s); - } + const bool bad_status = !s.ok(); + if (bad_status) aborted = true; ready_queue.Enqueue(rf); + if (bad_status) StartAbort(s); }; DispatchSend(rf, send_complete); dispatched = true; diff --git a/tensorflow/core/common_runtime/ring_reducer_test.cc b/tensorflow/core/common_runtime/ring_reducer_test.cc index 57c36d6582..e4387a074a 100644 --- a/tensorflow/core/common_runtime/ring_reducer_test.cc +++ b/tensorflow/core/common_runtime/ring_reducer_test.cc @@ -572,9 +572,9 @@ DEF_TEST(INT32, CPU, 2, 8, 3, 4095, 0) DEF_TEST(INT64, CPU, 1, 2, 1, 1001, 0) DEF_TEST(INT64, CPU, 2, 8, 3, 4095, 0) -// // Failure tests -// DEF_TEST(FLOAT, CPU, 2, 8, 1, 9408, 7) -// DEF_TEST(FLOAT, CPU, 2, 8, 2, 9408, 11) +// Failure tests +DEF_TEST(FLOAT, CPU, 2, 8, 1, 9408, 7) +DEF_TEST(FLOAT, CPU, 2, 8, 2, 9408, 11) #endif #ifdef GOOGLE_CUDA @@ -597,9 +597,9 @@ DEF_TEST(DOUBLE, GPU, 1, 2, 1, 1001, 0) // DEF_TEST(INT32, GPU, 1, 2, 1, 1001, 0) DEF_TEST(INT64, GPU, 1, 2, 1, 1001, 0) -// // Failure tests -// DEF_TEST(FLOAT, GPU, 1, 8, 1, 9408, 2) -// DEF_TEST(FLOAT, GPU, 1, 8, 2, 9408, 5) +// Failure tests +DEF_TEST(FLOAT, GPU, 1, 8, 1, 9408, 2) +DEF_TEST(FLOAT, GPU, 1, 8, 2, 9408, 5) #endif } // namespace -- GitLab From 46aec0d27f5d6fb3a0b81bc5a3384da11273dad6 Mon Sep 17 00:00:00 2001 From: Sung Jin Hwang Date: Thu, 19 Apr 2018 12:44:21 -0700 Subject: [PATCH 673/791] Make PmfToQuantizedCdf op to make adjustments if the sum of quantized pmf is less than 2**precision. Prior to the change, the op did nothing when the sum of quantized pmf was less than 2**precision. While the produced CDF was valid for range coders, adjustments to CDF could be made to achieve better compression rate. PiperOrigin-RevId: 193558740 --- .../contrib/coder/kernels/pmf_to_cdf_op.cc | 60 ++++++++++++++++--- .../coder/kernels/pmf_to_cdf_op_test.cc | 6 +- tensorflow/contrib/coder/ops/coder_ops.cc | 16 +++-- 3 files changed, 64 insertions(+), 18 deletions(-) diff --git a/tensorflow/contrib/coder/kernels/pmf_to_cdf_op.cc b/tensorflow/contrib/coder/kernels/pmf_to_cdf_op.cc index c787e8eded..bd5272ee6f 100644 --- a/tensorflow/contrib/coder/kernels/pmf_to_cdf_op.cc +++ b/tensorflow/contrib/coder/kernels/pmf_to_cdf_op.cc @@ -16,6 +16,7 @@ limitations under the License. #define EIGEN_USE_THREADS #include +#include #include #include #include @@ -79,8 +80,8 @@ class PmfToCdfOp : public OpKernel { } private: - struct Item { - Item(int32* p, double mass) : pointer(p), mass(mass) { + struct PenaltyItem { + PenaltyItem(int32* p, double mass) : pointer(p), mass(mass) { penalty = ComputeNextPenalty(); } @@ -90,7 +91,7 @@ class PmfToCdfOp : public OpKernel { penalty = ComputeNextPenalty(); } - friend bool operator<(const Item& lhs, const Item& rhs) { + friend bool operator<(const PenaltyItem& lhs, const PenaltyItem& rhs) { return lhs.penalty < rhs.penalty; } @@ -106,6 +107,34 @@ class PmfToCdfOp : public OpKernel { double penalty; }; + struct GainItem { + GainItem(int32* p, double mass) : pointer(p), mass(mass) { + gain = ComputeNextGain(); + } + + void Increase() { + CHECK_GT(*pointer, 0); + ++*pointer; + gain = ComputeNextGain(); + } + + friend bool operator>(const GainItem& lhs, const GainItem& rhs) { + return lhs.gain > rhs.gain; + } + + double ComputeNextGain() { + // Never increment zero value to non-zero value. + if (*pointer < 1) { + return -std::numeric_limits::infinity(); + } + return mass * (std::log2(*pointer + 1) - std::log2(*pointer)); + } + + int32* pointer; + double mass; + double gain; + }; + void PerShard(gtl::ArraySlice pmf, gtl::MutableArraySlice cdf) const { CHECK_EQ(pmf.size(), cdf.size()); @@ -121,7 +150,7 @@ class PmfToCdfOp : public OpKernel { int32 sum = std::accumulate(cdf.begin(), cdf.end(), 0); if (sum > normalizer) { - std::vector queue; + std::vector queue; queue.reserve(cdf.size()); for (int i = 0; i < cdf.size(); ++i) { queue.emplace_back(&cdf[i], pmf[i]); @@ -132,9 +161,26 @@ class PmfToCdfOp : public OpKernel { queue[0].Decrease(); // Performs a linear search because this find_if is likely to return // iterator very close to the begin. - auto iter = - std::find_if(std::next(queue.begin()), queue.end(), - [&queue](const Item& rhs) { return queue[0] < rhs; }); + auto iter = std::find_if( + std::next(queue.begin()), queue.end(), + [&queue](const PenaltyItem& rhs) { return queue[0] < rhs; }); + std::rotate(queue.begin(), std::next(queue.begin()), iter); + } + } else if (sum < normalizer) { + std::vector queue; + queue.reserve(cdf.size()); + for (int i = 0; i < cdf.size(); ++i) { + queue.emplace_back(&cdf[i], pmf[i]); + } + + std::sort(queue.begin(), queue.end(), std::greater()); + while (sum++ < normalizer) { + queue[0].Increase(); + // Performs a linear search because this find_if is likely to return + // iterator very close to the begin. + auto iter = std::find_if( + std::next(queue.begin()), queue.end(), + [&queue](const GainItem& rhs) { return queue[0] > rhs; }); std::rotate(queue.begin(), std::next(queue.begin()), iter); } } diff --git a/tensorflow/contrib/coder/kernels/pmf_to_cdf_op_test.cc b/tensorflow/contrib/coder/kernels/pmf_to_cdf_op_test.cc index c70e38faab..3408f6b519 100644 --- a/tensorflow/contrib/coder/kernels/pmf_to_cdf_op_test.cc +++ b/tensorflow/contrib/coder/kernels/pmf_to_cdf_op_test.cc @@ -82,7 +82,7 @@ class PmfToQuantizedCdfOpTest : public OpsTestBase { EXPECT_GT(diff, 0); } - EXPECT_LE(cdf_slice(cdf_slice.size() - 1), normalizer); + EXPECT_EQ(cdf_slice(cdf_slice.size() - 1), normalizer); } } }; @@ -98,6 +98,8 @@ TEST_F(PmfToQuantizedCdfOpTest, UnderSum) { GenerateData(&rand, {&matrix(i, 0), n}); } + pmf.flat() = pmf.flat() * 0.85f; + constexpr int kPrecision = 10; SetupOp(kPrecision, &pmf); TF_ASSERT_OK(RunOpKernel()); @@ -115,7 +117,7 @@ TEST_F(PmfToQuantizedCdfOpTest, OverSum) { matrix.setZero(); const std::size_t n = matrix.dimension(1) / 2; - random::PhiloxRandom gen; + random::PhiloxRandom gen(random::New64(), random::New64()); random::SimplePhilox rand(&gen); for (int64 i = 0; i < matrix.dimension(0); ++i) { GenerateData(&rand, {&matrix(i, 0), n}); diff --git a/tensorflow/contrib/coder/ops/coder_ops.cc b/tensorflow/contrib/coder/ops/coder_ops.cc index 9bb171298f..a185e07913 100644 --- a/tensorflow/contrib/coder/ops/coder_ops.cc +++ b/tensorflow/contrib/coder/ops/coder_ops.cc @@ -77,7 +77,7 @@ are incorrect. For this reason, the range coder uses integer arithmetics and avoids using any floating point operations internally, and `cdf` should contain integers representing quantized probability mass rather than floating points. -data: An int32 tensor. +data: An int16 tensor. cdf: An int32 tensor representing the CDF's of `data`. Each integer is divided by `2^precision` to represent a fraction. encoded: A range-coded scalar string. @@ -112,7 +112,7 @@ potential performance issues, the decoder does not return error status. encoded: A scalar string tensor from RangeEncode. shape: An int32 1-D tensor representing the shape of the data encoded by RangeEncode. -decoded: An int32 tensor with shape equal to `shape`. +decoded: An int16 tensor with shape equal to `shape`. precision: The number of bits for probability quantization. Must be <= 16, and must match the precision used by RangeEncode that produced `encoded`. )doc"); @@ -138,14 +138,12 @@ platforms. For entropy encoders and decoders to have the same quantized CDF on different platforms, the quantized CDF should be produced once and saved, then the saved quantized CDF should be used everywhere. -After quantization, if PMF sums to less than or equal to 2^precision, then this -is equivalent to cumsum over the last dimension. This op makes no effort to make -the sum close to 2^precision when the sum is already <= 2^precision. +After quantization, if PMF does not sum to 2^precision, then some values of PMF +are increased or decreased to adjust the sum to equal to 2^precision. -After quantization, if PMF sums to greater than 2^precision, then some values of -PMF is decreased to keep the sum no more than 2^precision. - -Note that the input PMF is pre-quantization. +Note that the input PMF is pre-quantization. The input PMF is not normalized +by this op prior to quantization. Therefore the user is responsible for +normalizing PMF if necessary. )doc"); // clang-format on } // namespace tensorflow -- GitLab From b3118b1f741896585d47184018f1d74d70e0e6c7 Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Thu, 19 Apr 2018 13:08:37 -0700 Subject: [PATCH 674/791] Update adam.py --- tensorflow/contrib/optimizer_v2/adam.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tensorflow/contrib/optimizer_v2/adam.py b/tensorflow/contrib/optimizer_v2/adam.py index 76a867039a..d538ad0fb0 100644 --- a/tensorflow/contrib/optimizer_v2/adam.py +++ b/tensorflow/contrib/optimizer_v2/adam.py @@ -40,19 +40,19 @@ class AdamOptimizer(optimizer_v2.OptimizerV2): Initialization: - $$m_0 \Leftarrow 0 (Initialize initial 1st moment vector)$$ - $$v_0 \Leftarrow 0 (Initialize initial 2nd moment vector)$$ - $$t \Leftarrow 0 (Initialize timestep)$$ + $$m_0 := 0 (Initialize initial 1st moment vector)$$ + $$v_0 := 0 (Initialize initial 2nd moment vector)$$ + $$t := 0 (Initialize timestep)$$ The update rule for `variable` with gradient `g` uses an optimization described at the end of section2 of the paper: - $$t \Leftarrow t + 1$$ - $$lr_t \Leftarrow \text{learning_rate} * \sqrt{(1 - beta_2^t) / (1 - beta_1^t)}$$ + $$t := t + 1$$ + $$lr_t := \text{learning_rate} * \sqrt{(1 - beta_2^t) / (1 - beta_1^t)}$$ - $$m_t \Leftarrow beta_1 * m_{t-1} + (1 - beta_1) * g$$ - $$v_t \Leftarrow beta_2 * v_{t-1} + (1 - beta_2) * g * g$$ - $$variable \Leftarrow variable - lr_t * m_t / (\sqrt{v_t} + \epsilon)$$ + $$m_t := beta_1 * m_{t-1} + (1 - beta_1) * g$$ + $$v_t := beta_2 * v_{t-1} + (1 - beta_2) * g * g$$ + $$variable := variable - lr_t * m_t / (\sqrt{v_t} + \epsilon)$$ The default value of 1e-8 for epsilon might not be a good default in general. For example, when training an Inception network on ImageNet a -- GitLab From 58f6760373b7a2d71053bd17b8017e57e5d1195d Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Thu, 19 Apr 2018 13:09:24 -0700 Subject: [PATCH 675/791] Update api_def_ApplyAdam.pbtxt --- tensorflow/core/api_def/base_api/api_def_ApplyAdam.pbtxt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/core/api_def/base_api/api_def_ApplyAdam.pbtxt b/tensorflow/core/api_def/base_api/api_def_ApplyAdam.pbtxt index fca8ba2530..b90f5473c8 100644 --- a/tensorflow/core/api_def/base_api/api_def_ApplyAdam.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_ApplyAdam.pbtxt @@ -82,9 +82,9 @@ END } summary: "Update \'*var\' according to the Adam algorithm." description: < Date: Thu, 19 Apr 2018 13:09:59 -0700 Subject: [PATCH 676/791] Update api_def_ResourceApplyAdam.pbtxt --- .../core/api_def/base_api/api_def_ResourceApplyAdam.pbtxt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/core/api_def/base_api/api_def_ResourceApplyAdam.pbtxt b/tensorflow/core/api_def/base_api/api_def_ResourceApplyAdam.pbtxt index 8b16d824bf..743247bb60 100644 --- a/tensorflow/core/api_def/base_api/api_def_ResourceApplyAdam.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_ResourceApplyAdam.pbtxt @@ -76,8 +76,8 @@ END } summary: "Update \'*var\' according to the Adam algorithm." description: < Date: Thu, 19 Apr 2018 13:11:04 -0700 Subject: [PATCH 677/791] Update adam.py --- tensorflow/python/training/adam.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tensorflow/python/training/adam.py b/tensorflow/python/training/adam.py index 9f523a3aca..6fa3ff6658 100644 --- a/tensorflow/python/training/adam.py +++ b/tensorflow/python/training/adam.py @@ -43,19 +43,19 @@ class AdamOptimizer(optimizer.Optimizer): Initialization: - $$m_0 \Leftarrow 0 (Initialize initial 1st moment vector)$$ - $$v_0 \Leftarrow 0 (Initialize initial 2nd moment vector)$$ - $$t \Leftarrow 0 (Initialize timestep)$$ + $$m_0 := 0 (Initialize initial 1st moment vector)$$ + $$v_0 := 0 (Initialize initial 2nd moment vector)$$ + $$t := 0 (Initialize timestep)$$ The update rule for `variable` with gradient `g` uses an optimization described at the end of section2 of the paper: - $$t \Leftarrow t + 1$$ - $$lr_t \Leftarrow \text{learning_rate} * \sqrt{(1 - beta_2^t) / (1 - beta_1^t)}$$ + $$t := t + 1$$ + $$lr_t := \text{learning_rate} * \sqrt{(1 - beta_2^t) / (1 - beta_1^t)}$$ - $$m_t \Leftarrow beta_1 * m_{t-1} + (1 - beta_1) * g$$ - $$v_t \Leftarrow beta_2 * v_{t-1} + (1 - beta_2) * g * g$$ - $$variable \Leftarrow variable - lr_t * m_t / (\sqrt{v_t} + \epsilon)$$ + $$m_t := beta_1 * m_{t-1} + (1 - beta_1) * g$$ + $$v_t := beta_2 * v_{t-1} + (1 - beta_2) * g * g$$ + $$variable := variable - lr_t * m_t / (\sqrt{v_t} + \epsilon)$$ The default value of 1e-8 for epsilon might not be a good default in general. For example, when training an Inception network on ImageNet a -- GitLab From b6686d2808b40ed985db2151bcf31961b53e49f5 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 13:09:07 -0700 Subject: [PATCH 678/791] Collective Ops Part 4 Add Broadcaster. A few minor adjustments to CollectiveParams and RMA. This change is part of a series of changes introducing infrastructure for collective ops and initial implementations of reduction and broadcast. PiperOrigin-RevId: 193562391 --- tensorflow/core/BUILD | 30 + .../base_collective_executor.cc | 81 +- .../common_runtime/base_collective_executor.h | 7 + tensorflow/core/common_runtime/broadcaster.cc | 249 ++++++ tensorflow/core/common_runtime/broadcaster.h | 66 ++ .../core/common_runtime/broadcaster_test.cc | 741 ++++++++++++++++++ .../collective_param_resolver_local.cc | 42 +- .../collective_param_resolver_local_test.cc | 8 +- .../common_runtime/collective_rma_local.h | 2 + tensorflow/core/framework/collective.cc | 15 +- tensorflow/core/framework/collective.h | 7 +- 11 files changed, 1220 insertions(+), 28 deletions(-) create mode 100644 tensorflow/core/common_runtime/broadcaster.cc create mode 100644 tensorflow/core/common_runtime/broadcaster.h create mode 100644 tensorflow/core/common_runtime/broadcaster_test.cc diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 54e7ab31d7..c15e7de186 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -2256,6 +2256,7 @@ CORE_CPU_LIB_HEADERS = CORE_CPU_BASE_HDRS + [ "common_runtime/allocator_retry.h", "common_runtime/base_collective_executor.h", "common_runtime/bfc_allocator.h", + "common_runtime/broadcaster.h", "common_runtime/buf_rendezvous.h", "common_runtime/build_graph_options.h", "common_runtime/collective_executor_mgr.h", @@ -2303,6 +2304,7 @@ tf_cuda_library( "common_runtime/allocator_retry.cc", "common_runtime/base_collective_executor.cc", "common_runtime/bfc_allocator.cc", + "common_runtime/broadcaster.cc", "common_runtime/buf_rendezvous.cc", "common_runtime/build_graph_options.cc", "common_runtime/collective_executor_mgr.cc", @@ -3140,6 +3142,34 @@ tf_cc_tests_gpu( ], ) +tf_cc_tests_gpu( + name = "broadcaster_test", + size = "small", + srcs = [ + "common_runtime/broadcaster_test.cc", + ], + linkstatic = tf_kernel_tests_linkstatic(), + tags = tf_cuda_tests_tags(), + deps = [ + ":all_kernels", + ":core", + ":core_cpu", + ":core_cpu_internal", + ":direct_session_internal", + ":framework", + ":framework_internal", + ":gpu_runtime", + ":lib", + ":lib_internal", + ":ops", + ":protos_all_cc", + ":protos_test_cc", + ":test", + ":test_main", + ":testlib", + ], +) + tf_cc_test_mkl( name = "mkl_runtime_tests", size = "small", diff --git a/tensorflow/core/common_runtime/base_collective_executor.cc b/tensorflow/core/common_runtime/base_collective_executor.cc index f6332fabdb..637b43c844 100644 --- a/tensorflow/core/common_runtime/base_collective_executor.cc +++ b/tensorflow/core/common_runtime/base_collective_executor.cc @@ -14,14 +14,13 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/common_runtime/base_collective_executor.h" -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" +#include "tensorflow/core/common_runtime/broadcaster.h" #include "tensorflow/core/common_runtime/copy_tensor.h" #include "tensorflow/core/common_runtime/device_mgr.h" #include "tensorflow/core/common_runtime/dma_helper.h" #include "tensorflow/core/common_runtime/process_util.h" #include "tensorflow/core/common_runtime/ring_reducer.h" #include "tensorflow/core/lib/core/notification.h" -#include "tensorflow/core/lib/strings/str_util.h" #define VALUE_IN_DEBUG_STRING false @@ -194,37 +193,68 @@ void BaseCollectiveExecutor::ExecuteAsync(OpKernelContext* ctx, const CollectiveParams& col_params, const string& exec_key, StatusCallback done) { - const Tensor* input = &ctx->input(0); + // On any individual collective Op failure we need to abort the + // BufRendezvous so that other Ops in the instance don't hang + // waiting for transmissions that will never happen. Do so after a + // delay so that the original error status is more likely to + // propagate up, and peers are unlikely to re-create the purged + // BufRendezvous by late-arriving requests. + StatusCallback done_safe = [this, done](const Status& s) { + if (!s.ok()) { + Ref(); // Ensure this lasts until the closure executes. + SchedNonBlockingClosureAfter(1000000, [this, s] { + remote_access_->buf_rendezvous()->StartAbort(s); + Unref(); + }); + } + done(s); + }; + Tensor* output = ctx->mutable_output(0); string error; switch (col_params.instance.type) { case REDUCTION_COLLECTIVE: { // TODO(tucker): support other reduction algorithms, // e.g. tree-reduce, hybrid tree/ring, delegate-to-NCCL, etc. + const Tensor* input = &ctx->input(0); RingReducer* reducer = CreateReducer(ctx, CtxParams(ctx), col_params, exec_key, step_id_, input, output, &error); if (!reducer) { - done(errors::Internal(error)); + done_safe(errors::Internal(error)); return; } // Run in an I/O thread, so as not to starve the executor threads. // TODO(tucker): Instead of forking every per-device Collective // Op off into its own thread, consider queuing them on a // fixed-size thread-pool dedicated to running CollectiveOps. - SchedClosure([reducer, done]() { - reducer->Run([reducer, done](const Status& s) { - done(s); + SchedClosure([reducer, done_safe]() { + reducer->Run([reducer, done_safe](const Status& s) { + done_safe(s); delete reducer; }); }); } break; - case BROADCAST_COLLECTIVE: - done(errors::Internal("Collective Broadcast unimplemented")); - break; + + case BROADCAST_COLLECTIVE: { + Broadcaster* broadcaster = CreateBroadcaster( + ctx, CtxParams(ctx), col_params, exec_key, step_id_, output, &error); + if (!broadcaster) { + done_safe(errors::Internal(error)); + return; + } + // Run in an I/O thread, so as not to starve the executor threads. + SchedClosure([broadcaster, done_safe]() { + broadcaster->Run([broadcaster, done_safe](const Status& s) { + done_safe(s); + delete broadcaster; + }); + }); + } break; + default: - done(errors::Internal("Unimplemented CollectiveType ", - col_params.instance.type)); + done_safe(errors::Internal("Unimplemented CollectiveType ", + col_params.instance.type)); } } @@ -254,4 +284,31 @@ RingReducer* BaseCollectiveExecutor::CreateReducer( } } +Broadcaster* BaseCollectiveExecutor::CreateBroadcaster( + OpKernelContext* ctx, OpKernelContext::Params* params, + const CollectiveParams& col_params, const string& exec_key, int64 step_id, + Tensor* output, string* error) { + switch (col_params.instance.data_type) { + case DT_INT32: + if (col_params.group.device_type == DEVICE_GPU) { + *error = + "Collective Broadcast does not support datatype DT_INT32 on " + "DEVICE_GPU"; + return nullptr; + } + TF_FALLTHROUGH_INTENDED; + case DT_FLOAT: + case DT_DOUBLE: + case DT_INT64: { + return new Broadcaster(this, dev_mgr_, ctx, params, col_params, exec_key, + step_id, output); + } break; + default: + *error = + strings::StrCat("Collective Broadcast does not support datatype ", + DataTypeString(col_params.instance.data_type)); + return nullptr; + } +} + } // namespace tensorflow diff --git a/tensorflow/core/common_runtime/base_collective_executor.h b/tensorflow/core/common_runtime/base_collective_executor.h index 58eaf31f71..462d6b7533 100644 --- a/tensorflow/core/common_runtime/base_collective_executor.h +++ b/tensorflow/core/common_runtime/base_collective_executor.h @@ -21,6 +21,7 @@ limitations under the License. #include "tensorflow/core/framework/device_attributes.pb.h" namespace tensorflow { +class Broadcaster; class DeviceMgr; class RingReducer; @@ -138,6 +139,12 @@ class BaseCollectiveExecutor : public CollectiveExecutor { const string& exec_key, int64 step_id, const Tensor* input, Tensor* output, string* error); + + Broadcaster* CreateBroadcaster(OpKernelContext* ctx, + OpKernelContext::Params* params, + const CollectiveParams& col_params, + const string& exec_key, int64 step_id, + Tensor* output, string* error); }; } // namespace tensorflow diff --git a/tensorflow/core/common_runtime/broadcaster.cc b/tensorflow/core/common_runtime/broadcaster.cc new file mode 100644 index 0000000000..5e8af8653d --- /dev/null +++ b/tensorflow/core/common_runtime/broadcaster.cc @@ -0,0 +1,249 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/core/common_runtime/broadcaster.h" + +#include "tensorflow/core/common_runtime/collective_rma_local.h" +#include "tensorflow/core/common_runtime/device_mgr.h" +#include "tensorflow/core/common_runtime/dma_helper.h" +#include "tensorflow/core/lib/core/notification.h" +#include "tensorflow/core/platform/env.h" + +// Set true for greater intelligibility of debug mode log messages. +#define READABLE_KEYS false + +namespace tensorflow { + +namespace { +// Key to be used for BufRendezvous by Broadcaster. +string BroadcastBufKey(const string& exec_key, int src_rank, int dst_rank) { + if (READABLE_KEYS) { + return strings::StrCat("broadcast(", exec_key, "):src(", src_rank, "):dst(", + dst_rank, ")"); + } else { + // TODO(tucker): Try a denser format, e.g. a 64 or 128 bit hash. + return strings::StrCat(exec_key, ":", src_rank, ":", dst_rank); + } +} +} // namespace + +Broadcaster::Broadcaster(CollectiveExecutor* col_exec, const DeviceMgr* dev_mgr, + OpKernelContext* ctx, OpKernelContext::Params* params, + const CollectiveParams& col_params, + const string& exec_key, int64 step_id, Tensor* output) + : col_exec_(col_exec), + dev_mgr_(dev_mgr), + ctx_(ctx), + col_params_(col_params), + exec_key_(exec_key), + rank_(col_params.subdiv_rank[0]), + is_source_(col_params.is_source), + output_(output), + done_(nullptr), + device_(nullptr) {} + +void Broadcaster::Run(StatusCallback done) { + // The optimal data transfer choreography is going to very platform dependent. + // That will be addressed by later improvements here or by platform-specific + // overrides of collective broadcast. The initial version is simply + // a binary tree that completely ignores DeviceLocality. + done_ = std::move(done); + + // Get the device for which we're executing and look up its locality. + status_ = dev_mgr_->LookupDevice( + col_params_.instance.device_names[col_params_.default_rank], &device_); + if (!status_.ok()) { + done_(status_); + return; + } + CHECK(device_); + device_locality_ = device_->attributes().locality(); + + RunTree(); +} + +// Binary tree parent/child relations are trivial to calculate, i.e. +// device at rank r is the parent of 2r+1 and 2r+2. The one exception +// is if the source is not rank 0. We treat that case as though the +// source is appended to the front of the rank ordering as well as +// continuing to occupy its current position. Hence we calculate as +// though each device's rank is actually r+1, then subtract 1 again to +// get the descendent ranks. If the source is not rank 0 then its +// decendents include both {0,1} and the descendents of its current +// position. Where a non-0-rank source is a descendent of another +// device, no send to it is necessary. + +/* static*/ +int Broadcaster::TreeRecvFrom(const CollectiveParams& cp) { + DCHECK_EQ(1, cp.subdiv_rank.size()); + if (cp.is_source) return -1; + int source_rank = cp.instance.impl_details.subdiv_source_rank[0]; + int my_rank = cp.subdiv_rank[0]; + if (source_rank == 0) { + return (my_rank - 1) / 2; + } else { + int predecessor_rank = (my_rank / 2) - 1; + return (predecessor_rank < 0) ? source_rank : predecessor_rank; + } +} + +/* static */ +void Broadcaster::TreeSendTo(const CollectiveParams& cp, + std::vector* targets) { + DCHECK_EQ(1, cp.subdiv_rank.size()); + targets->clear(); + int my_rank = cp.subdiv_rank[0]; + DCHECK_EQ(1, cp.instance.impl_details.subdiv_source_rank.size()); + int source_rank = cp.instance.impl_details.subdiv_source_rank[0]; + int successor_rank = 0; + if (source_rank == 0) { + successor_rank = (2 * my_rank) + 1; + } else { + successor_rank = (2 * (my_rank + 1)); + } + DCHECK_NE(successor_rank, my_rank); + if (cp.is_source && source_rank != 0) { + // The source sends to rank 0,1 in addition to its positional + // decendents. + if (cp.group.group_size > 1) { + targets->push_back(0); + } + if (cp.group.group_size > 2 && source_rank != 1) { + targets->push_back(1); + } + } + for (int i = 0; i < 2; ++i) { + if (successor_rank < cp.group.group_size && successor_rank != source_rank) { + targets->push_back(successor_rank); + } + ++successor_rank; + } +} + +// Execute a tree broadcast, i.e. each non-source device receives from +// one other and sends to up-to two others. +void Broadcaster::RunTree() { + mutex mu; + int pending_count = 0; // GUARDED_BY(mu) + condition_variable all_done; + std::vector send_to_ranks; + TreeSendTo(col_params_, &send_to_ranks); + + if (!is_source_) { + // Begin by receiving the value. + int recv_from_rank = TreeRecvFrom(col_params_); + Notification note; + DispatchRecv(recv_from_rank, output_, + [this, recv_from_rank, &mu, ¬e](const Status& s) { + mutex_lock l(mu); + status_.Update(s); + note.Notify(); + }); + note.WaitForNotification(); + } + + // Then forward value to all descendent devices. + if (status_.ok()) { + for (int i = 0; i < send_to_ranks.size(); ++i) { + int target_rank = send_to_ranks[i]; + { + mutex_lock l(mu); + ++pending_count; + } + DispatchSend( + target_rank, output_, + [this, target_rank, &mu, &pending_count, &all_done](const Status& s) { + status_.Update(s); + { + mutex_lock l(mu); + --pending_count; + if (pending_count == 0) { + all_done.notify_all(); + } + } + }); + } + } + + if (status_.ok() && is_source_) { + // Meanwhile, copy input to output if we weren't lucky enough to + // be able to reuse input as output. + const Tensor* input = &ctx_->input(0); + if (input != output_ && + (DMAHelper::base(input) != DMAHelper::base(output_))) { + { + mutex_lock l(mu); + ++pending_count; + } + DeviceContext* op_dev_ctx = ctx_->op_device_context(); + CollectiveRemoteAccessLocal::MemCpyAsync( + op_dev_ctx, op_dev_ctx, device_, device_, ctx_->input_alloc_attr(0), + ctx_->output_alloc_attr(0), input, output_, + [this, &mu, &pending_count, &all_done](const Status& s) { + status_.Update(s); + { + mutex_lock l(mu); + --pending_count; + if (0 == pending_count) { + all_done.notify_all(); + } + } + }); + } + } + + // Then wait for all pending actions to complete. + { + mutex_lock l(mu); + if (pending_count > 0) { + all_done.wait(l); + } + } + + VLOG(2) << "return status " << status_; + done_(status_); +} + +void Broadcaster::DispatchSend(int dst_rank, const Tensor* src_tensor, + const StatusCallback& done) { + string send_buf_key = BroadcastBufKey(exec_key_, rank_, dst_rank); + VLOG(1) << "DispatchSend " << send_buf_key << " from_device " + << device_->name(); + int dst_idx = + col_params_.instance.impl_details.subdiv_permutations[0][dst_rank]; + col_exec_->PostToPeer(col_params_.instance.device_names[dst_idx], + col_params_.instance.task_names[dst_idx], send_buf_key, + device_, ctx_->op_device_context(), + ctx_->output_alloc_attr(0), src_tensor, + device_locality_, done); +} + +void Broadcaster::DispatchRecv(int src_rank, Tensor* dst_tensor, + const StatusCallback& done) { + string recv_buf_key = BroadcastBufKey(exec_key_, src_rank, rank_); + int src_idx = + col_params_.instance.impl_details.subdiv_permutations[0][src_rank]; + VLOG(1) << "DispatchRecv " << recv_buf_key << " from_device " + << col_params_.instance.device_names[src_idx]; + int dst_idx = col_params_.instance.impl_details.subdiv_permutations[0][rank_]; + CHECK_EQ(col_params_.instance.device_names[dst_idx], device_->name()); + col_exec_->RecvFromPeer(col_params_.instance.device_names[src_idx], + col_params_.instance.task_names[src_idx], + col_params_.task.is_local[src_idx], recv_buf_key, + device_, ctx_->op_device_context(), + ctx_->output_alloc_attr(0), dst_tensor, + device_locality_, done); +} + +} // namespace tensorflow diff --git a/tensorflow/core/common_runtime/broadcaster.h b/tensorflow/core/common_runtime/broadcaster.h new file mode 100644 index 0000000000..bdf68f19ab --- /dev/null +++ b/tensorflow/core/common_runtime/broadcaster.h @@ -0,0 +1,66 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_BROADCASTER_H_ +#define TENSORFLOW_CORE_COMMON_RUNTIME_BROADCASTER_H_ + +#include +#include "tensorflow/core/common_runtime/base_collective_executor.h" +#include "tensorflow/core/framework/collective.h" +#include "tensorflow/core/framework/device_attributes.pb.h" + +namespace tensorflow { + +// Tree-algorithm implementation of collective broadcast. +class Broadcaster { + public: + Broadcaster(CollectiveExecutor* col_exec, const DeviceMgr* dev_mgr, + OpKernelContext* ctx, OpKernelContext::Params* params, + const CollectiveParams& col_params, const string& exec_key, + int64 step_id, Tensor* output); + + void Run(StatusCallback done); + + // Returns the rank of the device from which this device should receive + // its value, -1 if no value should be received. + static int TreeRecvFrom(const CollectiveParams& cp); + + // Populates targets with the ranks of the devices to which this device + // should forward the value. + static void TreeSendTo(const CollectiveParams& cp, std::vector* targets); + + private: + void DispatchSend(int dst_rank, const Tensor* src_tensor, + const StatusCallback& done); + void DispatchRecv(int src_rank, Tensor* dst_tensor, + const StatusCallback& done); + void RunTree(); + + Status status_; + CollectiveExecutor* col_exec_; // Not owned + const DeviceMgr* dev_mgr_; // Not owned + OpKernelContext* ctx_; // Not owned + const CollectiveParams& col_params_; + const string exec_key_; + const int rank_; + const bool is_source_; + Tensor* output_; // Not owned + std::unique_ptr ca_; + StatusCallback done_; + Device* device_; // The device for which this instance labors + DeviceLocality device_locality_; +}; + +} // namespace tensorflow +#endif // TENSORFLOW_CORE_COMMON_RUNTIME_BROADCASTER_H_ diff --git a/tensorflow/core/common_runtime/broadcaster_test.cc b/tensorflow/core/common_runtime/broadcaster_test.cc new file mode 100644 index 0000000000..89d39144b3 --- /dev/null +++ b/tensorflow/core/common_runtime/broadcaster_test.cc @@ -0,0 +1,741 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/core/common_runtime/broadcaster.h" + +#include +#include "tensorflow/core/common_runtime/base_collective_executor.h" +#include "tensorflow/core/common_runtime/collective_rma_local.h" +#include "tensorflow/core/common_runtime/device_mgr.h" +#include "tensorflow/core/common_runtime/device_resolver_local.h" +#include "tensorflow/core/common_runtime/dma_helper.h" +#include "tensorflow/core/common_runtime/process_util.h" +#include "tensorflow/core/common_runtime/test_collective_executor_mgr.h" +#include "tensorflow/core/common_runtime/threadpool_device.h" +#include "tensorflow/core/framework/collective.h" +#include "tensorflow/core/framework/fake_input.h" +#include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/framework/node_def_builder.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/lib/core/notification.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/public/session_options.h" +#include "tensorflow/core/public/version.h" + +namespace tensorflow { +namespace { + +static int64 kStepId = 123; +static int32 kNumSubdivs = 1; // Subdiv not yet meaningful for broadcast + +// The test harness won't allow a mixture of fixture and non-fixture +// tests in one file, so this is a trival fixture for tests that don't +// need the heavy-weight BroadcasterTest fixture. +class TrivialTest : public ::testing::Test { + protected: + TrivialTest() {} +}; + +// Tests of static TreeSendTo() and TreeRecvFrom() functions. +// D = number of devices +// S = source rank +// R = tested rank +// RF = receive-from rank +// ST = send_to rank vector +#define DEF_TL_TEST(D, S, R, RF, ST) \ + TEST_F(TrivialTest, TreeLinks_##D##Devs_##S##Source_##R##Rank) { \ + CollectiveParams cp; \ + cp.group.group_size = D; \ + cp.instance.impl_details.subdiv_source_rank = {S}; \ + cp.subdiv_rank = {R}; \ + cp.is_source = (S == R); \ + EXPECT_EQ(RF, Broadcaster::TreeRecvFrom(cp)); \ + std::vector expected = ST; \ + std::vector send_to; \ + Broadcaster::TreeSendTo(cp, &send_to); \ + ASSERT_EQ(expected.size(), send_to.size()); \ + for (int i = 0; i < expected.size(); ++i) { \ + EXPECT_EQ(expected[i], send_to[i]); \ + } \ + } + +#define V(...) std::vector({__VA_ARGS__}) + +// D S R RF ST +// 2 device cases +DEF_TL_TEST(2, 0, 0, -1, V(1)) +DEF_TL_TEST(2, 1, 0, 1, V()) +DEF_TL_TEST(2, 0, 1, 0, V()) +DEF_TL_TEST(2, 1, 1, -1, V(0)) +// 3 device cases +DEF_TL_TEST(3, 0, 0, -1, V(1, 2)) +DEF_TL_TEST(3, 0, 1, 0, V()) +DEF_TL_TEST(3, 0, 2, 0, V()) +DEF_TL_TEST(3, 1, 0, 1, V(2)) +DEF_TL_TEST(3, 1, 1, -1, V(0)) +DEF_TL_TEST(3, 1, 2, 0, V()) +DEF_TL_TEST(3, 2, 0, 2, V()) +DEF_TL_TEST(3, 2, 1, 2, V()) +DEF_TL_TEST(3, 2, 2, -1, V(0, 1)) +// 4 device cases +DEF_TL_TEST(4, 0, 0, -1, V(1, 2)) +DEF_TL_TEST(4, 0, 1, 0, V(3)) +DEF_TL_TEST(4, 0, 2, 0, V()) +DEF_TL_TEST(4, 0, 3, 1, V()) +DEF_TL_TEST(4, 1, 0, 1, V(2, 3)) +DEF_TL_TEST(4, 1, 1, -1, V(0)) +DEF_TL_TEST(4, 1, 2, 0, V()) +DEF_TL_TEST(4, 1, 3, 0, V()) +DEF_TL_TEST(4, 2, 0, 2, V(3)) +DEF_TL_TEST(4, 2, 1, 2, V()) +DEF_TL_TEST(4, 2, 2, -1, V(0, 1)) +DEF_TL_TEST(4, 2, 3, 0, V()) +DEF_TL_TEST(4, 3, 0, 3, V(2)) +DEF_TL_TEST(4, 3, 1, 3, V()) +DEF_TL_TEST(4, 3, 2, 0, V()) +DEF_TL_TEST(4, 3, 3, -1, V(0, 1)) +// 8 device cases +// D S R RF ST +DEF_TL_TEST(8, 0, 0, -1, V(1, 2)) +DEF_TL_TEST(8, 0, 1, 0, V(3, 4)) +DEF_TL_TEST(8, 0, 2, 0, V(5, 6)) +DEF_TL_TEST(8, 0, 3, 1, V(7)) +DEF_TL_TEST(8, 0, 4, 1, V()) +DEF_TL_TEST(8, 0, 5, 2, V()) +DEF_TL_TEST(8, 0, 6, 2, V()) +DEF_TL_TEST(8, 0, 7, 3, V()) +DEF_TL_TEST(8, 7, 0, 7, V(2, 3)) +DEF_TL_TEST(8, 7, 1, 7, V(4, 5)) +DEF_TL_TEST(8, 7, 2, 0, V(6)) +DEF_TL_TEST(8, 7, 3, 0, V()) +DEF_TL_TEST(8, 7, 4, 1, V()) +DEF_TL_TEST(8, 7, 5, 1, V()) +DEF_TL_TEST(8, 7, 6, 2, V()) +DEF_TL_TEST(8, 7, 7, -1, V(0, 1)) +#undef DEF_TL_TEST +#undef V + +// Wraps CollectiveRemoteAccessLocal with the ability to return an +// error status to the N'th action. +// TODO(tucker): factor out of this file and ring_reducer_test.cc +// into a single common source. +class FailTestRMA : public CollectiveRemoteAccessLocal { + public: + FailTestRMA(const DeviceMgr* dev_mgr, DeviceResolverInterface* dev_resolver, + int64 step_id, int fail_after) + : CollectiveRemoteAccessLocal(dev_mgr, dev_resolver, step_id), + fail_after_(fail_after) {} + + bool MaybeFail(const StatusCallback& done) { + bool fail_now = false; + { + mutex_lock l(mu_); + if (fail_after_ > 0) { + fail_now = (--fail_after_ == 0); + } + } + if (fail_now) { + auto error = errors::Internal("Deliberate failure"); + LOG(INFO) << "triggering failure " << error; + SchedNonBlockingClosureAfter( + 1000, [this, error] { buf_rendezvous()->StartAbort(error); }); + done(error); + return true; + } + return false; + } + + void RecvFromPeer(const string& peer_device, const string& peer_task, + bool peer_is_local, const string& key, Device* to_device, + DeviceContext* to_device_ctx, + const AllocatorAttributes& to_alloc_attr, Tensor* to_tensor, + const DeviceLocality& client_locality, + const StatusCallback& done) override { + if (MaybeFail(done)) return; + CollectiveRemoteAccessLocal::RecvFromPeer( + peer_device, peer_task, peer_is_local, key, to_device, to_device_ctx, + to_alloc_attr, to_tensor, client_locality, done); + } + + void PostToPeer(const string& peer_device, const string& peer_task, + const string& key, Device* from_device, + DeviceContext* from_device_ctx, + const AllocatorAttributes& from_alloc_attr, + const Tensor* from_tensor, + const DeviceLocality& client_locality, + const StatusCallback& done) override { + if (MaybeFail(done)) return; + CollectiveRemoteAccessLocal::PostToPeer( + peer_device, peer_task, key, from_device, from_device_ctx, + from_alloc_attr, from_tensor, client_locality, done); + } + + mutex mu_; + int fail_after_ GUARDED_BY(mu_); +}; + +class BroadcasterTest : public ::testing::Test { + protected: + BroadcasterTest() : device_type_(DEVICE_CPU) {} + + ~BroadcasterTest() override { + stop_ = true; + for (auto i : instances_) { + delete i; + } + if (col_exec_) col_exec_->Unref(); + } + + void SetUp() override { +#if GOOGLE_CUDA + auto device_factory = DeviceFactory::GetFactory("GPU"); + CHECK(device_factory); + SessionOptions options; + Status s = device_factory->CreateDevices( + options, "/job:worker/replica:0/task:0", &gpu_devices_); + CHECK(s.ok()); +#endif + } + + void Init(int num_workers, int num_devices, DataType dtype, + const DeviceType& device_type, int fail_after) { + device_type_ = device_type; + std::vector local_devices; + SessionOptions sess_opts; + sess_opts.env = Env::Default(); + Bytes mem_limit(4 << 20); + DeviceLocality dev_locality; + for (int wi = 0; wi < num_workers; ++wi) { + for (int di = 0; di < num_devices; ++di) { + if (device_type == DEVICE_CPU) { + string dev_name = strings::StrCat("/job:worker/replica:0/task:", wi, + "/device:CPU:", di); + local_devices.push_back(new ThreadPoolDevice( + sess_opts, dev_name, mem_limit, dev_locality, cpu_allocator())); + } else if (device_type == DEVICE_GPU && !gpu_devices_.empty()) { + int dev_idx = (wi * num_devices) + di; + if (dev_idx >= static_cast(gpu_devices_.size())) { + LOG(INFO) << "dev_mgr has access to limited GPUs, reusing for more " + "than one ring node."; + } else { + local_devices.push_back(gpu_devices_[dev_idx]); + } + } else { + LOG(FATAL) << "Unsupported device_type " << device_type; + } + } + } + if (!dev_mgr_ || device_type == DEVICE_CPU) { + dev_mgr_.reset(new DeviceMgr(local_devices)); + } + dev_resolver_.reset(new DeviceResolverLocal(dev_mgr_.get())); + rma_ = new FailTestRMA(dev_mgr_.get(), dev_resolver_.get(), kStepId, + fail_after); + col_exec_ = new BaseCollectiveExecutor(&col_exec_mgr_, rma_, kStepId, + dev_mgr_.get()); + col_params_.name = "test_collective"; + col_params_.instance.data_type = dtype; + static const int kGroupKey = 5; + col_params_.group.group_key = kGroupKey; + static const int kInstanceKey = 17; + col_params_.instance.instance_key = kInstanceKey; + col_params_.group.device_type = device_type; + col_params_.group.group_size = num_workers * num_devices; + col_params_.instance.impl_details.subdiv_offsets.clear(); + col_params_.instance.type = BROADCAST_COLLECTIVE; + col_params_.instance.impl_details.subdiv_permutations.resize(kNumSubdivs); + col_params_.subdiv_rank.resize(kNumSubdivs); + int subdiv_stride = num_devices / kNumSubdivs; + for (int sdi = 0; sdi < kNumSubdivs; ++sdi) { + col_params_.instance.impl_details.subdiv_offsets.push_back(sdi * + subdiv_stride); + col_params_.subdiv_rank[sdi] = sdi * subdiv_stride; + } + + // Set up a local device ring order that's not just 0,1,2... + std::vector local_ring_order; + for (int di = 0; di < num_devices; ++di) { + local_ring_order.push_back(di); + } + for (int di = 0; di < num_devices; ++di) { + bool is_odd = ((di % 2) == 1); + int other = (di + (is_odd ? 7 : 3)) % num_devices; + if (di == other) continue; + iter_swap(local_ring_order.begin() + di, + local_ring_order.begin() + other); + } + broadcast_dev_id_ = local_ring_order[0]; + string lro_buf; + for (auto d : local_ring_order) strings::StrAppend(&lro_buf, d, ", "); + VLOG(1) << "local_ring_order " << lro_buf; + + // Set up all of the fake device contexts. + for (int wi = 0; wi < num_workers; ++wi) { + for (int di = 0; di < num_devices; ++di) { + string task_name = strings::StrCat("/job:worker/replica:0/task:", wi); + string dev_name = strings::StrCat(task_name, "/device:CPU:", di); + if (device_type == DEVICE_GPU) { + dev_name = strings::StrCat(task_name, "/device:GPU:0"); + } + col_params_.instance.device_names.push_back(dev_name); + col_params_.instance.task_names.push_back(task_name); + // Normally each device would set is_local to its own perspective but + // this test runs in a single process so is_local is always true. + col_params_.task.is_local.push_back(true); + for (int sdi = 0; sdi < kNumSubdivs; ++sdi) { + int rotated_di = + (di + col_params_.instance.impl_details.subdiv_offsets[sdi]) % + num_devices; + col_params_.instance.impl_details.subdiv_permutations[sdi].push_back( + wi * num_devices + local_ring_order[rotated_di]); + } + } + } + for (int wi = 0; wi < num_workers; ++wi) { + for (int di = 0; di < num_devices; ++di) { + int rank = wi * num_devices + di; + instances_.push_back(new DeviceInstance( + rank, col_params_.instance.device_names[rank], device_type_, this)); + } + } + } + + typedef std::function InitFunc; + + void Broadcast() { + std::atomic done(0); + for (auto di : instances_) { + SchedClosure([di, &done] { + di->DoBroadcast(); + ++done; + }); + } + while (done < instances_.size()) { + if (stop_) break; + Env::Default()->SleepForMicroseconds(1000); + } + } + + std::unique_ptr GetKernel(const NodeDef& node, + const DeviceType& device_type, + DeviceBase* device) { + Status status; + std::unique_ptr k = CreateOpKernel( + device_type, device, device->GetAllocator(AllocatorAttributes()), node, + TF_GRAPH_DEF_VERSION, &status); + if (!status.ok()) { + LOG(FATAL) << status; + } + return k; + } + + std::unique_ptr GetCollectiveBcastSend( + const CollectiveParams& params, Tensor* input, + const DeviceType& device_type, DeviceBase* device) { + mutex_lock l(mu_); + NodeDef node_def; + NodeDefBuilder builder( + strings::StrCat("collective_bcast_send_", bcast_send_counter_++), + "CollectiveBcastSend"); + TF_CHECK_OK(builder.Attr("T", input->dtype()) + .Attr("group_size", params.group.group_size) + .Attr("group_key", params.group.group_key) + .Attr("instance_key", params.instance.instance_key) + .Attr("shape", input->shape()) + .Input(FakeInput(params.instance.data_type)) + .Finalize(&node_def)); + return GetKernel(node_def, device_type, device); + } + + std::unique_ptr GetCollectiveBcastRecv( + const CollectiveParams& params, const TensorShape& shape, + const DeviceType& device_type, DeviceBase* device) { + mutex_lock l(mu_); + NodeDef node_def; + NodeDefBuilder builder( + strings::StrCat("collective_bcast_recv_", bcast_recv_counter_++), + "CollectiveBcastRecv"); + TF_CHECK_OK(builder.Attr("T", params.instance.data_type) + .Attr("group_size", params.group.group_size) + .Attr("group_key", params.group.group_key) + .Attr("instance_key", params.instance.instance_key) + .Attr("shape", shape) + .Finalize(&node_def)); + return GetKernel(node_def, device_type, device); + } + + void BuildColParams() {} + + template + void RunTest(DataType dtype, const DeviceType& device_type, int num_workers, + int num_devices, int tensor_len, int fail_after) { + Init(num_workers, num_devices, dtype, device_type, fail_after); + + // Initialize each instance tensor with distinct values. + for (int di = 0; di < instances_.size(); ++di) { + DeviceInstance* instance = instances_[di]; + instance->InitTensor( + dtype, TensorShape({tensor_len}), [di, dtype](Tensor* t) { + for (size_t i = 0; i < t->NumElements(); ++i) { + // The cast is necessary to prevent clang-tidy from insisting + // that a faster non-open source function be substituted. + float value = pow(10, static_cast(di)) * i; + t->flat()(i) = value; + } + }); + } + + // Copy the expected value from the broadcast source tensor + std::vector expected(tensor_len, 0.0); + const CollectiveParams& cp = instances_[0]->col_params_; + int broadcast_dev_id = + cp.instance.impl_details.subdiv_permutations + [0][cp.instance.impl_details.subdiv_source_rank[0]]; + const Tensor* t = &instances_[broadcast_dev_id]->tensor_; + Tensor cpu_copy(dtype, TensorShape({tensor_len})); + if (device_type == DEVICE_GPU) { + Notification notification; + Device* dev = instances_[broadcast_dev_id]->device_; + auto* dev_info = dev->tensorflow_gpu_device_info(); + CHECK(dev_info); + dev_info->default_context->CopyDeviceTensorToCPU( + t, "" /*tensor_name*/, dev, &cpu_copy, + [this, ¬ification](Status s) { + TF_CHECK_OK(s); + notification.Notify(); + }); + notification.WaitForNotification(); + t = &cpu_copy; + } + for (size_t i = 0; i < t->NumElements(); ++i) { + expected[i] = t->flat()(i); + } + + Broadcast(); + + // At this point all of the ops have terminated. + for (int di = 0; di < instances_.size(); ++di) { + if (!instances_[di]->status_.ok()) { + ASSERT_GT(fail_after, 0); + ASSERT_EQ(instances_[di]->status_.error_message(), + "Deliberate failure"); + mutex_lock l(mu_); + ++failure_count_; + continue; + } + Tensor* inst = &instances_[di]->tensor_; + Tensor actual(dtype, TensorShape({tensor_len})); + if (device_type_ == DEVICE_CPU) { + CHECK(actual.CopyFrom(*inst, inst->shape())); + } else if (device_type_ == DEVICE_GPU) { + Notification notification; + Device* dev = instances_[di]->device_; + auto* dev_info = dev->tensorflow_gpu_device_info(); + CHECK(dev_info); + dev_info->default_context->CopyDeviceTensorToCPU( + inst, "" /*tensor_name*/, dev, &actual, + [this, ¬ification](Status s) { + TF_CHECK_OK(s); + notification.Notify(); + }); + notification.WaitForNotification(); + } + for (int i = 0; i < tensor_len; ++i) { + switch (dtype) { + case DT_FLOAT: + EXPECT_FLOAT_EQ(expected[i], actual.template flat()(i)) + << "Mismatch at device " << di << " index " << i; + break; + case DT_DOUBLE: + EXPECT_DOUBLE_EQ(expected[i], actual.template flat()(i)) + << "Mismatch at device " << di << " index " << i; + break; + case DT_INT32: + case DT_INT64: + EXPECT_EQ(expected[i], actual.template flat()(i)) + << "Mismatch at device " << di << " index " << i; + break; + default: + LOG(FATAL) << "unimplemented"; + } + } + } + + // Note that the order of operations during broadcast is + // non-deterministic and unlike the reduce case some Ops in the + // instance may succeed while others fail, even if a transmission + // failure occurs early in the operation chain. So, when an abort + // is specified we need to verify that at least one Op fails with + // the expected status and any Op that succeeds yeilds the correct + // value. + if (fail_after > 0) { + mutex_lock l(mu_); + EXPECT_GT(failure_count_, 0); + } + } + + class DeviceInstance { + public: + DeviceInstance(int rank, const string& dev_name, + const DeviceType& device_type, BroadcasterTest* parent) + : parent_(parent), + dev_name_(dev_name), + device_type_(device_type), + rank_(rank) { + TF_CHECK_OK(parent_->dev_mgr_->LookupDevice(dev_name, &device_)); + col_params_.name = parent_->col_params_.name; + col_params_.instance.data_type = parent_->col_params_.instance.data_type; + col_params_.group.group_key = parent_->col_params_.group.group_key; + col_params_.instance.instance_key = + parent_->col_params_.instance.instance_key; + col_params_.group.device_type = parent_->col_params_.group.device_type; + col_params_.group.group_size = parent_->col_params_.group.group_size; + col_params_.instance.device_names = + parent_->col_params_.instance.device_names; + col_params_.instance.task_names = + parent_->col_params_.instance.task_names; + col_params_.task.is_local = parent_->col_params_.task.is_local; + col_params_.instance.impl_details.subdiv_permutations = + parent_->col_params_.instance.impl_details.subdiv_permutations; + col_params_.subdiv_rank = parent_->col_params_.subdiv_rank; + + int group_size = col_params_.group.group_size; + CHECK_EQ(group_size, col_params_.instance.device_names.size()); + // Default rank is order in device_names. + col_params_.default_rank = rank; + // perm_rank is order in subdiv[0]: + int perm_rank = -1; + for (int i = 0; + i < col_params_.instance.impl_details.subdiv_permutations[0].size(); + ++i) { + if (rank == + col_params_.instance.impl_details.subdiv_permutations[0][i]) { + perm_rank = i; + break; + } + } + CHECK_GE(perm_rank, 0); + col_params_.instance.impl_details.subdiv_source_rank.resize(1, 0); + col_params_.is_source = + (perm_rank == + col_params_.instance.impl_details.subdiv_source_rank[0]); + // Set rank in all subdivs by finding that default_rank. + for (int sdi = 0; sdi < kNumSubdivs; ++sdi) { + for (int r = 0; + r < + col_params_.instance.impl_details.subdiv_permutations[sdi].size(); + ++r) { + if (col_params_.default_rank == + col_params_.instance.impl_details.subdiv_permutations[sdi][r]) { + col_params_.subdiv_rank[sdi] = r; + CHECK_EQ(0, sdi); + CHECK_EQ(perm_rank, col_params_.subdiv_rank[sdi]); + break; + } + } + } + CHECK_EQ(group_size, col_params_.task.is_local.size()); + CHECK_EQ(group_size, col_params_.instance.task_names.size()); + } + + void InitTensor(DataType dtype, const TensorShape& shape, + const InitFunc& f) { + tensor_ = + Tensor(device_->GetAllocator(AllocatorAttributes()), dtype, shape); + if (device_type_ == DEVICE_CPU) { + f(&tensor_); + } else if (device_type_ == DEVICE_GPU) { + Tensor cpu_tensor(dtype, shape); + f(&cpu_tensor); + Notification notification; + auto* dev_info = device_->tensorflow_gpu_device_info(); + CHECK(dev_info); + dev_info->default_context->CopyCPUTensorToDevice( + &cpu_tensor, device_, &tensor_, [this, ¬ification](Status s) { + TF_CHECK_OK(s); + notification.Notify(); + }); + notification.WaitForNotification(); + } else { + LOG(FATAL) << "Unsupported device_type " << device_type_; + } + } + + void DoBroadcast() { + // Prepare an OpKernelContext. + OpKernelContext::Params op_params; + op_params.step_id = parent_->step_id_; + op_params.device = device_; + gtl::InlinedVector inputs; + inputs.push_back(TensorValue(&tensor_)); + op_params.inputs = &inputs; + gtl::InlinedVector input_aa( + {AllocatorAttributes()}); + op_params.input_alloc_attrs = &input_aa; + gtl::InlinedVector input_dc; + DeviceContext* dev_ctx = nullptr; + auto* dev_info = device_->tensorflow_gpu_device_info(); + if (dev_info) { + dev_ctx = dev_info->default_context; + dev_ctx->Ref(); + } else { + dev_ctx = new DeviceContext; + } + input_dc.push_back(dev_ctx); + op_params.input_device_contexts = &input_dc; + op_params.op_device_context = dev_ctx; + int forward_from[] = {0}; + if (col_params_.is_source) { + op_params.forward_from_array = &forward_from[0]; + } + AllocatorAttributes generic_alloc_attr; + op_params.output_attr_array = &generic_alloc_attr; + std::unique_ptr op = + col_params_.is_source + ? parent_->GetCollectiveBcastSend(col_params_, &tensor_, + DEVICE_CPU, device_) + : parent_->GetCollectiveBcastRecv(col_params_, tensor_.shape(), + DEVICE_CPU, device_); + op_params.op_kernel = op.get(); + OpKernelContext ctx(&op_params, 1); + + Tensor* output_tensor_ptr = nullptr; + if (col_params_.is_source) { + TF_CHECK_OK(ctx.forward_input_or_allocate_output( + {0}, 0, tensor_.shape(), &output_tensor_ptr)); + } else { + TF_CHECK_OK( + ctx.allocate_output(0, tensor_.shape(), &output_tensor_ptr)); + } + CHECK_EQ(output_tensor_ptr, ctx.mutable_output(0)); + + // Prepare a Broadcaster instance. + string exec_key = + strings::StrCat(col_params_.instance.instance_key, ":0:0"); + Broadcaster broadcaster(parent_->col_exec_, parent_->dev_mgr_.get(), &ctx, + &op_params, col_params_, exec_key, kStepId, + output_tensor_ptr); + + // Start execution in a threadpool then wait for completion. + Notification notification; + broadcaster.Run([this, ¬ification](Status s) { + status_ = s; + notification.Notify(); + }); + notification.WaitForNotification(); + if (status_.ok()) { + CHECK(tensor_.CopyFrom(*ctx.mutable_output(0), tensor_.shape())); + } + + dev_ctx->Unref(); + } + + BroadcasterTest* parent_; + string dev_name_; + DeviceType device_type_ = DEVICE_CPU; + int rank_; + Tensor tensor_; + Device* device_; + CollectiveParams col_params_; + std::unique_ptr ca_; + std::unique_ptr ctx_; + Status status_; + }; // class DeviceInstance + + bool stop_ = false; + int64 step_id_ = kStepId; + int broadcast_dev_id_ = 0; + DeviceType device_type_; + TestCollectiveExecutorMgr col_exec_mgr_; + CollectiveExecutor* col_exec_ = nullptr; + CollectiveRemoteAccessLocal* rma_; + std::unique_ptr dev_resolver_; + std::vector instances_; + CollectiveParams col_params_; + std::vector gpu_devices_; + std::unique_ptr dev_mgr_; + mutex mu_; + int bcast_recv_counter_ GUARDED_BY(mu_) = 0; + int bcast_send_counter_ GUARDED_BY(mu_) = 0; + int failure_count_ GUARDED_BY(mu_) = 0; +}; + +// Tests of full broadcast algorithm, with different device and +// data types. +// B = data element type +// T = device type +// W = number of workers +// D = number of devices per worker +// L = tensor length +// A = abort after count +#define DEF_TEST(B, T, W, D, L, A) \ + TEST_F(BroadcasterTest, \ + DaTy##B##_DevTy##T##_Wkr##W##_Dev##D##_Len##L##_Abt##A) { \ + DataType dtype = DT_##B; \ + switch (dtype) { \ + case DT_FLOAT: { \ + RunTest(dtype, DEVICE_##T, W, D, L, A); \ + } break; \ + case DT_DOUBLE: { \ + RunTest(dtype, DEVICE_##T, W, D, L, A); \ + } break; \ + case DT_INT32: { \ + RunTest(dtype, DEVICE_##T, W, D, L, A); \ + } break; \ + case DT_INT64: { \ + RunTest(dtype, DEVICE_##T, W, D, L, A); \ + } break; \ + default: \ + LOG(FATAL) << "Unimplemented"; \ + } \ + } + +#ifndef GOOGLE_CUDA +// B T W D L A +DEF_TEST(FLOAT, CPU, 1, 2, 1, 0) +DEF_TEST(FLOAT, CPU, 1, 2, 1001, 0) +DEF_TEST(FLOAT, CPU, 2, 1, 128, 0) +DEF_TEST(FLOAT, CPU, 2, 4, 128, 0) +DEF_TEST(FLOAT, CPU, 2, 8, 4095, 0) +DEF_TEST(FLOAT, CPU, 4, 4, 1045991, 0) + +DEF_TEST(DOUBLE, CPU, 2, 4, 128, 0) +DEF_TEST(INT32, CPU, 2, 4, 128, 0) +DEF_TEST(INT64, CPU, 2, 4, 128, 0) + +// Failure cases +DEF_TEST(FLOAT, CPU, 2, 4, 128, 1) +DEF_TEST(FLOAT, CPU, 2, 4, 128, 5) +#endif + +#ifdef GOOGLE_CUDA +// Can only set W=1 for GPU tests. +// B T W D L A +DEF_TEST(FLOAT, GPU, 1, 2, 1, 0) +DEF_TEST(FLOAT, GPU, 1, 2, 33, 0) +DEF_TEST(FLOAT, GPU, 1, 3, 64, 0) +DEF_TEST(FLOAT, GPU, 1, 8, 1001, 0) +DEF_TEST(FLOAT, GPU, 1, 8, 4095, 0) +DEF_TEST(FLOAT, GPU, 1, 8, 1045991, 0) + +DEF_TEST(DOUBLE, GPU, 1, 8, 1001, 0) +DEF_TEST(INT64, GPU, 1, 8, 1001, 0) + +// Failure cases +DEF_TEST(FLOAT, GPU, 1, 8, 128, 6) +#endif + +} // namespace +} // namespace tensorflow diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local.cc b/tensorflow/core/common_runtime/collective_param_resolver_local.cc index 393d3f824d..bdddf927d8 100644 --- a/tensorflow/core/common_runtime/collective_param_resolver_local.cc +++ b/tensorflow/core/common_runtime/collective_param_resolver_local.cc @@ -250,6 +250,38 @@ GlobalDeviceMap EstablishGlobalRank( return gdm; } +// Count the devices associated with each task and set +// cp->same_num_devices_per_task. Requires cp->instance.task_names +// be sorted. +void SetDevPerTask(CollectiveParams* cp) { + cp->instance.same_num_devices_per_task = false; + if (cp->instance.task_names.empty()) return; + int dev_per_task = -1; + int count = 0; + const string* last_task_name = &cp->instance.task_names[0]; + for (const string& task_name : cp->instance.task_names) { + if (task_name != *last_task_name) { + CHECK_GT(count, 0); + if (dev_per_task < 0) { + dev_per_task = count; + } else { + CHECK_GT(dev_per_task, 0); + if (count != dev_per_task) return; + } + count = 1; + last_task_name = &task_name; + } else { + ++count; + } + } + CHECK_GT(count, 0); + if ((dev_per_task > 0) && (count != dev_per_task)) { + return; + } + cp->instance.same_num_devices_per_task = true; + CHECK_EQ((cp->group.group_size % cp->group.num_tasks), 0); +} + // Sort cp->instance.device_names lexicographically, but do by first // computing a reordering permutation so we can keep cp->instance.task_names // in corresponding order. @@ -278,6 +310,7 @@ void SortDevicesAndTasks(CollectiveParams* cp) { cp->instance.device_names = std::move(new_devs); cp->instance.task_names = std::move(new_tasks); VLOG(1) << "Modified device_names on " << cp; + SetDevPerTask(cp); } // Establish the requested number of subdivision permutations based on the @@ -343,17 +376,18 @@ void GenerateSubdivPerms(const string& device, int source_rank, if (cp->instance.type == BROADCAST_COLLECTIVE) { CHECK_GE(source_rank, 0); - cp->subdiv_source_rank.resize( + cp->instance.impl_details.subdiv_source_rank.resize( cp->instance.impl_details.subdiv_offsets.size(), -1); - for (int sdi = 0; sdi < cp->subdiv_source_rank.size(); ++sdi) { + for (int sdi = 0; sdi < cp->instance.impl_details.subdiv_source_rank.size(); + ++sdi) { for (int j = 0; j < cp->group.group_size; ++j) { if (cp->instance.impl_details.subdiv_permutations[sdi][j] == source_rank) { - cp->subdiv_source_rank[sdi] = j; + cp->instance.impl_details.subdiv_source_rank[sdi] = j; break; } } - CHECK_GE(cp->subdiv_source_rank[sdi], 0); + CHECK_GE(cp->instance.impl_details.subdiv_source_rank[sdi], 0); } } diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc b/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc index 4e3c7125f2..4e33c4779a 100644 --- a/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc +++ b/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc @@ -91,9 +91,10 @@ TEST_F(CollectiveParamResolverLocalTest, CompleteParamsReduction1Task) { EXPECT_TRUE(cps[i].task.is_local[j]); } EXPECT_EQ(cps[i].subdiv_rank[0], i); - EXPECT_EQ(cps[i].subdiv_source_rank.size(), 0); + EXPECT_EQ(cps[i].instance.impl_details.subdiv_source_rank.size(), 0); EXPECT_FALSE(cps[i].is_source); EXPECT_EQ(cps[i].default_rank, i); + EXPECT_TRUE(cps[i].instance.same_num_devices_per_task); } } @@ -138,10 +139,11 @@ TEST_F(CollectiveParamResolverLocalTest, CompleteParamsBroadcast1Task) { } ASSERT_GT(cps[i].subdiv_rank.size(), 0); EXPECT_EQ(cps[i].subdiv_rank[0], i); - ASSERT_GT(cps[i].subdiv_source_rank.size(), 0); - EXPECT_EQ(cps[i].subdiv_source_rank[0], 1); + ASSERT_GT(cps[i].instance.impl_details.subdiv_source_rank.size(), 0); + EXPECT_EQ(cps[i].instance.impl_details.subdiv_source_rank[0], 1); EXPECT_EQ(cps[i].is_source, (i == 1)); EXPECT_EQ(cps[i].default_rank, i); + EXPECT_TRUE(cps[i].instance.same_num_devices_per_task); } } diff --git a/tensorflow/core/common_runtime/collective_rma_local.h b/tensorflow/core/common_runtime/collective_rma_local.h index d25dd5f04a..716e23bfa1 100644 --- a/tensorflow/core/common_runtime/collective_rma_local.h +++ b/tensorflow/core/common_runtime/collective_rma_local.h @@ -67,6 +67,8 @@ class CollectiveRemoteAccessLocal : public PerStepCollectiveRemoteAccess { dev_resolver_->ClearTask(task); } + BufRendezvous* buf_rendezvous() override { return &buf_rendezvous_; } + // Copy utility that always copies bytes from src to dst even if // they are on the same device, unlike CopyTensor::ViaDMA which will // just change the dst buffer pointer in that case. diff --git a/tensorflow/core/framework/collective.cc b/tensorflow/core/framework/collective.cc index a26f2c2f31..d4ac50cbbe 100644 --- a/tensorflow/core/framework/collective.cc +++ b/tensorflow/core/framework/collective.cc @@ -38,6 +38,7 @@ CollInstanceParams& CollInstanceParams::operator=( device_names.clear(); device_names.assign(other.device_names.begin(), other.device_names.end()); task_names.assign(other.task_names.begin(), other.task_names.end()); + same_num_devices_per_task = other.same_num_devices_per_task; impl_details.subdiv_offsets.assign( other.impl_details.subdiv_offsets.begin(), other.impl_details.subdiv_offsets.end()); @@ -76,6 +77,13 @@ string CollInstanceParams::ToString() const { } strings::StrAppend(&v, "}"); // one subdiv } + if (!impl_details.subdiv_source_rank.empty()) { + strings::StrAppend(&v, " subdiv_source_rank={"); + for (const auto& r : impl_details.subdiv_source_rank) { + strings::StrAppend(&v, r, ","); + } + strings::StrAppend(&v, "}"); + } strings::StrAppend(&v, "}"); // all subdivs return v; } @@ -98,13 +106,6 @@ string CollectiveParams::ToString() const { for (const auto& r : subdiv_rank) { strings::StrAppend(&v, r, ","); } - if (!subdiv_source_rank.empty()) { - strings::StrAppend(&v, " subdiv_rank={"); - for (const auto& r : subdiv_source_rank) { - strings::StrAppend(&v, r, ","); - } - strings::StrAppend(&v, "}"); - } strings::StrAppend(&v, "}}"); return v; } diff --git a/tensorflow/core/framework/collective.h b/tensorflow/core/framework/collective.h index 5810c7fa54..40d82ab0e9 100644 --- a/tensorflow/core/framework/collective.h +++ b/tensorflow/core/framework/collective.h @@ -79,6 +79,8 @@ struct CollInstanceParams { std::vector device_names; // Task name prefix of corresponding device name. std::vector task_names; + // True if every task has the same number of devices. + bool same_num_devices_per_task; CollImplDetails impl_details; string ToString() const; CollInstanceParams& operator=(const struct CollInstanceParams& other); @@ -102,7 +104,6 @@ struct CollectiveParams { bool is_source; // broadcast only // Rank of this device in each subdivision permutation. std::vector subdiv_rank; - std::vector subdiv_source_rank; std::unique_ptr merge_op; // reduction only std::unique_ptr final_op; // reduction only string ToString() const; @@ -284,12 +285,14 @@ class CollectiveExecutor : public PeerAccessInterface, public core::RefCounted { TF_DISALLOW_COPY_AND_ASSIGN(CollectiveExecutor); }; -// Interface of a helper object that provices a CollectiveExecutor with +// Interface of a helper object that provides a CollectiveExecutor with // all of the remote access it needs. class CollectiveRemoteAccess : public PeerAccessInterface, public DeviceResolverInterface { public: virtual ~CollectiveRemoteAccess() {} + + virtual BufRendezvous* buf_rendezvous() = 0; }; // A per-step version of CollectiveRemoteAccess that cleans up outstanding -- GitLab From 55706e693ab20f6200061fb73067cbf27707cccd Mon Sep 17 00:00:00 2001 From: Igor Saprykin Date: Thu, 19 Apr 2018 13:19:27 -0700 Subject: [PATCH 679/791] Support various shapes in TPU DistributionStrategy. PiperOrigin-RevId: 193563912 --- .../distribute/python/minimize_loss_test.py | 11 +--- .../distribute/python/single_loss_example.py | 5 +- .../contrib/distribute/python/tpu_strategy.py | 61 +++++++++++++------ .../contrib/distribute/python/values.py | 33 ++++++++++ 4 files changed, 80 insertions(+), 30 deletions(-) diff --git a/tensorflow/contrib/distribute/python/minimize_loss_test.py b/tensorflow/contrib/distribute/python/minimize_loss_test.py index 6c73250ded..43b2e91cbf 100644 --- a/tensorflow/contrib/distribute/python/minimize_loss_test.py +++ b/tensorflow/contrib/distribute/python/minimize_loss_test.py @@ -57,25 +57,18 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase): model_fn, dataset_fn, layer = minimize_loss_example( optimizer_fn, use_bias=True, use_callable_loss=use_callable_loss) - def tpu_dataset_fn(): - return dataset_fn().batch(2) # TODO(isaprykin): Eliminate `is_tpu`. Probably add a # `DistributionStrategy.create_monitor` so that each DistributionStrategy # could influence its training loop. That method would return an instance # of Monitor. TPUMonitor would execute tpu.initialize_system() and # tpu.shutdown_system(). iterator = distribution.distribute_dataset( - tpu_dataset_fn if is_tpu else dataset_fn).make_one_shot_iterator() + dataset_fn).make_one_shot_iterator() def run_step(): - # TODO(isaprykin): Make iterator get_next() return a list of sub- - # batches for each iteration. Pass iterator.get_next() and not iterator - # to call_for_each_tower. return distribution.group( distribution.call_for_each_tower( - model_fn, - iterator.get_next() if not is_tpu else iterator, - run_concurrently=layer.built)) + model_fn, iterator.get_next(), run_concurrently=layer.built)) if not context.executing_eagerly(): with self.test_session() as sess: diff --git a/tensorflow/contrib/distribute/python/single_loss_example.py b/tensorflow/contrib/distribute/python/single_loss_example.py index 9e8f919c8a..abd13c6cc6 100644 --- a/tensorflow/contrib/distribute/python/single_loss_example.py +++ b/tensorflow/contrib/distribute/python/single_loss_example.py @@ -54,7 +54,7 @@ def minimize_loss_example(optimizer_fn, """Example of non-distribution-aware legacy code.""" def dataset_fn(): - return dataset_ops.Dataset.from_tensors([[1.]]).repeat() + return dataset_ops.Dataset.from_tensors([[1.]]).repeat().batch(2) # An Optimizer instance is created either outside or inside model_fn. outer_optimizer = None @@ -63,10 +63,11 @@ def minimize_loss_example(optimizer_fn, layer = core.Dense(1, use_bias=use_bias) - def model_fn(x): + def model_fn(xs): """A very simple model written by the user.""" def loss_fn(): + x = math_ops.reduce_mean(xs, keepdims=True) y = array_ops.reshape(layer(x), []) - constant_op.constant(1.) return y * y diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py index 804217b5ce..ceb52ceca7 100644 --- a/tensorflow/contrib/distribute/python/tpu_strategy.py +++ b/tensorflow/contrib/distribute/python/tpu_strategy.py @@ -23,6 +23,7 @@ from __future__ import print_function from tensorflow.contrib import tpu from tensorflow.contrib.distribute.python import one_device_strategy +from tensorflow.contrib.distribute.python import values from tensorflow.contrib.tpu.python.ops import tpu_ops from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes @@ -33,35 +34,48 @@ from tensorflow.python.ops import control_flow_ops # TODO(isaprykin): Consider whether inheriting is really appropriate. class TPUStrategy(one_device_strategy.OneDeviceStrategy): + """Experimental TPU distribution strategy implementation.""" - def __init__(self, master=None, iterations=None, model_dir=None): + def __init__(self, + global_batch_size=2, + num_cores_per_host=2, + iterations_per_step=2): + # TODO(isaprykin): Generalize the defaults. super(TPUStrategy, self).__init__('/cpu:0') + # TODO(isaprykin): Auto-detect number of cores and hosts. + self._num_cores_per_host = num_cores_per_host + self._global_batch_size = global_batch_size + # TODO(isaprykin): This might have to be per-call. + self._iterations_per_step = iterations_per_step + + def distribute_dataset(self, dataset_fn): + return values.PerIterationDataset( + self._call_dataset_fn(dataset_fn), self._iterations_per_step) def _call_for_each_tower(self, fn, *args, **kwargs): kwargs.pop('run_concurrently', None) - # TODO(isaprykin): Give an API for many iterations per step. - iterations = 1 + # TODO(isaprykin): Support variable arguments similar to PerDevice+regroup. + inputs = args[0] - # TODO(isaprykin): Do not hard code shapes and input format :) - # TODO(isaprykin): Detect the number of TPU cores automatically. - - def dequeueing_fn(*args, **kwargs): - del args, kwargs - x, = tpu.infeed_dequeue_tuple(dtypes=[dtypes.float32], shapes=[[1, 1, 1]]) - return fn(x) - - iterator = args[0] + sharded_shape = [None] # Python 2 nonlocal. def infeed_input(i): """Get input, split it and then enqueue.""" - batches = iterator.get_next() - batches = array_ops.split(batches, 2) + batches = array_ops.gather(inputs, i) + + # TODO(isaprykin): Handle partial batch. + global_shape = [self._global_batch_size] + list(batches.get_shape())[1:] + sharded_shape[0] = ([self._global_batch_size / self._num_cores_per_host] + + list(global_shape)[1:]) + + batches.set_shape(global_shape) + batches = array_ops.split(batches, self._num_cores_per_host) infeeds = [ tpu_ops.infeed_enqueue_tuple( - inputs=[batches[j]], shapes=[[1, 1, 1]], device_ordinal=j) - for j in range(2) + inputs=[batches[j]], shapes=[sharded_shape[0]], device_ordinal=j) + for j in range(self._num_cores_per_host) ] with ops.control_dependencies(infeeds): @@ -69,14 +83,23 @@ class TPUStrategy(one_device_strategy.OneDeviceStrategy): with ops.device('/task:0/device:CPU:0'): enqueue_ops = control_flow_ops.while_loop( - lambda i: i < iterations, + lambda i: i < self._iterations_per_step, infeed_input, [constant_op.constant(0)], parallel_iterations=1) + assert sharded_shape[0] + + def dequeueing_fn(*args, **kwargs): + del args, kwargs + x, = tpu.infeed_dequeue_tuple( + dtypes=[dtypes.float32], shapes=[sharded_shape[0]]) + return fn(x) + def iterate_on_tpu(): - return tpu.repeat(iterations, dequeueing_fn, []) + return tpu.repeat(self._iterations_per_step, dequeueing_fn, []) with one_device_strategy._OneDeviceTowerContext(self): # pylint: disable=protected-access - tpu_result = tpu.batch_parallel(iterate_on_tpu, [], num_shards=2) + tpu_result = tpu.batch_parallel( + iterate_on_tpu, [], num_shards=self._num_cores_per_host) return control_flow_ops.group(tpu_result, enqueue_ops) diff --git a/tensorflow/contrib/distribute/python/values.py b/tensorflow/contrib/distribute/python/values.py index 18fedd2775..62016c3a78 100644 --- a/tensorflow/contrib/distribute/python/values.py +++ b/tensorflow/contrib/distribute/python/values.py @@ -570,6 +570,39 @@ class PerDeviceDataset(object): dataset_iterator, self._devices, self._prefetch_on_device) +class MultiIterator(object): + """Iterator that returns results of multiple get_next()s.""" + + def __init__(self, dataset_iterator, iterations): + self._dataset_iterator = dataset_iterator + self._iterations = iterations + + def get_next(self, name=None): + return [ + self._dataset_iterator.get_next(name=name) + for _ in range(self._iterations) + ] + + @property + def initializer(self): + return self._dataset_iterator.initializer + + +class PerIterationDataset(object): + + def __init__(self, dataset, iterations): + self._dataset = dataset + self._iterations = iterations + + def make_one_shot_iterator(self): + iterator = self._dataset.make_one_shot_iterator() + return MultiIterator(iterator, self._iterations) + + def make_initializable_iterator(self): + iterator = self._dataset.make_initializable_iterator() + return MultiIterator(iterator, self._iterations) + + class MapOutput(object): """Map can result in multiple outputs per device.""" -- GitLab From 7f1e64eb94447665047fac16c67b5351bcf3c8a3 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 13:21:25 -0700 Subject: [PATCH 680/791] Allow output has a different shape from input in the image.transform (#17011). PiperOrigin-RevId: 193564222 --- tensorflow/contrib/image/kernels/image_ops.cc | 7 ++- tensorflow/contrib/image/kernels/image_ops.h | 2 +- tensorflow/contrib/image/ops/image_ops.cc | 52 +++++++++++++++++-- .../python/kernel_tests/image_ops_test.py | 30 +++++++++++ .../contrib/image/python/ops/image_ops.py | 39 ++++++++------ 5 files changed, 107 insertions(+), 23 deletions(-) diff --git a/tensorflow/contrib/image/kernels/image_ops.cc b/tensorflow/contrib/image/kernels/image_ops.cc index c2e32da133..ae4b1ba62a 100644 --- a/tensorflow/contrib/image/kernels/image_ops.cc +++ b/tensorflow/contrib/image/kernels/image_ops.cc @@ -70,6 +70,7 @@ class ImageProjectiveTransform : public OpKernel { void Compute(OpKernelContext* ctx) override { const Tensor& images_t = ctx->input(0); const Tensor& transform_t = ctx->input(1); + const Tensor& output_dim = ctx->input(2); OP_REQUIRES(ctx, images_t.shape().dims() == 4, errors::InvalidArgument("Input images must have rank 4")); OP_REQUIRES(ctx, @@ -83,7 +84,11 @@ class ImageProjectiveTransform : public OpKernel { auto images = images_t.tensor(); auto transform = transform_t.matrix(); Tensor* output_t; - OP_REQUIRES_OK(ctx, ctx->allocate_output(0, images_t.shape(), &output_t)); + // Image is NHWC format. + auto output_shape = images_t.shape(); + output_shape.set_dim(1, output_dim.vec()(0)); + output_shape.set_dim(2, output_dim.vec()(1)); + OP_REQUIRES_OK(ctx, ctx->allocate_output(0, output_shape, &output_t)); auto output = output_t->tensor(); (FillProjectiveTransform(interpolation_))( ctx->eigen_device(), &output, images, transform); diff --git a/tensorflow/contrib/image/kernels/image_ops.h b/tensorflow/contrib/image/kernels/image_ops.h index ad50133061..2320329b92 100644 --- a/tensorflow/contrib/image/kernels/image_ops.h +++ b/tensorflow/contrib/image/kernels/image_ops.h @@ -161,7 +161,7 @@ struct FillProjectiveTransform { void operator()(const Device& device, OutputType* output, const InputType& images, const TransformsType& transform) const { - output->device(device) = images.generate( + output->device(device) = output->generate( ProjectiveGenerator(images, transform, interpolation_)); } }; diff --git a/tensorflow/contrib/image/ops/image_ops.cc b/tensorflow/contrib/image/ops/image_ops.cc index 68771b3d05..4c6d8c0d19 100644 --- a/tensorflow/contrib/image/ops/image_ops.cc +++ b/tensorflow/contrib/image/ops/image_ops.cc @@ -19,9 +19,55 @@ limitations under the License. namespace tensorflow { +using shape_inference::DimensionHandle; using shape_inference::InferenceContext; using shape_inference::ShapeHandle; +namespace { + +// Sets output[0] to shape [batch_dim,height,width,channel_dim], where +// height and width come from the size_tensor. +Status SetOutputToSizedImage(InferenceContext* c, DimensionHandle batch_dim, + int size_input_idx, DimensionHandle channel_dim) { + // Verify shape of size input. + ShapeHandle size; + TF_RETURN_IF_ERROR(c->WithRank(c->input(size_input_idx), 1, &size)); + DimensionHandle unused; + TF_RETURN_IF_ERROR(c->WithValue(c->Dim(size, 0), 2, &unused)); + + // Get size values from the size tensor. + const Tensor* size_tensor = c->input_tensor(size_input_idx); + DimensionHandle width; + DimensionHandle height; + if (size_tensor == nullptr) { + width = c->UnknownDim(); + height = c->UnknownDim(); + } else { + // TODO(petewarden) - Remove once we have constant evaluation in C++ only. + if (size_tensor->dtype() != DT_INT32) { + return errors::InvalidArgument( + "Bad size input type for SetOutputToSizedImage: Expected DT_INT32 " + "but got ", + DataTypeString(size_tensor->dtype()), " for input #", size_input_idx, + " in ", c->DebugString()); + } + auto vec = size_tensor->vec(); + height = c->MakeDim(vec(0)); + width = c->MakeDim(vec(1)); + } + c->set_output(0, c->MakeShape({batch_dim, height, width, channel_dim})); + return Status::OK(); +} + +Status ResizeShapeFn(InferenceContext* c) { + ShapeHandle input; + TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 4, &input)); + return SetOutputToSizedImage(c, c->Dim(input, 0), 2 /* size_input_idx */, + c->Dim(input, 3)); +} + +} // namespace + // TODO(ringwalt): Add a "fill_mode" argument with "constant", "mirror", etc. // TODO(ringwalt): Add a "fill_constant" argument for constant mode (default 0). // TODO(ringwalt): Add an "output_shape" argument. This is sufficient to @@ -29,13 +75,11 @@ using shape_inference::ShapeHandle; REGISTER_OP("ImageProjectiveTransform") .Input("images: dtype") .Input("transforms: float32") + .Input("output_shape: int32") .Attr("dtype: {uint8, int32, int64, float32, float64}") .Attr("interpolation: string") .Output("transformed_images: dtype") - .SetShapeFn([](InferenceContext* c) { - c->set_output(0, c->input(0)); - return Status::OK(); - }) + .SetShapeFn(ResizeShapeFn) .Doc(R"doc( Applies the given transform to each of the images. diff --git a/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py b/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py index b50177ae56..c0151d320f 100644 --- a/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py +++ b/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py @@ -195,10 +195,40 @@ class ImageOpsTest(test_util.TensorFlowTestCase): x_init_value=test_image) self.assertLess(left_err, 1e-10) + def _test_grad_different_shape(self, input_shape, output_shape): + with self.test_session(): + test_image_shape = input_shape + test_image = np.random.randn(*test_image_shape) + test_image_tensor = constant_op.constant( + test_image, shape=test_image_shape) + test_transform = image_ops.angles_to_projective_transforms( + np.pi / 2, 4, 4) + + if len(output_shape) == 2: + resize_shape = output_shape + elif len(output_shape) == 3: + resize_shape = output_shape[0:2] + elif len(output_shape) == 4: + resize_shape = output_shape[1:3] + output = image_ops.transform( + images=test_image_tensor, + transforms=test_transform, + output_shape=resize_shape) + left_err = gradient_checker.compute_gradient_error( + test_image_tensor, + test_image_shape, + output, + output_shape, + x_init_value=test_image) + self.assertLess(left_err, 1e-10) + def test_grad(self): self._test_grad([16, 16]) self._test_grad([4, 12, 12]) self._test_grad([3, 4, 12, 12]) + self._test_grad_different_shape([16, 16], [8, 8]) + self._test_grad_different_shape([4, 12, 3], [8, 24, 3]) + self._test_grad_different_shape([3, 4, 12, 3], [3, 8, 24, 3]) class BipartiteMatchTest(test_util.TensorFlowTestCase): diff --git a/tensorflow/contrib/image/python/ops/image_ops.py b/tensorflow/contrib/image/python/ops/image_ops.py index c139ae89d8..0cb7bdc75d 100644 --- a/tensorflow/contrib/image/python/ops/image_ops.py +++ b/tensorflow/contrib/image/python/ops/image_ops.py @@ -212,7 +212,11 @@ def translations_to_projective_transforms(translations, name=None): axis=1) -def transform(images, transforms, interpolation="NEAREST", name=None): +def transform(images, + transforms, + output_shape=None, + interpolation="NEAREST", + name=None): """Applies the given transform(s) to the image(s). Args: @@ -228,7 +232,10 @@ def transform(images, transforms, interpolation="NEAREST", name=None): where `k = c0 x + c1 y + 1`. The transforms are *inverted* compared to the transform mapping input points to output points. Note that gradients are not backpropagated into transformation parameters. + output_shape: Output dimesion after the transform, [height, width]. + If None, output is the same size as input image. interpolation: Interpolation mode. Supported values: "NEAREST", "BILINEAR". + name: The name of the op. Returns: Image(s) with the same type and shape as `images`, with the given @@ -255,6 +262,14 @@ def transform(images, transforms, interpolation="NEAREST", name=None): else: raise TypeError("Images should have rank between 2 and 4.") + if output_shape is None: + output_shape = images.get_shape()[1:3] + elif len(output_shape) != 2: + raise TypeError( + "output_shape must either be None or a vector of 2 elements.") + output_shape = ops.convert_to_tensor( + output_shape, name="output_shape", dtype=dtypes.int32) + if len(transform_or_transforms.get_shape()) == 1: transforms = transform_or_transforms[None] elif transform_or_transforms.get_shape().ndims is None: @@ -265,7 +280,7 @@ def transform(images, transforms, interpolation="NEAREST", name=None): else: raise TypeError("Transforms should have rank 1 or 2.") output = gen_image_ops.image_projective_transform( - images, transforms, interpolation=interpolation.upper()) + images, transforms, output_shape, interpolation=interpolation.upper()) if len(image_or_images.get_shape()) == 2: return output[0, :, :, 0] elif len(image_or_images.get_shape()) == 3: @@ -375,14 +390,6 @@ def _image_projective_transform_grad(op, grad): if image_or_images.dtype.base_dtype not in _IMAGE_DTYPES: raise TypeError("Invalid dtype %s." % image_or_images.dtype) - if len(image_or_images.get_shape()) == 2: - images = image_or_images[None, :, :, None] - elif len(image_or_images.get_shape()) == 3: - images = image_or_images[None, :, :, :] - elif len(image_or_images.get_shape()) == 4: - images = image_or_images - else: - raise TypeError("Images should have rank between 2 and 4") if len(transform_or_transforms.get_shape()) == 1: transforms = transform_or_transforms[None] elif len(transform_or_transforms.get_shape()) == 2: @@ -395,13 +402,11 @@ def _image_projective_transform_grad(op, grad): inverse = linalg_ops.matrix_inverse(transforms) transforms = matrices_to_flat_transforms(inverse) output = gen_image_ops.image_projective_transform( - grad, transforms, interpolation=interpolation) - if len(image_or_images.get_shape()) == 2: - return [output[0, :, :, 0], None] - elif len(image_or_images.get_shape()) == 3: - return [output[0, :, :, :], None] - else: - return [output, None] + images=grad, + transforms=transforms, + output_shape=image_or_images.get_shape()[1:3], + interpolation=interpolation) + return [output, None, None] def bipartite_match(distance_mat, -- GitLab From ab47eb8d9bcac55fd19b0e862cf9a2a7de195787 Mon Sep 17 00:00:00 2001 From: Asim Shankar Date: Thu, 19 Apr 2018 13:38:43 -0700 Subject: [PATCH 681/791] tools/lib_package: Fix typo in README PiperOrigin-RevId: 193566850 --- tensorflow/tools/lib_package/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/tools/lib_package/README.md b/tensorflow/tools/lib_package/README.md index 7008148260..cb6aef2624 100644 --- a/tensorflow/tools/lib_package/README.md +++ b/tensorflow/tools/lib_package/README.md @@ -35,8 +35,8 @@ The following commands: bazel test --config opt //tensorflow/tools/lib_package:libtensorflow_test bazel build --config opt \ //tensorflow/tools/lib_package:libtensorflow_jni.tar.gz \ - //tensorflow/tools/lib_package:libtensorflow.jar \ - //tensorflow/tools/lib_package:libtensorflow-src.jar + //tensorflow/java:libtensorflow.jar \ + //tensorflow/java:libtensorflow-src.jar ``` test and produce the following: @@ -44,9 +44,9 @@ test and produce the following: - The native library (`libtensorflow_jni.so`) packaged in an archive at: `bazel-bin/tensorflow/tools/lib_package/libtensorflow_jni.tar.gz` - The Java archive at: - `bazel-bin/tensorflow/tools/lib_package/libtensorflow.jar` + `bazel-bin/tensorflow/java/libtensorflow.jar` - The Java archive for Java sources at: - `bazel-bin/tensorflow/tools/lib_package/libtensorflow-src.jar` + `bazel-bin/tensorflow/java/libtensorflow-src.jar` ## Release -- GitLab From 1e7289fc0e64a706bb1867cfe5a8c5f5d2f7150f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 14:05:06 -0700 Subject: [PATCH 682/791] Make flat_transforms_to_matrices and matrices_to_flat_transforms public available. PiperOrigin-RevId: 193571089 --- tensorflow/contrib/image/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tensorflow/contrib/image/__init__.py b/tensorflow/contrib/image/__init__.py index e982030bc8..8f406ace1d 100755 --- a/tensorflow/contrib/image/__init__.py +++ b/tensorflow/contrib/image/__init__.py @@ -25,6 +25,8 @@ projective transforms (including rotation) are supported. @@angles_to_projective_transforms @@compose_transforms @@adjust_yiq_hsv +@@flat_transforms_to_matrices +@@matrices_to_flat_transforms @@random_yiq_hsv @@rotate @@transform @@ -58,6 +60,8 @@ from tensorflow.contrib.image.python.ops.distort_image_ops import random_hsv_in_ from tensorflow.contrib.image.python.ops.image_ops import angles_to_projective_transforms from tensorflow.contrib.image.python.ops.image_ops import compose_transforms from tensorflow.contrib.image.python.ops.image_ops import connected_components +from tensorflow.contrib.image.python.ops.image_ops import flat_transforms_to_matrices +from tensorflow.contrib.image.python.ops.image_ops import matrices_to_flat_transforms from tensorflow.contrib.image.python.ops.image_ops import rotate from tensorflow.contrib.image.python.ops.image_ops import transform from tensorflow.contrib.image.python.ops.image_ops import translate -- GitLab From ab5abfa42bdced7bf1c371e5e1224bdc1fafdcc1 Mon Sep 17 00:00:00 2001 From: Asim Shankar Date: Thu, 19 Apr 2018 14:10:01 -0700 Subject: [PATCH 683/791] RecordReader: Simplify interface contract and implementation. Prior to this change, RecordReader had the following contract: - Records can be read in any order, EXCEPT if compression or buffering was enabled. - If the underlying file is being concurrently written to then calls to ReadRecord() may fail (because of an incomplete record near the end of a file), but a retry may succeed (once the record is written), EXCEPT if compression or buffering is enabled (in which case the failure will be terminal). This "retry-may-succeed" behavior is relied upon by tensorboard (https://github.com/tensorflow/tensorboard/blob/1.7/tensorboard/backend/event_processing/event_file_loader.py#L55) where one process (typically the model training process) is writing tf.summary events to an event file and another process (tensorboard) is concurrently reading it. With this change, the intent is to remove the EXCEPTions and have the same behavior irrespective of compression/buffering. Additionally, fix a memory leak when ZlibInputStream::Reset() is invoked. PiperOrigin-RevId: 193571934 --- tensorflow/core/lib/io/record_reader.cc | 147 ++++---------- tensorflow/core/lib/io/record_reader.h | 16 +- tensorflow/core/lib/io/recordio_test.cc | 212 ++++++++++++++------- tensorflow/core/lib/io/zlib_inputstream.cc | 9 +- tensorflow/core/lib/io/zlib_inputstream.h | 10 +- 5 files changed, 206 insertions(+), 188 deletions(-) diff --git a/tensorflow/core/lib/io/record_reader.cc b/tensorflow/core/lib/io/record_reader.cc index 6de850bb20..c24628be57 100644 --- a/tensorflow/core/lib/io/record_reader.cc +++ b/tensorflow/core/lib/io/record_reader.cc @@ -56,110 +56,55 @@ RecordReaderOptions RecordReaderOptions::CreateRecordReaderOptions( RecordReader::RecordReader(RandomAccessFile* file, const RecordReaderOptions& options) - : src_(file), options_(options) { + : options_(options), + input_stream_(new RandomAccessInputStream(file)), + last_read_failed_(false) { if (options.buffer_size > 0) { - input_stream_.reset(new BufferedInputStream(file, options.buffer_size)); - } else { - input_stream_.reset(new RandomAccessInputStream(file)); + input_stream_.reset(new BufferedInputStream(input_stream_.release(), + options.buffer_size, true)); } if (options.compression_type == RecordReaderOptions::ZLIB_COMPRESSION) { // We don't have zlib available on all embedded platforms, so fail. #if defined(IS_SLIM_BUILD) LOG(FATAL) << "Zlib compression is unsupported on mobile platforms."; #else // IS_SLIM_BUILD - zlib_input_stream_.reset(new ZlibInputStream( - input_stream_.get(), options.zlib_options.input_buffer_size, - options.zlib_options.output_buffer_size, options.zlib_options)); + input_stream_.reset(new ZlibInputStream( + input_stream_.release(), options.zlib_options.input_buffer_size, + options.zlib_options.output_buffer_size, options.zlib_options, true)); #endif // IS_SLIM_BUILD } else if (options.compression_type == RecordReaderOptions::NONE) { // Nothing to do. } else { - LOG(FATAL) << "Unspecified compression type :" << options.compression_type; + LOG(FATAL) << "Unrecognized compression type :" << options.compression_type; } } // Read n+4 bytes from file, verify that checksum of first n bytes is // stored in the last 4 bytes and store the first n bytes in *result. -// May use *storage as backing store. -Status RecordReader::ReadChecksummed(uint64 offset, size_t n, - StringPiece* result, string* storage) { +// +// offset corresponds to the user-provided value to ReadRecord() +// and is used only in error messages. +Status RecordReader::ReadChecksummed(uint64 offset, size_t n, string* result) { if (n >= SIZE_MAX - sizeof(uint32)) { return errors::DataLoss("record size too large"); } const size_t expected = n + sizeof(uint32); - storage->resize(expected); - -#if !defined(IS_SLIM_BUILD) - if (zlib_input_stream_) { - // If we have a zlib compressed buffer, we assume that the - // file is being read sequentially, and we use the underlying - // implementation to read the data. - // - // No checks are done to validate that the file is being read - // sequentially. At some point the zlib input buffer may support - // seeking, possibly inefficiently. - TF_RETURN_IF_ERROR(zlib_input_stream_->ReadNBytes(expected, storage)); - - if (storage->size() != expected) { - if (storage->empty()) { - return errors::OutOfRange("eof"); - } else { - return errors::DataLoss("truncated record at ", offset); - } - } + TF_RETURN_IF_ERROR(input_stream_->ReadNBytes(expected, result)); - uint32 masked_crc = core::DecodeFixed32(storage->data() + n); - if (crc32c::Unmask(masked_crc) != crc32c::Value(storage->data(), n)) { - return errors::DataLoss("corrupted record at ", offset); - } - *result = StringPiece(storage->data(), n); - } else { -#endif // IS_SLIM_BUILD - if (options_.buffer_size > 0) { - // If we have a buffer, we assume that the file is being read - // sequentially, and we use the underlying implementation to read the - // data. - // - // No checks are done to validate that the file is being read - // sequentially. - TF_RETURN_IF_ERROR(input_stream_->ReadNBytes(expected, storage)); - - if (storage->size() != expected) { - if (storage->empty()) { - return errors::OutOfRange("eof"); - } else { - return errors::DataLoss("truncated record at ", offset); - } - } - - const uint32 masked_crc = core::DecodeFixed32(storage->data() + n); - if (crc32c::Unmask(masked_crc) != crc32c::Value(storage->data(), n)) { - return errors::DataLoss("corrupted record at ", offset); - } - *result = StringPiece(storage->data(), n); + if (result->size() != expected) { + if (result->empty()) { + return errors::OutOfRange("eof"); } else { - // This version supports reading from arbitrary offsets - // since we are accessing the random access file directly. - StringPiece data; - TF_RETURN_IF_ERROR(src_->Read(offset, expected, &data, &(*storage)[0])); - if (data.size() != expected) { - if (data.empty()) { - return errors::OutOfRange("eof"); - } else { - return errors::DataLoss("truncated record at ", offset); - } - } - const uint32 masked_crc = core::DecodeFixed32(data.data() + n); - if (crc32c::Unmask(masked_crc) != crc32c::Value(data.data(), n)) { - return errors::DataLoss("corrupted record at ", offset); - } - *result = StringPiece(data.data(), n); + return errors::DataLoss("truncated record at ", offset); } -#if !defined(IS_SLIM_BUILD) } -#endif // IS_SLIM_BUILD + const uint32 masked_crc = core::DecodeFixed32(result->data() + n); + if (crc32c::Unmask(masked_crc) != crc32c::Value(result->data(), n)) { + return errors::DataLoss("corrupted record at ", offset); + } + result->resize(n); return Status::OK(); } @@ -167,50 +112,42 @@ Status RecordReader::ReadRecord(uint64* offset, string* record) { static const size_t kHeaderSize = sizeof(uint64) + sizeof(uint32); static const size_t kFooterSize = sizeof(uint32); + // Position the input stream. + int64 curr_pos = input_stream_->Tell(); + int64 desired_pos = static_cast(*offset); + if (curr_pos > desired_pos || curr_pos < 0 /* EOF */ || + (curr_pos == desired_pos && last_read_failed_)) { + last_read_failed_ = false; + TF_RETURN_IF_ERROR(input_stream_->Reset()); + TF_RETURN_IF_ERROR(input_stream_->SkipNBytes(desired_pos)); + } else if (curr_pos < desired_pos) { + TF_RETURN_IF_ERROR(input_stream_->SkipNBytes(desired_pos - curr_pos)); + } + DCHECK_EQ(desired_pos, input_stream_->Tell()); + // Read header data. - StringPiece lbuf; - Status s = ReadChecksummed(*offset, sizeof(uint64), &lbuf, record); + Status s = ReadChecksummed(*offset, sizeof(uint64), record); if (!s.ok()) { + last_read_failed_ = true; return s; } - const uint64 length = core::DecodeFixed64(lbuf.data()); + const uint64 length = core::DecodeFixed64(record->data()); // Read data - StringPiece data; - s = ReadChecksummed(*offset + kHeaderSize, length, &data, record); + s = ReadChecksummed(*offset + kHeaderSize, length, record); if (!s.ok()) { + last_read_failed_ = true; if (errors::IsOutOfRange(s)) { s = errors::DataLoss("truncated record at ", *offset); } return s; } - if (record->data() != data.data()) { - // RandomAccessFile placed the data in some other location. - memmove(&(*record)[0], data.data(), data.size()); - } - - record->resize(data.size()); - *offset += kHeaderSize + length + kFooterSize; + DCHECK_EQ(*offset, input_stream_->Tell()); return Status::OK(); } -Status RecordReader::SkipNBytes(uint64 offset) { -#if !defined(IS_SLIM_BUILD) - if (zlib_input_stream_) { - TF_RETURN_IF_ERROR(zlib_input_stream_->SkipNBytes(offset)); - } else { -#endif - if (options_.buffer_size > 0) { - TF_RETURN_IF_ERROR(input_stream_->SkipNBytes(offset)); - } -#if !defined(IS_SLIM_BUILD) - } -#endif - return Status::OK(); -} // namespace io - SequentialRecordReader::SequentialRecordReader( RandomAccessFile* file, const RecordReaderOptions& options) : underlying_(file, options), offset_(0) {} diff --git a/tensorflow/core/lib/io/record_reader.h b/tensorflow/core/lib/io/record_reader.h index 26278e0328..f6d587dfa0 100644 --- a/tensorflow/core/lib/io/record_reader.h +++ b/tensorflow/core/lib/io/record_reader.h @@ -69,25 +69,14 @@ class RecordReader { // Read the record at "*offset" into *record and update *offset to // point to the offset of the next record. Returns OK on success, // OUT_OF_RANGE for end of file, or something else for an error. - // - // Note: if buffering is used (with or without compression), access must be - // sequential. Status ReadRecord(uint64* offset, string* record); - // Skip the records till "offset". Returns OK on success, - // OUT_OF_RANGE for end of file, or something else for an error. - Status SkipNBytes(uint64 offset); - private: - Status ReadChecksummed(uint64 offset, size_t n, StringPiece* result, - string* storage); + Status ReadChecksummed(uint64 offset, size_t n, string* result); - RandomAccessFile* src_; RecordReaderOptions options_; std::unique_ptr input_stream_; -#if !defined(IS_SLIM_BUILD) - std::unique_ptr zlib_input_stream_; -#endif // IS_SLIM_BUILD + bool last_read_failed_; TF_DISALLOW_COPY_AND_ASSIGN(RecordReader); }; @@ -121,7 +110,6 @@ class SequentialRecordReader { return errors::InvalidArgument( "Trying to seek offset: ", offset, " which is less than the current offset: ", offset_); - TF_RETURN_IF_ERROR(underlying_.SkipNBytes(offset - offset_)); offset_ = offset; return Status::OK(); } diff --git a/tensorflow/core/lib/io/recordio_test.cc b/tensorflow/core/lib/io/recordio_test.cc index 63235761d9..da514bd21c 100644 --- a/tensorflow/core/lib/io/recordio_test.cc +++ b/tensorflow/core/lib/io/recordio_test.cc @@ -26,10 +26,11 @@ limitations under the License. namespace tensorflow { namespace io { +namespace { // Construct a string of the specified length made out of the supplied // partial string. -static string BigString(const string& partial_string, size_t n) { +string BigString(const string& partial_string, size_t n) { string result; while (result.size() < n) { result.append(partial_string); @@ -39,62 +40,66 @@ static string BigString(const string& partial_string, size_t n) { } // Construct a string from a number -static string NumberString(int n) { +string NumberString(int n) { char buf[50]; snprintf(buf, sizeof(buf), "%d.", n); return string(buf); } // Return a skewed potentially long string -static string RandomSkewedString(int i, random::SimplePhilox* rnd) { +string RandomSkewedString(int i, random::SimplePhilox* rnd) { return BigString(NumberString(i), rnd->Skewed(17)); } -class RecordioTest : public ::testing::Test { +class StringDest : public WritableFile { + public: + explicit StringDest(string* contents) : contents_(contents) {} + + Status Close() override { return Status::OK(); } + Status Flush() override { return Status::OK(); } + Status Sync() override { return Status::OK(); } + Status Append(const StringPiece& slice) override { + contents_->append(slice.data(), slice.size()); + return Status::OK(); + } + private: - class StringDest : public WritableFile { - public: - string contents_; - - Status Close() override { return Status::OK(); } - Status Flush() override { return Status::OK(); } - Status Sync() override { return Status::OK(); } - Status Append(const StringPiece& slice) override { - contents_.append(slice.data(), slice.size()); - return Status::OK(); + string* contents_; +}; + +class StringSource : public RandomAccessFile { + public: + explicit StringSource(string* contents) + : contents_(contents), force_error_(false) {} + + Status Read(uint64 offset, size_t n, StringPiece* result, + char* scratch) const override { + if (force_error_) { + force_error_ = false; + return errors::DataLoss("read error"); } - }; - - class StringSource : public RandomAccessFile { - public: - StringPiece contents_; - mutable bool force_error_; - mutable bool returned_partial_; - StringSource() : force_error_(false), returned_partial_(false) {} - - Status Read(uint64 offset, size_t n, StringPiece* result, - char* scratch) const override { - EXPECT_FALSE(returned_partial_) << "must not Read() after eof/error"; - - if (force_error_) { - force_error_ = false; - returned_partial_ = true; - return errors::DataLoss("read error"); - } - - if (offset >= contents_.size()) { - return errors::OutOfRange("end of file"); - } - - if (contents_.size() < offset + n) { - n = contents_.size() - offset; - returned_partial_ = true; - } - *result = StringPiece(contents_.data() + offset, n); - return Status::OK(); + + if (offset >= contents_->size()) { + return errors::OutOfRange("end of file"); + } + + if (contents_->size() < offset + n) { + n = contents_->size() - offset; } - }; + *result = StringPiece(contents_->data() + offset, n); + return Status::OK(); + } + + void force_error() { force_error_ = true; } + + private: + string* contents_; + mutable bool force_error_; +}; +class RecordioTest : public ::testing::Test { + private: + string contents_; StringDest dest_; StringSource source_; bool reading_; @@ -104,7 +109,9 @@ class RecordioTest : public ::testing::Test { public: RecordioTest() - : reading_(false), + : dest_(&contents_), + source_(&contents_), + reading_(false), readpos_(0), writer_(new RecordWriter(&dest_)), reader_(new RecordReader(&source_)) {} @@ -119,12 +126,11 @@ class RecordioTest : public ::testing::Test { TF_ASSERT_OK(writer_->WriteRecord(StringPiece(msg))); } - size_t WrittenBytes() const { return dest_.contents_.size(); } + size_t WrittenBytes() const { return contents_.size(); } string Read() { if (!reading_) { reading_ = true; - source_.contents_ = StringPiece(dest_.contents_); } string record; Status s = reader_->ReadRecord(&readpos_, &record); @@ -137,26 +143,20 @@ class RecordioTest : public ::testing::Test { } } - void IncrementByte(int offset, int delta) { - dest_.contents_[offset] += delta; - } + void IncrementByte(int offset, int delta) { contents_[offset] += delta; } - void SetByte(int offset, char new_byte) { - dest_.contents_[offset] = new_byte; - } + void SetByte(int offset, char new_byte) { contents_[offset] = new_byte; } - void ShrinkSize(int bytes) { - dest_.contents_.resize(dest_.contents_.size() - bytes); - } + void ShrinkSize(int bytes) { contents_.resize(contents_.size() - bytes); } void FixChecksum(int header_offset, int len) { // Compute crc of type/len/data - uint32_t crc = crc32c::Value(&dest_.contents_[header_offset + 6], 1 + len); + uint32_t crc = crc32c::Value(&contents_[header_offset + 6], 1 + len); crc = crc32c::Mask(crc); - core::EncodeFixed32(&dest_.contents_[header_offset], crc); + core::EncodeFixed32(&contents_[header_offset], crc); } - void ForceError() { source_.force_error_ = true; } + void ForceError() { source_.force_error(); } void StartReadingAt(uint64_t initial_offset) { readpos_ = initial_offset; } @@ -165,7 +165,6 @@ class RecordioTest : public ::testing::Test { Write("bar"); Write(BigString("x", 10000)); reading_ = true; - source_.contents_ = StringPiece(dest_.contents_); uint64 offset = WrittenBytes() + offset_past_end; string record; Status s = reader_->ReadRecord(&offset, &record); @@ -217,16 +216,100 @@ TEST_F(RecordioTest, RandomRead) { ASSERT_EQ("EOF", Read()); } +void TestNonSequentialReads(const RecordWriterOptions& writer_options, + const RecordReaderOptions& reader_options) { + string contents; + StringDest dst(&contents); + RecordWriter writer(&dst, writer_options); + for (int i = 0; i < 10; ++i) { + TF_ASSERT_OK(writer.WriteRecord(NumberString(i))) << i; + } + TF_ASSERT_OK(writer.Close()); + + StringSource file(&contents); + RecordReader reader(&file, reader_options); + + string record; + // First read sequentially to fill in the offsets table. + uint64 offsets[10] = {0}; + uint64 offset = 0; + for (int i = 0; i < 10; ++i) { + offsets[i] = offset; + TF_ASSERT_OK(reader.ReadRecord(&offset, &record)) << i; + } + + // Read randomly: First go back to record #3 then forward to #8. + offset = offsets[3]; + TF_ASSERT_OK(reader.ReadRecord(&offset, &record)); + EXPECT_EQ("3.", record); + EXPECT_EQ(offsets[4], offset); + + offset = offsets[8]; + TF_ASSERT_OK(reader.ReadRecord(&offset, &record)); + EXPECT_EQ("8.", record); + EXPECT_EQ(offsets[9], offset); +} + +TEST_F(RecordioTest, NonSequentialReads) { + TestNonSequentialReads(RecordWriterOptions(), RecordReaderOptions()); +} + +TEST_F(RecordioTest, NonSequentialReadsWithReadBuffer) { + RecordReaderOptions options; + options.buffer_size = 1 << 10; + TestNonSequentialReads(RecordWriterOptions(), options); +} + +TEST_F(RecordioTest, NonSequentialReadsWithCompression) { + TestNonSequentialReads( + RecordWriterOptions::CreateRecordWriterOptions("ZLIB"), + RecordReaderOptions::CreateRecordReaderOptions("ZLIB")); +} + // Tests of all the error paths in log_reader.cc follow: -static void AssertHasSubstr(StringPiece s, StringPiece expected) { +void AssertHasSubstr(StringPiece s, StringPiece expected) { EXPECT_TRUE(str_util::StrContains(s, expected)) << s << " does not contain " << expected; } +void TestReadError(const RecordWriterOptions& writer_options, + const RecordReaderOptions& reader_options) { + const string wrote = BigString("well hello there!", 100); + string contents; + StringDest dst(&contents); + TF_ASSERT_OK(RecordWriter(&dst, writer_options).WriteRecord(wrote)); + + StringSource file(&contents); + RecordReader reader(&file, reader_options); + + uint64 offset = 0; + string read; + file.force_error(); + Status status = reader.ReadRecord(&offset, &read); + ASSERT_TRUE(errors::IsDataLoss(status)); + ASSERT_EQ(0, offset); + + // A failed Read() shouldn't update the offset, and thus a retry shouldn't + // lose the record. + status = reader.ReadRecord(&offset, &read); + ASSERT_TRUE(status.ok()) << status; + EXPECT_GT(offset, 0); + EXPECT_EQ(wrote, read); +} + TEST_F(RecordioTest, ReadError) { - Write("foo"); - ForceError(); - AssertHasSubstr(Read(), "Data loss"); + TestReadError(RecordWriterOptions(), RecordReaderOptions()); +} + +TEST_F(RecordioTest, ReadErrorWithBuffering) { + RecordReaderOptions options; + options.buffer_size = 1 << 20; + TestReadError(RecordWriterOptions(), options); +} + +TEST_F(RecordioTest, ReadErrorWithCompression) { + TestReadError(RecordWriterOptions::CreateRecordWriterOptions("ZLIB"), + RecordReaderOptions::CreateRecordReaderOptions("ZLIB")); } TEST_F(RecordioTest, CorruptLength) { @@ -257,5 +340,6 @@ TEST_F(RecordioTest, ReadEnd) { CheckOffsetPastEndReturnsNoRecords(0); } TEST_F(RecordioTest, ReadPastEnd) { CheckOffsetPastEndReturnsNoRecords(5); } +} // namespace } // namespace io } // namespace tensorflow diff --git a/tensorflow/core/lib/io/zlib_inputstream.cc b/tensorflow/core/lib/io/zlib_inputstream.cc index 984fbc2810..bf8dcf0988 100644 --- a/tensorflow/core/lib/io/zlib_inputstream.cc +++ b/tensorflow/core/lib/io/zlib_inputstream.cc @@ -25,8 +25,9 @@ ZlibInputStream::ZlibInputStream( InputStreamInterface* input_stream, size_t input_buffer_bytes, // size of z_stream.next_in buffer size_t output_buffer_bytes, // size of z_stream.next_out buffer - const ZlibCompressionOptions& zlib_options) - : input_stream_(input_stream), + const ZlibCompressionOptions& zlib_options, bool owns_input_stream) + : owns_input_stream_(owns_input_stream), + input_stream_(input_stream), input_buffer_capacity_(input_buffer_bytes), output_buffer_capacity_(output_buffer_bytes), z_stream_input_(new Bytef[input_buffer_capacity_]), @@ -41,10 +42,14 @@ ZlibInputStream::~ZlibInputStream() { if (z_stream_) { inflateEnd(z_stream_.get()); } + if (owns_input_stream_) { + delete input_stream_; + } } Status ZlibInputStream::Reset() { TF_RETURN_IF_ERROR(input_stream_->Reset()); + inflateEnd(z_stream_.get()); InitZlibBuffer(); bytes_read_ = 0; return Status::OK(); diff --git a/tensorflow/core/lib/io/zlib_inputstream.h b/tensorflow/core/lib/io/zlib_inputstream.h index 9c7e14441c..6099e2455d 100644 --- a/tensorflow/core/lib/io/zlib_inputstream.h +++ b/tensorflow/core/lib/io/zlib_inputstream.h @@ -40,10 +40,13 @@ class ZlibInputStream : public InputStreamInterface { // Create a ZlibInputStream for `input_stream` with a buffer of size // `input_buffer_bytes` bytes for reading contents from `input_stream` and // another buffer with size `output_buffer_bytes` for caching decompressed - // contents. Does *not* take ownership of "input_stream". + // contents. + // + // Takes ownership of `input_stream` iff `owns_input_stream` is true. ZlibInputStream(InputStreamInterface* input_stream, size_t input_buffer_bytes, size_t output_buffer_bytes, - const ZlibCompressionOptions& zlib_options); + const ZlibCompressionOptions& zlib_options, + bool owns_input_stream = false); ~ZlibInputStream(); @@ -65,7 +68,8 @@ class ZlibInputStream : public InputStreamInterface { private: void InitZlibBuffer(); - InputStreamInterface* input_stream_; // Not owned + const bool owns_input_stream_; + InputStreamInterface* input_stream_; size_t input_buffer_capacity_; // Size of z_stream_input_ size_t output_buffer_capacity_; // Size of z_stream_output_ char* next_unread_byte_; // Next unread byte in z_stream_output_ -- GitLab From a4945fc86cabcf3d5f0b9eaac21bb7c1d1146d57 Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Thu, 19 Apr 2018 14:30:27 -0700 Subject: [PATCH 684/791] The HLO element type converter must remove side effecting instructions like Rng The CPU backend does not know how to lower bf16 typed RNG nodes so even unused instances of these can't remain in the HLO IR. HloComputation::ReplaceInstruction keeps these Rng nodes around since it doesn't remove side effecting nodes. PiperOrigin-RevId: 193575183 --- .../xla/service/hlo_element_type_converter.cc | 15 ++++- .../hlo_element_type_converter_test.cc | 66 +++++++++++++++++++ .../compiler/xla/service/hlo_instruction.cc | 37 ++++++++--- .../compiler/xla/service/hlo_instruction.h | 28 +++++--- tensorflow/compiler/xla/util.h | 10 +++ 5 files changed, 139 insertions(+), 17 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_element_type_converter.cc b/tensorflow/compiler/xla/service/hlo_element_type_converter.cc index c782d1b0ad..d236f83aeb 100644 --- a/tensorflow/compiler/xla/service/hlo_element_type_converter.cc +++ b/tensorflow/compiler/xla/service/hlo_element_type_converter.cc @@ -178,24 +178,37 @@ StatusOr HloElementTypeConverter::Run(HloModule* module) { if (hlo->shape().element_type() == eliminate_type_) { Shape shape = ShapeUtil::ChangeElementType(hlo->shape(), replace_with_type_); + new_hlo = computation->AddInstruction( hlo->CloneWithNewOperands(shape, new_operands, hlo->GetModule())); + TF_RETURN_IF_ERROR(new_hlo->CopyAllControlDepsFrom(hlo)); + new_hlo = ToElementType(new_hlo, eliminate_type_); } else if (ShapeUtil::IsTuple(hlo->shape())) { Shape old_shape = hlo->shape(); Shape new_shape = GetConvertedTupleShape(hlo->shape(), eliminate_type_, replace_with_type_); + new_hlo = computation->AddInstruction(hlo->CloneWithNewOperands( new_shape, new_operands, hlo->GetModule())); + TF_RETURN_IF_ERROR(new_hlo->CopyAllControlDepsFrom(hlo)); + // Convert the elements of the result of `new_hlo` to produce a new // tuple with shape `old_shape`. new_hlo = ConvertTupleElements(new_hlo, old_shape); } else { new_hlo = computation->AddInstruction(hlo->CloneWithNewOperands( hlo->shape(), new_operands, hlo->GetModule())); + TF_RETURN_IF_ERROR(new_hlo->CopyAllControlDepsFrom(hlo)); } - TF_RETURN_IF_ERROR(computation->ReplaceInstruction(hlo, new_hlo)); + TF_RETURN_IF_ERROR(hlo->ReplaceAllUsesWith(new_hlo)); + TF_RETURN_IF_ERROR(hlo->DropAllControlDeps()); + + // NB! We want to replace and remove side effecting instructions like Rng + // as well so we can't rely HloComputation::ReplaceInstruction to reliably + // remove the replaced instruction. + TF_RETURN_IF_ERROR(computation->RemoveInstruction(hlo)); changed = true; } } diff --git a/tensorflow/compiler/xla/service/hlo_element_type_converter_test.cc b/tensorflow/compiler/xla/service/hlo_element_type_converter_test.cc index cb94d9f19b..5c5a059e0f 100644 --- a/tensorflow/compiler/xla/service/hlo_element_type_converter_test.cc +++ b/tensorflow/compiler/xla/service/hlo_element_type_converter_test.cc @@ -22,6 +22,12 @@ namespace { namespace op = xla::testing::opcode_matchers; +using ::testing::Contains; +using ::testing::ElementsAre; +using ::testing::Eq; +using ::testing::Not; +using ::testing::ResultOf; + class HloElementTypeConverterTest : public HloTestBase { public: std::unique_ptr CreateModuleFromHloString( @@ -117,5 +123,65 @@ TEST_F(HloElementTypeConverterTest, BatchNormGradBF16Converted) { op::Convert(op::GetTupleElement(batch_norm, 2)))); } +TEST_F(HloElementTypeConverterTest, RngIsRemoved) { + const string& hlo_string = R"( +HloModule RngIsRemoved + +ENTRY main { + constant.3 = bf16[] constant(0) + constant.4 = bf16[] constant(1) + ROOT rng = bf16[1,1000,20]{2,1,0} rng(constant.3, constant.4), distribution=rng_uniform +} + )"; + auto module = CreateModuleFromHloString(hlo_string); + HloElementTypeConverter type_converter(BF16, F32); + TF_ASSERT_OK_AND_ASSIGN(bool converted, type_converter.Run(module.get())); + EXPECT_TRUE(converted); + + std::function is_bf16_rng = + [](const HloInstruction* inst) { + return inst->shape().element_type() == BF16 && + inst->opcode() == HloOpcode::kRng; + }; + + EXPECT_THAT(module->entry_computation()->instructions(), + Not(Contains(ResultOf(is_bf16_rng, Eq(true))))); +} + +TEST_F(HloElementTypeConverterTest, RngCtrlDep) { + const string& hlo_string = R"( +HloModule RngIsRemoved + +ENTRY main { + constant.3 = bf16[] constant(0) + constant.4 = bf16[] constant(1) + rng0 = bf16[1,2000,20]{2,1,0} rng(constant.3, constant.4), distribution=rng_uniform + ROOT rng1 = bf16[1,1000,20]{2,1,0} rng(constant.3, constant.4), control-predecessors={%rng0}, distribution=rng_uniform +} + )"; + auto module = CreateModuleFromHloString(hlo_string); + + HloElementTypeConverter type_converter(BF16, F32); + TF_ASSERT_OK_AND_ASSIGN(bool converted, type_converter.Run(module.get())); + EXPECT_TRUE(converted); + + HloInstruction *rng0, *rng1; + for (auto* inst : module->entry_computation()->instructions()) { + if (inst->opcode() == HloOpcode::kRng) { + const Shape& shape = inst->shape(); + ASSERT_EQ(shape.dimensions_size(), 3); + ASSERT_TRUE(shape.dimensions(1) == 2000 || shape.dimensions(1) == 1000); + if (shape.dimensions(1) == 2000) { + rng0 = inst; + } else { + rng1 = inst; + } + } + } + + EXPECT_THAT(rng0->control_successors(), ElementsAre(rng1)); + EXPECT_THAT(rng1->control_predecessors(), ElementsAre(rng0)); +} + } // namespace } // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index 6303bcc59f..a638d54d85 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -1678,14 +1678,35 @@ Status HloInstruction::AddControlDependencyTo(HloInstruction* instruction) { } Status HloInstruction::RemoveControlDependencyTo(HloInstruction* instruction) { - auto succ_it = std::find(control_successors_.begin(), - control_successors_.end(), instruction); - TF_RET_CHECK(succ_it != control_successors_.end()); - control_successors_.erase(succ_it); - auto pred_it = std::find(instruction->control_predecessors_.begin(), - instruction->control_predecessors_.end(), this); - TF_RET_CHECK(pred_it != instruction->control_predecessors_.end()); - instruction->control_predecessors_.erase(pred_it); + TF_RET_CHECK(instruction->parent() == parent()); + TF_RETURN_IF_ERROR(EraseElementFromVector(&control_successors_, instruction)); + TF_RETURN_IF_ERROR( + EraseElementFromVector(&instruction->control_predecessors_, this)); + return Status::OK(); +} + +Status HloInstruction::DropAllControlDeps() { + for (auto* ctrl_succ : control_successors_) { + TF_RETURN_IF_ERROR( + EraseElementFromVector(&ctrl_succ->control_predecessors_, this)); + } + for (auto* ctrl_pred : control_predecessors_) { + TF_RETURN_IF_ERROR( + EraseElementFromVector(&ctrl_pred->control_successors_, this)); + } + control_successors_.clear(); + control_predecessors_.clear(); + return Status::OK(); +} + +Status HloInstruction::CopyAllControlDepsFrom(const HloInstruction* inst) { + for (auto* ctrl_pred : inst->control_predecessors()) { + TF_RETURN_IF_ERROR(ctrl_pred->AddControlDependencyTo(this)); + } + + for (auto* ctrl_succ : inst->control_successors()) { + TF_RETURN_IF_ERROR(this->AddControlDependencyTo(ctrl_succ)); + } return Status::OK(); } diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h index 5a7394f7a6..a5e9aecb9e 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.h +++ b/tensorflow/compiler/xla/service/hlo_instruction.h @@ -557,6 +557,18 @@ class HloInstruction { // 'instruction'. Status RemoveControlDependencyTo(HloInstruction* instruction); + // Drops all control predecessors and successors from this HLO instruction. + Status DropAllControlDeps(); + + // Copies the control predecessors and successors on this HLO instruction to + // `inst`. Does not do a deep copy so this makes sense only if `inst` and + // this HLO are in the same module. + // + // Depending on the use cases we see in practice, in the future we may + // consider folding the logic here into Clone, CloneWithNewOperands and + // ReplaceAllUsesWith by treating control dependencies like data dependencies. + Status CopyAllControlDepsFrom(const HloInstruction* inst); + // Returns the set of control predecessors (successors) of this // instruction. Control predecessors (successors) must execute before (after) // the current instruction. @@ -1148,17 +1160,17 @@ class HloInstruction { // Clones the HLO instruction. The clone will have the same opcode, shape, and // operands. After creation the clone has no uses. "this" (the instruction // cloned from) is not changed. Suffix is the string to append to the name of - // the instruction to form the name of the cloned instruction. - // If the module pointer is not nullptr, it will be the module where - // the cloned computations will be added to (in order to support deep - // cloning). + // the instruction to form the name of the cloned instruction. If the module + // pointer is not nullptr, it will be the module where the cloned computations + // will be added to (in order to support deep cloning). Ignores the control + // predecessors and successors of this HLO instruction. std::unique_ptr Clone(const string& suffix = "clone", HloModule* module = nullptr) const; - // Clones the HLO instruction as above but with new shape and operands. - // If the module pointer is not nullptr, it will be the module where - // the cloned computations will be added to (in order to support deep - // cloning). + // Clones the HLO instruction as above but with new shape and operands. If + // the module pointer is not nullptr, it will be the module where the cloned + // computations will be added to (in order to support deep cloning). Ignores + // the control predecessors and successors of this HLO instruction. std::unique_ptr CloneWithNewOperands( const Shape& shape, tensorflow::gtl::ArraySlice operands, HloModule* module = nullptr) const; diff --git a/tensorflow/compiler/xla/util.h b/tensorflow/compiler/xla/util.h index 2da9f9ed6f..be33bd6dd1 100644 --- a/tensorflow/compiler/xla/util.h +++ b/tensorflow/compiler/xla/util.h @@ -528,6 +528,16 @@ bool IsInt32(T x) { // value is implementation-defined." return static_cast(x) == x; } + +template +Status EraseElementFromVector(std::vector* container, const T& value) { + // c_find returns a const_iterator which does not seem to work on gcc 4.8.4, + // and this breaks the ubuntu/xla_gpu build bot. + auto it = std::find(container->begin(), container->end(), value); + TF_RET_CHECK(it != container->end()); + container->erase(it); + return Status::OK(); +} } // namespace xla #define XLA_LOG_LINES(SEV, STRING) \ -- GitLab From 1aa032b94f630845abf6c3dce8d6623ae9e35b0f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 14:35:27 -0700 Subject: [PATCH 685/791] Replaced calls to deprecated tensorflow::StringPiece methods with their tensorflow::str_util equivalents. This will allow the deprecated methods to be removed. PiperOrigin-RevId: 193575992 --- tensorflow/core/platform/test_main.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/platform/test_main.cc b/tensorflow/core/platform/test_main.cc index 677114f5f2..e57bbd80af 100644 --- a/tensorflow/core/platform/test_main.cc +++ b/tensorflow/core/platform/test_main.cc @@ -26,7 +26,7 @@ limitations under the License. #include -#include "tensorflow/core/lib/core/stringpiece.h" +#include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/platform/stacktrace_handler.h" #include "tensorflow/core/platform/test.h" #include "tensorflow/core/platform/test_benchmark.h" @@ -37,7 +37,7 @@ GTEST_API_ int main(int argc, char** argv) { tensorflow::testing::InstallStacktraceHandler(); testing::InitGoogleTest(&argc, argv); for (int i = 1; i < argc; i++) { - if (tensorflow::StringPiece(argv[i]).starts_with("--benchmarks=")) { + if (tensorflow::str_util::StartsWith(argv[i], "--benchmarks=")) { const char* pattern = argv[i] + strlen("--benchmarks="); tensorflow::testing::Benchmark::Run(pattern); return 0; -- GitLab From 470842748b9ee219fa0fcb8e3de25720960c83e3 Mon Sep 17 00:00:00 2001 From: Olivia Nordquist Date: Thu, 19 Apr 2018 14:59:25 -0700 Subject: [PATCH 686/791] disabling opensource testing for failing xla test PiperOrigin-RevId: 193579805 --- tensorflow/compiler/xla/python/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/compiler/xla/python/BUILD b/tensorflow/compiler/xla/python/BUILD index 0517a5502e..0b9333b406 100644 --- a/tensorflow/compiler/xla/python/BUILD +++ b/tensorflow/compiler/xla/python/BUILD @@ -8,6 +8,7 @@ py_library( name = "xla_client", srcs = ["xla_client.py"], srcs_version = "PY2AND3", + tags = ["no_oss"], visibility = ["//visibility:public"], deps = [ ":pywrap_xla", -- GitLab From 2d0a7087a14f015ea49f4b8feb70e0b5ecd41b28 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 15:09:58 -0700 Subject: [PATCH 687/791] Only generate floating points that are fractions like n / 256, since they are RGB pixels. This fixes RGBToHSVTest.testBatch on low-precision dtypes like bfloat16. PiperOrigin-RevId: 193581652 --- tensorflow/compiler/tests/image_ops_test.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tensorflow/compiler/tests/image_ops_test.py b/tensorflow/compiler/tests/image_ops_test.py index 5b19e993ec..42e637734c 100644 --- a/tensorflow/compiler/tests/image_ops_test.py +++ b/tensorflow/compiler/tests/image_ops_test.py @@ -34,20 +34,23 @@ from tensorflow.python.ops import image_ops from tensorflow.python.platform import test +def GenerateNumpyRandomRGB(shape): + # Only generate floating points that are fractions like n / 256, since they + # are RGB pixels. Some low-precision floating point types in this test can't + # handle arbitrary precision floating points well. + return np.random.randint(0, 256, shape) / 256. + + class RGBToHSVTest(XLATestCase): def testBatch(self): - # TODO(b/78230407): Reenable the test on GPU. - if self.device == "XLA_GPU": - return - # Build an arbitrary RGB image np.random.seed(7) batch_size = 5 shape = (batch_size, 2, 7, 3) for nptype in self.float_types: - inp = np.random.rand(*shape).astype(nptype) + inp = GenerateNumpyRandomRGB(shape).astype(nptype) # Convert to HSV and back, as a batch and individually with self.test_session() as sess: @@ -87,7 +90,7 @@ class RGBToHSVTest(XLATestCase): def testRGBToHSVNumpy(self): """Tests the RGB to HSV conversion matches a reference implementation.""" for nptype in self.float_types: - rgb_flat = np.random.random(64 * 3).reshape((64, 3)).astype(nptype) + rgb_flat = GenerateNumpyRandomRGB((64, 3)).astype(nptype) rgb_np = rgb_flat.reshape(4, 4, 4, 3) hsv_np = np.array([ colorsys.rgb_to_hsv( -- GitLab From 38c0d7e1c0ee0617cf73ccf6809bd55d70089233 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 15:27:19 -0700 Subject: [PATCH 688/791] Convert a local variable and mutex to a struct so GUARDED_BY annotation works correctly. PiperOrigin-RevId: 193584438 --- tensorflow/core/kernels/sdca_ops.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tensorflow/core/kernels/sdca_ops.cc b/tensorflow/core/kernels/sdca_ops.cc index 55e68b348b..05c835ebc4 100644 --- a/tensorflow/core/kernels/sdca_ops.cc +++ b/tensorflow/core/kernels/sdca_ops.cc @@ -156,8 +156,10 @@ void DoCompute(const ComputeOptions& options, OpKernelContext* const context) { } else { examples.RandomShuffle(); } - mutex mu; - Status train_step_status GUARDED_BY(mu); + struct { + mutex mu; + Status value GUARDED_BY(mu); + } train_step_status; std::atomic atomic_index(-1); auto train_step = [&](const int64 begin, const int64 end) { // The static_cast here is safe since begin and end can be at most @@ -171,8 +173,8 @@ void DoCompute(const ComputeOptions& options, OpKernelContext* const context) { const Status conversion_status = options.loss_updater->ConvertLabel(&example_label); if (!conversion_status.ok()) { - mutex_lock l(mu); - train_step_status = conversion_status; + mutex_lock l(train_step_status.mu); + train_step_status.value = conversion_status; // Return from this worker thread - the calling thread is // responsible for checking context status and returning on error. return; @@ -217,7 +219,8 @@ void DoCompute(const ComputeOptions& options, OpKernelContext* const context) { Shard(worker_threads.num_threads, worker_threads.workers, examples.num_examples(), kCostPerUnit, train_step); - OP_REQUIRES_OK(context, train_step_status); + mutex_lock l(train_step_status.mu); + OP_REQUIRES_OK(context, train_step_status.value); } } // namespace -- GitLab From 4bcf49c4b22205fc829f89da96e37f366c9fa9e6 Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Thu, 19 Apr 2018 15:29:21 -0700 Subject: [PATCH 689/791] Prevent a bool field from being accessed when uninitialized. PiperOrigin-RevId: 193584746 --- tensorflow/core/distributed_runtime/message_wrappers.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/distributed_runtime/message_wrappers.h b/tensorflow/core/distributed_runtime/message_wrappers.h index 92c5668e3a..72a0c7edd8 100644 --- a/tensorflow/core/distributed_runtime/message_wrappers.h +++ b/tensorflow/core/distributed_runtime/message_wrappers.h @@ -353,7 +353,7 @@ class InMemoryRunGraphRequest : public MutableRunGraphRequestWrapper { private: string session_handle_; - bool create_worker_session_called_; + bool create_worker_session_called_ = false; string graph_handle_; int64 step_id_; ExecutorOpts exec_opts_; -- GitLab From 4868ddd508a567a497935378956e9da18976f152 Mon Sep 17 00:00:00 2001 From: Rohan Jain Date: Thu, 19 Apr 2018 15:32:37 -0700 Subject: [PATCH 690/791] Simplifying cols_to_vars update PiperOrigin-RevId: 193585237 --- tensorflow/python/feature_column/feature_column.py | 6 ++---- tensorflow/python/feature_column/feature_column_test.py | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/tensorflow/python/feature_column/feature_column.py b/tensorflow/python/feature_column/feature_column.py index 87a52f8441..a7c4eabcb2 100644 --- a/tensorflow/python/feature_column/feature_column.py +++ b/tensorflow/python/feature_column/feature_column.py @@ -417,10 +417,8 @@ def linear_model(features, trainable=trainable, name='linear_model') retval = linear_model_layer(features) # pylint: disable=not-callable - if cols_to_vars is None: - return retval - for k, v in linear_model_layer.cols_to_vars().items(): - cols_to_vars[k] = v + if cols_to_vars is not None: + cols_to_vars.update(linear_model_layer.cols_to_vars()) return retval diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py index 49e06b8245..d963dd9b55 100644 --- a/tensorflow/python/feature_column/feature_column_test.py +++ b/tensorflow/python/feature_column/feature_column_test.py @@ -1269,10 +1269,8 @@ def get_keras_linear_model_predictions(features, trainable, name='linear_model') retval = keras_linear_model(features) # pylint: disable=not-callable - if cols_to_vars is None: - return retval - for k, v in keras_linear_model.cols_to_vars().items(): - cols_to_vars[k] = v + if cols_to_vars is not None: + cols_to_vars.update(keras_linear_model.cols_to_vars()) return retval -- GitLab From f500bcb889b3598f386f59eb69a79af6b704bf50 Mon Sep 17 00:00:00 2001 From: joel-shor Date: Fri, 20 Apr 2018 01:41:28 +0300 Subject: [PATCH 691/791] [tf.data] Allow `sample_from_datasets` to accept a tf.Dataset object for `weights`. Tested: bazel test :interleave_dataset_op_test --- .../interleave_dataset_op_test.py | 59 +++++++++++-------- .../contrib/data/python/ops/interleave_ops.py | 25 ++++---- 2 files changed, 45 insertions(+), 39 deletions(-) diff --git a/tensorflow/contrib/data/python/kernel_tests/interleave_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/interleave_dataset_op_test.py index ff6d0c31aa..43aa4b1bd0 100644 --- a/tensorflow/contrib/data/python/kernel_tests/interleave_dataset_op_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/interleave_dataset_op_test.py @@ -928,8 +928,7 @@ class DirectedInterleaveDatasetTest(test.TestCase): sess.run(next_element) def _normalize(self, vec): - batched = (len(vec.shape) == 2) - return vec / vec.sum(axis=1, keepdims=True) if batched else vec / vec.sum() + return vec / vec.sum() def _chi2(self, expected, actual): actual = np.asarray(actual) @@ -938,35 +937,43 @@ class DirectedInterleaveDatasetTest(test.TestCase): chi2 = np.sum(diff * diff / expected, axis=0) return chi2 + def _testSampleFromDatasetsHelper(self, weights, num_datasets, num_samples): + # Create a dataset that samples each integer in `[0, num_datasets)` + # with probability given by `weights[i]`. + dataset = interleave_ops.sample_from_datasets([ + dataset_ops.Dataset.from_tensors(i).repeat(None) + for i in range(num_datasets) + ], weights) + dataset = dataset.take(num_samples) + iterator = dataset.make_one_shot_iterator() + next_element = iterator.get_next() + + with self.test_session() as sess: + freqs = np.zeros([num_datasets]) + for _ in range(num_samples): + freqs[sess.run(next_element)] += 1 + with self.assertRaises(errors.OutOfRangeError): + sess.run(next_element) + + return freqs + def testSampleFromDatasets(self): - random_seed.set_random_seed(1618) + random_seed.set_random_seed(1619) num_samples = 10000 - rand_probs = self._normalize(np.random.random_sample((10,))) - rand_probs2 = self._normalize(np.random.random_sample((15,))) + rand_probs = self._normalize(np.random.random_sample((15,))) - for probs in [[.5, .5], [.85, .05, .1], rand_probs, rand_probs2]: + # Use chi-squared test to assert that the observed distribution matches the + # expected distribution. Based on the implementation in + # "tensorflow/python/kernel_tests/multinomial_op_test.py". + for probs in [[.85, .05, .1], rand_probs]: probs = np.asarray(probs) + classes = len(probs) + freqs = self._testSampleFromDatasetsHelper(probs, classes, num_samples) + self.assertLess(self._chi2(probs, freqs / num_samples), 1e-3) - # Create a dataset that samples each integer in `[0, probs.shape[0])` - # with probability given by `probs[i]`. - dataset = interleave_ops.sample_from_datasets([ - dataset_ops.Dataset.from_tensors(i).repeat(None) - for i in range(probs.shape[0]) - ], probs) - dataset = dataset.take(num_samples) - iterator = dataset.make_one_shot_iterator() - next_element = iterator.get_next() - - with self.test_session() as sess: - freqs = np.zeros_like(probs) - for _ in range(num_samples): - freqs[sess.run(next_element)] += 1 - with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) - - # Use chi-squared test to assert that the observed distribution - # matches the expected distribution. Based on the implementation - # in "tensorflow/python/kernel_tests/multinomial_op_test.py". + # Also check that `weights` as a dataset samples correctly. + probs_ds = dataset_ops.Dataset.from_tensors(probs).repeat() + freqs = self._testSampleFromDatasetsHelper(probs_ds, classes, num_samples) self.assertLess(self._chi2(probs, freqs / num_samples), 1e-3) def testErrors(self): diff --git a/tensorflow/contrib/data/python/ops/interleave_ops.py b/tensorflow/contrib/data/python/ops/interleave_ops.py index 106a1ef388..5ae1fa9e9e 100644 --- a/tensorflow/contrib/data/python/ops/interleave_ops.py +++ b/tensorflow/contrib/data/python/ops/interleave_ops.py @@ -200,10 +200,10 @@ def sample_from_datasets(datasets, weights=None, seed=None): Args: datasets: A list of @{tf.data.Dataset} objects with compatible structure. - weights: (Optional.) A list of `len(datasets)` floating-point values, - where `weights[i]` represents the probability with which an element - should be sampled from `datasets[i]`. Defaults to a uniform distribution - across `datasets`. + weights: (Optional.) A list of `len(datasets)` floating-point values or a + @{tf.data.Dataset} object, where `weights[i]` represents the probability + with which an element should be sampled from `datasets[i]`. Defaults to a + uniform distribution across `datasets`. seed: (Optional.) A `tf.int64` scalar `tf.Tensor`, representing the random seed that will be used to create the distribution. See @{tf.set_random_seed} for behavior. @@ -219,24 +219,23 @@ def sample_from_datasets(datasets, weights=None, seed=None): """ num_datasets = len(datasets) if weights is None: - weights = array_ops.ones( - [num_datasets], dtype=dtypes.float32, name="weights") - else: + weights = dataset_ops.Dataset.from_tensors([1.0] * num_datasets).repeat() + elif not isinstance(weights, dataset_ops.Dataset): weights = ops.convert_to_tensor(weights, name="weights") if weights.dtype not in (dtypes.float32, dtypes.float64): raise TypeError("`weights` must be convertible to a tensor of " "`tf.float32` or `tf.float64` elements.") if not weights.shape.is_compatible_with([num_datasets]): raise ValueError("`weights` must be a vector of length `len(datasets)`.") + weights = dataset_ops.Dataset.from_tensors(weights).repeat() # The `stateless_multinomial()` op expects log-probabilities, as opposed to # weights. - logits = math_ops.log(weights, name="logits") - - def select_dataset(seed): + logits_ds = weights.map(lambda *p: math_ops.log(p, name="logits")) + def select_dataset(logits, seed): return array_ops.squeeze( - stateless.stateless_multinomial([logits], 1, seed=seed), axis=[0, 1]) - - selector_input = random_ops.RandomDataset(seed).batch(2).map(select_dataset) + stateless.stateless_multinomial(logits, 1, seed=seed), axis=[0, 1]) + selector_input = dataset_ops.Dataset.zip( + (logits_ds, random_ops.RandomDataset(seed).batch(2))).map(select_dataset) return DirectedInterleaveDataset(selector_input, datasets) -- GitLab From d5c32f4ccc85ad0d13f3a1f83e063211504cf976 Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Thu, 19 Apr 2018 15:55:53 -0700 Subject: [PATCH 692/791] Internal-only change. PiperOrigin-RevId: 193588868 --- tensorflow/contrib/data/python/kernel_tests/BUILD | 1 + tensorflow/contrib/estimator/BUILD | 1 + tensorflow/contrib/learn/BUILD | 5 ++++- tensorflow/python/kernel_tests/BUILD | 3 +++ tensorflow/python/kernel_tests/linalg/BUILD | 5 ++++- 5 files changed, 13 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD index 83daa04efc..05a4f5028a 100644 --- a/tensorflow/contrib/data/python/kernel_tests/BUILD +++ b/tensorflow/contrib/data/python/kernel_tests/BUILD @@ -216,6 +216,7 @@ py_test( srcs_version = "PY2AND3", tags = [ "no_pip", + "noasan", # times out "optonly", ], deps = [ diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD index 9e88bc7de1..62ddb3d290 100644 --- a/tensorflow/contrib/estimator/BUILD +++ b/tensorflow/contrib/estimator/BUILD @@ -447,6 +447,7 @@ py_test( srcs_version = "PY2AND3", tags = [ "no_pip", + "noasan", # times out "notsan", ], deps = [ diff --git a/tensorflow/contrib/learn/BUILD b/tensorflow/contrib/learn/BUILD index d665fc9335..3b053cd4c6 100644 --- a/tensorflow/contrib/learn/BUILD +++ b/tensorflow/contrib/learn/BUILD @@ -281,7 +281,10 @@ py_test( size = "medium", srcs = ["python/learn/estimators/estimator_test.py"], srcs_version = "PY2AND3", - tags = ["manual"], + tags = [ + "manual", + "noasan", # times out + ], deps = [ ":learn", "//tensorflow/contrib/framework:framework_py", diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index 9440f2a4f9..8628ca5d40 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -1190,6 +1190,9 @@ cuda_py_test( "//tensorflow/python/eager:context", ], shard_count = 10, + tags = [ + "noasan", # times out + ], ) cuda_py_test( diff --git a/tensorflow/python/kernel_tests/linalg/BUILD b/tensorflow/python/kernel_tests/linalg/BUILD index 4e3f24890b..7ffa48b653 100644 --- a/tensorflow/python/kernel_tests/linalg/BUILD +++ b/tensorflow/python/kernel_tests/linalg/BUILD @@ -123,7 +123,10 @@ cuda_py_test( "//tensorflow/python:platform_test", ], shard_count = 5, - tags = ["optonly"], + tags = [ + "noasan", # times out + "optonly", + ], ) cuda_py_test( -- GitLab From 9e5fdb83e609701457f6fdc2d153b1f7e83ead6c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 15:56:17 -0700 Subject: [PATCH 693/791] Automated g4 rollback of changelist 193564222 PiperOrigin-RevId: 193588935 --- tensorflow/contrib/image/kernels/image_ops.cc | 7 +-- tensorflow/contrib/image/kernels/image_ops.h | 2 +- tensorflow/contrib/image/ops/image_ops.cc | 52 ++----------------- .../python/kernel_tests/image_ops_test.py | 30 ----------- .../contrib/image/python/ops/image_ops.py | 39 ++++++-------- 5 files changed, 23 insertions(+), 107 deletions(-) diff --git a/tensorflow/contrib/image/kernels/image_ops.cc b/tensorflow/contrib/image/kernels/image_ops.cc index ae4b1ba62a..c2e32da133 100644 --- a/tensorflow/contrib/image/kernels/image_ops.cc +++ b/tensorflow/contrib/image/kernels/image_ops.cc @@ -70,7 +70,6 @@ class ImageProjectiveTransform : public OpKernel { void Compute(OpKernelContext* ctx) override { const Tensor& images_t = ctx->input(0); const Tensor& transform_t = ctx->input(1); - const Tensor& output_dim = ctx->input(2); OP_REQUIRES(ctx, images_t.shape().dims() == 4, errors::InvalidArgument("Input images must have rank 4")); OP_REQUIRES(ctx, @@ -84,11 +83,7 @@ class ImageProjectiveTransform : public OpKernel { auto images = images_t.tensor(); auto transform = transform_t.matrix(); Tensor* output_t; - // Image is NHWC format. - auto output_shape = images_t.shape(); - output_shape.set_dim(1, output_dim.vec()(0)); - output_shape.set_dim(2, output_dim.vec()(1)); - OP_REQUIRES_OK(ctx, ctx->allocate_output(0, output_shape, &output_t)); + OP_REQUIRES_OK(ctx, ctx->allocate_output(0, images_t.shape(), &output_t)); auto output = output_t->tensor(); (FillProjectiveTransform(interpolation_))( ctx->eigen_device(), &output, images, transform); diff --git a/tensorflow/contrib/image/kernels/image_ops.h b/tensorflow/contrib/image/kernels/image_ops.h index 2320329b92..ad50133061 100644 --- a/tensorflow/contrib/image/kernels/image_ops.h +++ b/tensorflow/contrib/image/kernels/image_ops.h @@ -161,7 +161,7 @@ struct FillProjectiveTransform { void operator()(const Device& device, OutputType* output, const InputType& images, const TransformsType& transform) const { - output->device(device) = output->generate( + output->device(device) = images.generate( ProjectiveGenerator(images, transform, interpolation_)); } }; diff --git a/tensorflow/contrib/image/ops/image_ops.cc b/tensorflow/contrib/image/ops/image_ops.cc index 4c6d8c0d19..68771b3d05 100644 --- a/tensorflow/contrib/image/ops/image_ops.cc +++ b/tensorflow/contrib/image/ops/image_ops.cc @@ -19,55 +19,9 @@ limitations under the License. namespace tensorflow { -using shape_inference::DimensionHandle; using shape_inference::InferenceContext; using shape_inference::ShapeHandle; -namespace { - -// Sets output[0] to shape [batch_dim,height,width,channel_dim], where -// height and width come from the size_tensor. -Status SetOutputToSizedImage(InferenceContext* c, DimensionHandle batch_dim, - int size_input_idx, DimensionHandle channel_dim) { - // Verify shape of size input. - ShapeHandle size; - TF_RETURN_IF_ERROR(c->WithRank(c->input(size_input_idx), 1, &size)); - DimensionHandle unused; - TF_RETURN_IF_ERROR(c->WithValue(c->Dim(size, 0), 2, &unused)); - - // Get size values from the size tensor. - const Tensor* size_tensor = c->input_tensor(size_input_idx); - DimensionHandle width; - DimensionHandle height; - if (size_tensor == nullptr) { - width = c->UnknownDim(); - height = c->UnknownDim(); - } else { - // TODO(petewarden) - Remove once we have constant evaluation in C++ only. - if (size_tensor->dtype() != DT_INT32) { - return errors::InvalidArgument( - "Bad size input type for SetOutputToSizedImage: Expected DT_INT32 " - "but got ", - DataTypeString(size_tensor->dtype()), " for input #", size_input_idx, - " in ", c->DebugString()); - } - auto vec = size_tensor->vec(); - height = c->MakeDim(vec(0)); - width = c->MakeDim(vec(1)); - } - c->set_output(0, c->MakeShape({batch_dim, height, width, channel_dim})); - return Status::OK(); -} - -Status ResizeShapeFn(InferenceContext* c) { - ShapeHandle input; - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 4, &input)); - return SetOutputToSizedImage(c, c->Dim(input, 0), 2 /* size_input_idx */, - c->Dim(input, 3)); -} - -} // namespace - // TODO(ringwalt): Add a "fill_mode" argument with "constant", "mirror", etc. // TODO(ringwalt): Add a "fill_constant" argument for constant mode (default 0). // TODO(ringwalt): Add an "output_shape" argument. This is sufficient to @@ -75,11 +29,13 @@ Status ResizeShapeFn(InferenceContext* c) { REGISTER_OP("ImageProjectiveTransform") .Input("images: dtype") .Input("transforms: float32") - .Input("output_shape: int32") .Attr("dtype: {uint8, int32, int64, float32, float64}") .Attr("interpolation: string") .Output("transformed_images: dtype") - .SetShapeFn(ResizeShapeFn) + .SetShapeFn([](InferenceContext* c) { + c->set_output(0, c->input(0)); + return Status::OK(); + }) .Doc(R"doc( Applies the given transform to each of the images. diff --git a/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py b/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py index c0151d320f..b50177ae56 100644 --- a/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py +++ b/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py @@ -195,40 +195,10 @@ class ImageOpsTest(test_util.TensorFlowTestCase): x_init_value=test_image) self.assertLess(left_err, 1e-10) - def _test_grad_different_shape(self, input_shape, output_shape): - with self.test_session(): - test_image_shape = input_shape - test_image = np.random.randn(*test_image_shape) - test_image_tensor = constant_op.constant( - test_image, shape=test_image_shape) - test_transform = image_ops.angles_to_projective_transforms( - np.pi / 2, 4, 4) - - if len(output_shape) == 2: - resize_shape = output_shape - elif len(output_shape) == 3: - resize_shape = output_shape[0:2] - elif len(output_shape) == 4: - resize_shape = output_shape[1:3] - output = image_ops.transform( - images=test_image_tensor, - transforms=test_transform, - output_shape=resize_shape) - left_err = gradient_checker.compute_gradient_error( - test_image_tensor, - test_image_shape, - output, - output_shape, - x_init_value=test_image) - self.assertLess(left_err, 1e-10) - def test_grad(self): self._test_grad([16, 16]) self._test_grad([4, 12, 12]) self._test_grad([3, 4, 12, 12]) - self._test_grad_different_shape([16, 16], [8, 8]) - self._test_grad_different_shape([4, 12, 3], [8, 24, 3]) - self._test_grad_different_shape([3, 4, 12, 3], [3, 8, 24, 3]) class BipartiteMatchTest(test_util.TensorFlowTestCase): diff --git a/tensorflow/contrib/image/python/ops/image_ops.py b/tensorflow/contrib/image/python/ops/image_ops.py index 0cb7bdc75d..c139ae89d8 100644 --- a/tensorflow/contrib/image/python/ops/image_ops.py +++ b/tensorflow/contrib/image/python/ops/image_ops.py @@ -212,11 +212,7 @@ def translations_to_projective_transforms(translations, name=None): axis=1) -def transform(images, - transforms, - output_shape=None, - interpolation="NEAREST", - name=None): +def transform(images, transforms, interpolation="NEAREST", name=None): """Applies the given transform(s) to the image(s). Args: @@ -232,10 +228,7 @@ def transform(images, where `k = c0 x + c1 y + 1`. The transforms are *inverted* compared to the transform mapping input points to output points. Note that gradients are not backpropagated into transformation parameters. - output_shape: Output dimesion after the transform, [height, width]. - If None, output is the same size as input image. interpolation: Interpolation mode. Supported values: "NEAREST", "BILINEAR". - name: The name of the op. Returns: Image(s) with the same type and shape as `images`, with the given @@ -262,14 +255,6 @@ def transform(images, else: raise TypeError("Images should have rank between 2 and 4.") - if output_shape is None: - output_shape = images.get_shape()[1:3] - elif len(output_shape) != 2: - raise TypeError( - "output_shape must either be None or a vector of 2 elements.") - output_shape = ops.convert_to_tensor( - output_shape, name="output_shape", dtype=dtypes.int32) - if len(transform_or_transforms.get_shape()) == 1: transforms = transform_or_transforms[None] elif transform_or_transforms.get_shape().ndims is None: @@ -280,7 +265,7 @@ def transform(images, else: raise TypeError("Transforms should have rank 1 or 2.") output = gen_image_ops.image_projective_transform( - images, transforms, output_shape, interpolation=interpolation.upper()) + images, transforms, interpolation=interpolation.upper()) if len(image_or_images.get_shape()) == 2: return output[0, :, :, 0] elif len(image_or_images.get_shape()) == 3: @@ -390,6 +375,14 @@ def _image_projective_transform_grad(op, grad): if image_or_images.dtype.base_dtype not in _IMAGE_DTYPES: raise TypeError("Invalid dtype %s." % image_or_images.dtype) + if len(image_or_images.get_shape()) == 2: + images = image_or_images[None, :, :, None] + elif len(image_or_images.get_shape()) == 3: + images = image_or_images[None, :, :, :] + elif len(image_or_images.get_shape()) == 4: + images = image_or_images + else: + raise TypeError("Images should have rank between 2 and 4") if len(transform_or_transforms.get_shape()) == 1: transforms = transform_or_transforms[None] elif len(transform_or_transforms.get_shape()) == 2: @@ -402,11 +395,13 @@ def _image_projective_transform_grad(op, grad): inverse = linalg_ops.matrix_inverse(transforms) transforms = matrices_to_flat_transforms(inverse) output = gen_image_ops.image_projective_transform( - images=grad, - transforms=transforms, - output_shape=image_or_images.get_shape()[1:3], - interpolation=interpolation) - return [output, None, None] + grad, transforms, interpolation=interpolation) + if len(image_or_images.get_shape()) == 2: + return [output[0, :, :, 0], None] + elif len(image_or_images.get_shape()) == 3: + return [output[0, :, :, :], None] + else: + return [output, None] def bipartite_match(distance_mat, -- GitLab From d4402725d2f6d9a8c5273ab1474117a27dd455c9 Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Thu, 19 Apr 2018 16:30:02 -0700 Subject: [PATCH 694/791] Make xla/service:cpu_plugin depend on the StreamExecutor host platform. PiperOrigin-RevId: 193593761 --- tensorflow/compiler/xla/service/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index 9009cbf845..d5d09bd8a3 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -699,6 +699,7 @@ cc_library( "//tensorflow/compiler/xla/service/cpu:cpu_compiler", "//tensorflow/compiler/xla/service/cpu:cpu_transfer_manager", "//tensorflow/core:stream_executor_no_cuda", + "//tensorflow/stream_executor:stream_executor_impl", ], ) -- GitLab From 704ac94a8e362feb3710391787342fe36187b9ef Mon Sep 17 00:00:00 2001 From: Benoit Steiner Date: Thu, 19 Apr 2018 16:30:26 -0700 Subject: [PATCH 695/791] Cleaned up the handling of merge nodes PiperOrigin-RevId: 193593810 --- .../core/grappler/costs/graph_properties.cc | 89 +++++++------------ 1 file changed, 32 insertions(+), 57 deletions(-) diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc index dd2d53dfdf..a0125ce342 100644 --- a/tensorflow/core/grappler/costs/graph_properties.cc +++ b/tensorflow/core/grappler/costs/graph_properties.cc @@ -670,6 +670,29 @@ class SymbolicShapeRefiner { return true; } + Status AddNode(const Node* node) { + // Create the inference context for this node. + std::vector input_shapes(node->num_inputs()); + std::vector>> + input_handle_shapes_and_types(node->num_inputs()); + std::vector input_tensors(node->num_inputs(), nullptr); + std::vector input_tensors_as_shapes; + + NodeContext& node_ctx = node_to_context_[node]; + TF_RETURN_IF_ERROR( + function_library_.LookUp(node->type_string(), &node_ctx.op_data)); + + node_ctx.inference_context.reset(new InferenceContext( + graph_def_version_, &node->def(), node->op_def(), input_shapes, + input_tensors, input_tensors_as_shapes, + std::move(input_handle_shapes_and_types))); + const Status s = node_ctx.inference_context->construction_status(); + if (!s.ok()) { + node_ctx.inference_context.reset(nullptr); + } + return s; + } + private: // Return the one ShapeHandle used to denote a fully unknown shape for a node // output. @@ -698,29 +721,6 @@ class SymbolicShapeRefiner { return dim; } - Status AddNode(const Node* node) { - // Create the inference context for this node. - std::vector input_shapes(node->num_inputs()); - std::vector>> - input_handle_shapes_and_types(node->num_inputs()); - std::vector input_tensors(node->num_inputs(), nullptr); - std::vector input_tensors_as_shapes; - - NodeContext& node_ctx = node_to_context_[node]; - TF_RETURN_IF_ERROR( - function_library_.LookUp(node->type_string(), &node_ctx.op_data)); - - node_ctx.inference_context.reset(new InferenceContext( - graph_def_version_, &node->def(), node->op_def(), input_shapes, - input_tensors, input_tensors_as_shapes, - std::move(input_handle_shapes_and_types))); - const Status s = node_ctx.inference_context->construction_status(); - if (!s.ok()) { - node_ctx.inference_context.reset(nullptr); - } - return s; - } - struct NodeContext { const OpRegistrationData* op_data; std::unique_ptr inference_context; @@ -929,37 +929,16 @@ Status GraphProperties::UpdateMergeNode(SymbolicShapeRefiner* shape_refiner, bool* new_shapes) const { InferenceContext* c = shape_refiner->GetContext(node); if (!c) { - // The shape refiner can't handle loops. Therefore we first need to remove - // all edges - std::vector edges; - std::vector edge_ptrs; - for (const Edge* edge : node->in_edges()) { - if (!edge->IsControlEdge()) { - edges.push_back(*edge); - edge_ptrs.push_back(edge); - } - } - for (const Edge* edge : edge_ptrs) { - if (!edge->IsControlEdge()) { - graph_->RemoveEdge(edge); - } - } // Now we can run shape inference - TF_RETURN_IF_ERROR(shape_refiner->UpdateNode(node, relax, new_shapes)); - // And add all the edges back - for (const Edge& edge : edges) { - graph_->AddEdge(edge.src(), edge.src_output(), edge.dst(), - edge.dst_input()); - } - - c = shape_refiner->GetContext(node); + TF_RETURN_IF_ERROR(shape_refiner->AddNode(node)); + c = CHECK_NOTNULL(shape_refiner->GetContext(node)); *new_shapes = true; - CHECK_NE(c, nullptr); - } - ShapeHandle out1; - TF_RETURN_IF_ERROR(c->WithRank(c->output(1), 0, &out1)); - c->set_output(1, out1); + // Infer the shape of the second output once and for all since it never + // changes. + ShapeHandle out1 = c->Scalar(); + c->set_output(1, out1); + } ShapeHandle out; bool out_initialized = false; @@ -981,11 +960,7 @@ Status GraphProperties::UpdateMergeNode(SymbolicShapeRefiner* shape_refiner, continue; } ShapeHandle input = in->output(e->src_output()); - if (relax) { - c->RelaxInput(e->dst_input(), input); - } else { - c->MergeInput(e->dst_input(), input); - } + c->SetInput(e->dst_input(), input); if (!out_initialized) { out_initialized = true; out = input; @@ -998,7 +973,7 @@ Status GraphProperties::UpdateMergeNode(SymbolicShapeRefiner* shape_refiner, } } - if (!shape_refiner->EquivalentShapes(out, c->output(0))) { + if (*new_shapes || !shape_refiner->EquivalentShapes(out, c->output(0))) { c->set_output(0, out); *new_shapes = true; } -- GitLab From c93a883fcea141dc0f63fe63afcd9490e39e3eaf Mon Sep 17 00:00:00 2001 From: Mark Heffernan Date: Thu, 19 Apr 2018 16:35:40 -0700 Subject: [PATCH 696/791] Improve error messages for LiteralTestUtil::Near. Previously error messages for mismatches were difficult to read with much of the space taken by useless stack traces. This CL cleans up the message considerably and adds additional information including statistics about the values and mismatches. PiperOrigin-RevId: 193594593 --- .../compiler/xla/tests/literal_test_util.cc | 772 +++++++++++------- .../compiler/xla/tests/literal_test_util.h | 9 +- .../xla/tests/literal_test_util_test.cc | 2 +- 3 files changed, 473 insertions(+), 310 deletions(-) diff --git a/tensorflow/compiler/xla/tests/literal_test_util.cc b/tensorflow/compiler/xla/tests/literal_test_util.cc index 81630df34c..c28f79ae38 100644 --- a/tensorflow/compiler/xla/tests/literal_test_util.cc +++ b/tensorflow/compiler/xla/tests/literal_test_util.cc @@ -39,6 +39,11 @@ limitations under the License. namespace xla { +using ::tensorflow::strings::Appendf; +using ::tensorflow::strings::Printf; +using ::tensorflow::strings::StrAppend; +using ::tensorflow::strings::StrCat; + /* static */ ::testing::AssertionResult LiteralTestUtil::EqualShapes( const Shape& expected, const Shape& actual) { if (ShapeUtil::IsTuple(expected) != ShapeUtil::IsTuple(actual)) { @@ -173,14 +178,11 @@ template auto lhs_double = static_cast(lhs); auto rhs_double = static_cast(rhs); if (ulhs != urhs) { - return ::testing::AssertionFailure() << tensorflow::strings::Printf( + return ::testing::AssertionFailure() << Printf( "floating values are not bitwise-equal; and equality testing " "was requested: %s=%g=%a vs %s=%g=%a", - tensorflow::strings::StrCat(tensorflow::strings::Hex(ulhs)) - .c_str(), - lhs_double, lhs_double, - tensorflow::strings::StrCat(tensorflow::strings::Hex(urhs)) - .c_str(), + StrCat(tensorflow::strings::Hex(ulhs)).c_str(), lhs_double, + lhs_double, StrCat(tensorflow::strings::Hex(urhs)).c_str(), rhs_double, rhs_double); } return ::testing::AssertionSuccess(); @@ -264,9 +266,7 @@ bool ExpectLiteralsEqual(const Literal& expected, const Literal& actual, << "expected:\n" << expected.ToString() << "\n\tvs actual:\n" << actual.ToString() - << (message.empty() - ? "" - : tensorflow::strings::StrCat("\nmessage: ", message)); + << (message.empty() ? "" : StrCat("\nmessage: ", message)); } /* static */ void LiteralTestUtil::ExpectNotEqual(const Literal& expected, @@ -321,9 +321,8 @@ bool ExpectLiteralsEqual(const Literal& expected, const Literal& actual, case TUPLE: { bool tuple_match = true; for (int i = 0; i < ShapeUtil::TupleElementCount(expected.shape()); ++i) { - SCOPED_TRACE(tensorflow::strings::StrCat( - "Tuple index ", i, " in ", - ShapeUtil::HumanString(expected.shape()))); + SCOPED_TRACE(StrCat("Tuple index ", i, " in ", + ShapeUtil::HumanString(expected.shape()))); // Create LiteralViews of the expected and actual elements. auto result = Equal(LiteralView::Create(expected, {i}), @@ -350,227 +349,301 @@ bool ExpectLiteralsEqual(const Literal& expected, const Literal& actual, namespace { +// Gets the total element count. For tuples, this is not the count of tuple +// elements, but the sum of elements of each tuple element. +int64 RecursiveElementCount(const Shape& shape) { + if (ShapeUtil::IsTuple(shape)) { + const int64 tuple_elements = ShapeUtil::TupleElementCount(shape); + int64 total = 0; + for (int64 i = 0; i < tuple_elements; ++i) { + total += RecursiveElementCount(ShapeUtil::GetTupleElementShape(shape, i)); + } + return total; + } else { + return ShapeUtil::ElementsIn(shape); + } +} + +// Calling ToString on a literal with over 100 million elements takes around +// 3 minutes. The utility of printing a literal with >1000 elements is +// questionable, especially when writing the Literal proto to disk is orders +// of magnitude faster. +string TruncateHugeLiteral(const Literal& literal) { + return RecursiveElementCount(literal.shape()) < 1000 + ? literal.ToString() + : "[TRUNCATED, Literal with more than 1000 values]"; +} + +// Returns whether the actual and expected values are mismatched with respect to +// nans. 'relaxed_nans' is interpreted as in xla::ErrorSpec. +template +bool NanMismatch(NativeT expected, NativeT actual, bool relaxed_nans) { + if (relaxed_nans) { + return !std::isnan(expected) && std::isnan(actual); + } else { + return std::isnan(expected) != std::isnan(actual); + } +} + +template <> +bool NanMismatch(complex64 expected, complex64 actual, + bool relaxed_nans) { + return NanMismatch(expected.real(), actual.real(), relaxed_nans) || + NanMismatch(expected.imag(), actual.imag(), relaxed_nans); +} + +template <> +bool NanMismatch(half expected, half actual, bool relaxed_nans) { + return NanMismatch(static_cast(expected), + static_cast(actual), relaxed_nans); +} + +// Converts the given floating-point value to a string. +template +string FpValueToString(NativeT value) { + return Printf("%8.4g", static_cast(value)); +} + +template <> +string FpValueToString(complex64 value) { + return Printf("%8.4g + %8.4fi", value.real(), value.imag()); +} + +// Returns the absolute value of the given floating point value. This function +// is used instead of std::abs directly in order to allow type-dependent +// implementations for NearComparator. +template +float FpAbsoluteValue(NativeT value) { + return std::abs(value); +} + +template <> +float FpAbsoluteValue(bfloat16 value) { + return FpAbsoluteValue(static_cast(value)); +} + +template <> +float FpAbsoluteValue(half value) { + return FpAbsoluteValue(static_cast(value)); +} + // Helper class for comparing floating-point literals within an error bound. +template class NearComparator { public: - explicit NearComparator(ErrorSpec error) : error_(error) {} + // Compares the two array literals elementwise and returns an assertion + // result. The assertion result is successful if all actual and expected + // elements are within the given error bound. In case of error, the assertion + // result contains a detailed error message in case of failure. + static ::testing::AssertionResult Compare(const Literal& expected, + const Literal& actual, + ErrorSpec error, + bool detailed_message) { + NearComparator comparator(expected, actual, error, + detailed_message); + return comparator.Run(); + } + + private: + // Data structure encapsulating metadata about a single element mismatch. + struct Mismatch { + NativeT actual; + NativeT expected; + float rel_error; + float abs_error; + + // The linear index of the failure within the shape. This linear index is + // from the 'actual' literal. + int64 linear_index; + + bool operator<(const Mismatch& other) const { + return rel_error < other.rel_error; + } - // Compares the two literals elementwise. EXPECTs each pair of elements to be - // within the error bound. Emits useful log messages and dumps literals to - // temporary files on failure. Returns true if literals match. - bool ExpectNear(const Literal& expected, const Literal& actual) { + string ToString(const Shape& shape) const { + return Printf( + "actual %s, expected %s, index %s, rel error %8.3g, abs error %8.3g", + FpValueToString(actual).c_str(), FpValueToString(expected).c_str(), + LiteralTestUtil::MultiIndexAsString( + IndexUtil::LinearIndexToMultidimensionalIndex(shape, + linear_index)) + .c_str(), + rel_error, abs_error); + } + }; + + explicit NearComparator(const Literal& expected, const Literal& actual, + ErrorSpec error, bool detailed_message) + : expected_(expected), + actual_(actual), + error_(error), + detailed_message_(detailed_message), + abs_value_buckets_(kAbsValueBucketBounds.size() - 1, {0, 0}), + abs_error_buckets_(kErrorBucketBounds.size(), 0), + rel_error_buckets_(kErrorBucketBounds.size(), 0) {} + + // Runs the comparison between expected and actual literals. + ::testing::AssertionResult Run() { VLOG(1) << "expected:"; - XLA_VLOG_LINES(1, TruncateHugeLiteral(expected)); + XLA_VLOG_LINES(1, TruncateHugeLiteral(expected_)); VLOG(1) << "actual:"; - XLA_VLOG_LINES(1, TruncateHugeLiteral(actual)); + XLA_VLOG_LINES(1, TruncateHugeLiteral(actual_)); // If the shapes mismatch, we simply fail the expectation instead of // printing out data, as it's a type error rather than a value error. ::testing::AssertionResult equal_shapes = - LiteralTestUtil::EqualShapes(expected.shape(), actual.shape()); + LiteralTestUtil::EqualShapes(expected_.shape(), actual_.shape()); if (!equal_shapes) { - EXPECT_TRUE(equal_shapes); - return false; + return equal_shapes; } - - // Set up members used during the comparison. - num_miscompares_ = 0; - abs_diff_sum_ = 0.0; - abs_expected_sum_ = 0.0; - abs_diff_miscompare_sum_ = 0.0; - abs_expected_miscompare_sum_ = 0.0; - max_rel_err_ = 0.0; - max_abs_err_ = 0.0; - first_linear_index_ = -1; - last_linear_index_ = -1; - max_rel_linear_index_ = -1; - max_abs_linear_index_ = -1; - miscompares_ = Literal(ShapeUtil::ChangeElementType(actual.shape(), PRED)); - miscompares_.PopulateWithValue(false); - multi_index_.resize(expected.shape().dimensions_size(), 0); - - switch (expected.shape().element_type()) { - case BF16: - ExpectLiteralsNear(expected, actual, 0); - break; - case F16: - ExpectLiteralsNear(expected, actual, 0); - break; - case F32: - ExpectLiteralsNear(expected, actual, 0); - break; - case F64: - ExpectLiteralsNear(expected, actual, 0); - break; - case C64: - ExpectLiteralsNear(expected, actual, 0); - break; - default: - LOG(FATAL) << "Unsupported primitive type in near comparator: " - << PrimitiveType_Name(expected.shape().element_type()) - << ". Must be floating-point type."; + if (!ShapeUtil::IsArray(expected_.shape())) { + return ::testing::AssertionFailure() << "Expected array shape"; } - if (num_miscompares_ > 0) { - if (!VLOG_IS_ON(1)) { - LOG(INFO) << "expected: " << ShapeUtil::HumanString(expected.shape()) - << " " << TruncateHugeLiteral(expected); - LOG(INFO) << "actual: " << ShapeUtil::HumanString(actual.shape()) - << " " << TruncateHugeLiteral(actual); - LOG(INFO) << "Dumping literals to temp files..."; - WriteLiteralToTempFile(expected, "expected"); - WriteLiteralToTempFile(actual, "actual"); - WriteLiteralToTempFile(miscompares_, "miscompares"); - } - EXPECT_TRUE(num_miscompares_ == 0) - << "\nmax relative mismatch at index " - << LiteralTestUtil::MultiIndexAsString( - IndexUtil::LinearIndexToMultidimensionalIndex( - actual.shape(), max_rel_linear_index_)) - << "\nmaximum relative error " << max_rel_err_ - << "\nmax absolute mismatch at index " - << LiteralTestUtil::MultiIndexAsString( - IndexUtil::LinearIndexToMultidimensionalIndex( - actual.shape(), max_abs_linear_index_)) - << "\nmaximum absolute error " << max_abs_err_ - << "\nfirst mismatch at index " - << LiteralTestUtil::MultiIndexAsString( - IndexUtil::LinearIndexToMultidimensionalIndex( - actual.shape(), first_linear_index_)) - << "\nlast mismatch at index " - << LiteralTestUtil::MultiIndexAsString( - IndexUtil::LinearIndexToMultidimensionalIndex( - actual.shape(), last_linear_index_)) - << "\ntotal absolute error " << abs_diff_sum_ - << "\ntotal absolute error of miscompares " - << abs_diff_miscompare_sum_ << "\ntotal relative error " - << (abs_diff_sum_ / abs_expected_sum_) - << "\ntotal relative error of miscompares " - << (abs_diff_miscompare_sum_ / abs_expected_miscompare_sum_) - << "\nfailure count " << num_miscompares_; + mismatches_ = Literal(ShapeUtil::ChangeElementType(actual_.shape(), PRED)); + mismatches_.PopulateWithValue(false); + + CompareLiterals(); + + if (num_mismatches_ == 0) { + return ::testing::AssertionSuccess(); + } else if (!VLOG_IS_ON(1)) { + LOG(INFO) << "expected: " << ShapeUtil::HumanString(expected_.shape()) + << " " << TruncateHugeLiteral(expected_); + LOG(INFO) << "actual: " << ShapeUtil::HumanString(actual_.shape()) + << " " << TruncateHugeLiteral(actual_); + LOG(INFO) << "Dumping literals to temp files..."; + WriteLiteralToTempFile(expected_, "expected"); + WriteLiteralToTempFile(actual_, "actual"); + WriteLiteralToTempFile(mismatches_, "mismatches"); } - return num_miscompares_ == 0; + return ::testing::AssertionFailure() << ErrorMessage(); } - private: - template - bool NanMismatch(NativeT expected, NativeT actual, bool relaxed_nans) { - if (relaxed_nans) { - return !std::isnan(expected) && std::isnan(actual); - } else { - return std::isnan(expected) != std::isnan(actual); + // Insert the given absolute value into the absolute value bucket vector. The + // bounds of the buckets are given by kAbsValueBucketBounds. + void UpdateAbsValueBucket(NativeT value, bool is_mismatch) { + // Adjust the bucket containing the absolute values of the 'actual' + // elements. + const float abs_value = FpAbsoluteValue(value); + for (int i = 0; i < abs_value_buckets_.size(); ++i) { + if (i == abs_value_buckets_.size() - 1 || + (abs_value >= kAbsValueBucketBounds[i] && + abs_value < kAbsValueBucketBounds[i + 1])) { + // The first value of the pair is the count of elements in the bucket, + // the second is the count of mismatches in the bucket. + abs_value_buckets_[i].first++; + if (is_mismatch) { + abs_value_buckets_[i].second++; + } + return; + } } } - template - void ExpectNear(NativeT expected, NativeT actual, - const ::testing::Message& message) { - EXPECT_NEAR(expected, actual, error_.abs) - << "expected:\n " << expected << "\n\tvs actual:\n " << actual << "\n" - << message; - } - - // EXPECTs that the two given scalar values are within the error bound. Keeps - // track of how many mismatches have occurred to keep the size of the output - // manageable. - template - bool ExpectValuesNear(NativeT expected, NativeT actual) { - if (expected == actual) { - return true; + // Insert the given error into the given error bucket vector. + void UpdateErrorBucket( + float error, tensorflow::gtl::MutableArraySlice error_buckets) { + CHECK_EQ(error_buckets.size(), kErrorBucketBounds.size()); + for (int i = 0; i < error_buckets.size(); ++i) { + if (error >= kErrorBucketBounds[i]) { + error_buckets[i]++; + } } - - const float abs_diff = std::abs(actual - expected); - const float rel_err = abs_diff / std::abs(expected); - const bool nan_mismatch = - NanMismatch(expected, actual, error_.relaxed_nans); - const bool mismatch = - (nan_mismatch || (abs_diff >= error_.abs && rel_err >= error_.rel)); - return !mismatch; } - // Assumes that expected vs actual fail ExpectValuesNear. - template - void UpdateAndLogMiscompares(const NativeT expected, const NativeT actual, - const Shape& shape, const int64 linear_index) { - const float abs_diff = std::abs(actual - expected); - const float rel_err = abs_diff / std::abs(expected); - abs_diff_sum_ += abs_diff; - abs_expected_sum_ += std::abs(expected); - if (rel_err > max_rel_err_ || std::isnan(rel_err)) { - max_rel_err_ = rel_err; - max_rel_linear_index_ = linear_index; + // Compares the two given elements from the expected and actual literals at + // the given literal_index and keeps track of various mismatch statistics. + void CompareValues(NativeT expected, NativeT actual, int64 linear_index) { + const bool is_nan_mismatch = + NanMismatch(expected, actual, error_.relaxed_nans); + float abs_error; + float rel_error; + if (actual == expected) { + abs_error = 0; + rel_error = 0; + } else if (is_nan_mismatch) { + num_nan_mismatches_++; + // A nan mismatch is considered to have infinite error. rel_error is used + // for sorting a std::set of the top mismatchs, and a nan value here will + // result in undefined behavior because nan's do not satisfy the strict + // weak ordering requirement of std containers. + abs_error = std::numeric_limits::infinity(); + rel_error = std::numeric_limits::infinity(); + } else { + abs_error = FpAbsoluteValue(actual - expected); + rel_error = abs_error / FpAbsoluteValue(expected); } - if (abs_diff > max_abs_err_ || std::isnan(abs_diff)) { - max_abs_err_ = abs_diff; - max_abs_linear_index_ = linear_index; + const bool is_abs_mismatch = abs_error > error_.abs; + const bool is_rel_mismatch = rel_error > error_.rel; + const bool is_mismatch = + is_nan_mismatch || (is_abs_mismatch && is_rel_mismatch); + + // Update the error of the relative bucket only if the *absolute* error + // bound is exceeded and vice versa. + if (is_abs_mismatch) { + num_abs_mismatches_++; + UpdateErrorBucket(rel_error, &rel_error_buckets_); } - if (VLOG_IS_ON(10)) { - VLOG(10) << tensorflow::strings::Printf( - "index %s abs_diff %f rel_err %f", - LiteralTestUtil::MultiIndexAsString( - IndexUtil::LinearIndexToMultidimensionalIndex(shape, - linear_index)) - .c_str(), - abs_diff, rel_err); + if (is_rel_mismatch) { + num_rel_mismatches_++; + UpdateErrorBucket(abs_error, &abs_error_buckets_); } - abs_diff_miscompare_sum_ += abs_diff; - abs_expected_miscompare_sum_ += std::abs(expected); - const int64 kMaxFailures = 2; - if (num_miscompares_ < kMaxFailures) { - const auto multi_index = - IndexUtil::LinearIndexToMultidimensionalIndex(shape, linear_index); - ::testing::Message msg; - msg << "mismatch at index " - << LiteralTestUtil::MultiIndexAsString(multi_index) << " abs diff " - << abs_diff << " rel err " << rel_err << " failure #" - << num_miscompares_; - ExpectNear(expected, actual, msg); - } else if (num_miscompares_ == kMaxFailures) { - LOG(ERROR) << "reached max 'loud' failure count; silently proceeding..."; + + UpdateAbsValueBucket(actual, is_mismatch); + + if (!is_mismatch) { + return; } - if (num_miscompares_ == 0) { - first_linear_index_ = linear_index; + + num_mismatches_++; + + // Keep track of the kTopRelativeErrorCount relative error mismatches. + if (top_rel_mismatches_.size() < kTopRelativeErrorCount || + rel_error > top_rel_mismatches_.begin()->rel_error) { + Mismatch mismatch = {actual, expected, rel_error, abs_error, + linear_index}; + top_rel_mismatches_.insert(mismatch); + if (top_rel_mismatches_.size() > kTopRelativeErrorCount) { + top_rel_mismatches_.erase(top_rel_mismatches_.begin()); + } } - num_miscompares_++; - last_linear_index_ = linear_index; - miscompares_.data()[linear_index] = true; + + mismatches_.data()[linear_index] = true; } - // Recursive function which compares the two given literals elementwise. - template - void ExpectLiteralsNear(const Literal& expected, const Literal& actual, - int64 dimension) { + // Compares the two literals elementwise. + void CompareLiterals() { // Fast path optimization for the case were layouts match. - if (LayoutUtil::Equal(actual.shape().layout(), expected.shape().layout())) { + if (LayoutUtil::Equal(actual_.shape().layout(), + expected_.shape().layout())) { tensorflow::gtl::ArraySlice expected_data = - expected.data(); + expected_.data(); tensorflow::gtl::ArraySlice actual_data = - actual.data(); + actual_.data(); const int64 len = expected_data.size(); for (int64 i = 0; i < len; ++i) { - const bool near = ExpectValuesNear(expected_data[i], actual_data[i]); - if (!near) { - UpdateAndLogMiscompares(expected_data[i], actual_data[i], - actual.shape(), i); - } + CompareValues(expected_data[i], actual_data[i], i); } return; } + std::vector multi_index(ShapeUtil::Rank(actual_.shape()), 0); + CompareLiteralsSlow(0, &multi_index); + } - if (dimension == expected.shape().dimensions_size()) { - bool near = ExpectValuesNear(expected.Get(multi_index_), - actual.Get(multi_index_)); - if (!near) { - UpdateAndLogMiscompares( - expected.Get(multi_index_), - actual.Get(multi_index_), actual.shape(), - IndexUtil::MultidimensionalIndexToLinearIndex(actual.shape(), - multi_index_)); - } + // Slow path for CompareLiterals when 'actual' and 'expected' literals have + // different layouts. In this case, multidimensional indices are constructed + // and indexed for each element. + void CompareLiteralsSlow(int64 dimension, std::vector* multi_index) { + if (dimension == multi_index->size()) { + CompareValues(expected_.Get(*multi_index), + actual_.Get(*multi_index), + IndexUtil::MultidimensionalIndexToLinearIndex( + actual_.shape(), *multi_index)); } else { - for (int64 i = 0; i < expected.shape().dimensions(dimension); ++i) { - multi_index_[dimension] = i; - ExpectLiteralsNear(expected, actual, dimension + 1); + for (int64 i = 0; i < expected_.shape().dimensions(dimension); ++i) { + (*multi_index)[dimension] = i; + CompareLiteralsSlow(dimension + 1, multi_index); } } } @@ -580,159 +653,247 @@ class NearComparator { int64 now_usec = tensorflow::Env::Default()->NowMicros(); string filename = tensorflow::io::JoinPath( tensorflow::testing::TmpDir(), - tensorflow::strings::Printf("tempfile-%s-%llx-%s", Hostname().c_str(), - now_usec, name.c_str())); + Printf("tempfile-%s-%llx-%s", Hostname().c_str(), now_usec, + name.c_str())); TF_CHECK_OK(tensorflow::WriteBinaryProto(tensorflow::Env::Default(), filename, literal.ToProto())); LOG(ERROR) << "wrote to " << name << " file: " << filename; } - // Gets the total element count. For tuples, this is not the count of tuple - // elements, but the sum of elements of each tuple element. - int64 RecursiveElementCount(const Shape& shape) { - if (ShapeUtil::IsTuple(shape)) { - const int64 tuple_elements = ShapeUtil::TupleElementCount(shape); - int64 total = 0; - for (int64 i = 0; i < tuple_elements; ++i) { - total += - RecursiveElementCount(ShapeUtil::GetTupleElementShape(shape, i)); - } - return total; - } else { - return ShapeUtil::ElementsIn(shape); + // Returns an error message string with a detailed breakdown of the + // mismatches. Called after calling Run(). + string ErrorMessage() { + string out; + int64 element_count = ShapeUtil::ElementsIn(actual_.shape()); + + auto percent_string = [](float a, float b) { + float pct = b == 0.0 ? 0.0 : 100.0 * a / b; + return Printf("%0.4f%%", pct); + }; + + Appendf(&out, + "\nMismatch count %lld (%s) in shape %s (%lld elements), abs bound " + "%g, rel bound %g\n", + num_mismatches_, + percent_string(num_mismatches_, element_count).c_str(), + ShapeUtil::HumanString(actual_.shape()).c_str(), + ShapeUtil::ElementsIn(actual_.shape()), error_.abs, error_.rel); + if (num_nan_mismatches_ > 0) { + StrAppend(&out, "nan mismatches ", num_nan_mismatches_, "\n"); + } + Appendf(&out, "Top relative error mismatches:\n"); + for (auto it = top_rel_mismatches_.rbegin(); + it != top_rel_mismatches_.rend(); ++it) { + StrAppend(&out, " ", it->ToString(actual_.shape()).c_str(), "\n"); } - } - // Calling ToString on a literal with over 100 million elements takes around - // 3 minutes. The utility of printing a literal with >1000 elements is - // questionable, especially when writing the Literal proto to disk is orders - // of magnitude faster. - string TruncateHugeLiteral(const Literal& literal) { - return RecursiveElementCount(literal.shape()) < 1000 - ? literal.ToString() - : "[TRUNCATED, Literal with more than 1000 values]"; - } + if (!detailed_message_) { + return out; + } - ErrorSpec error_; + StrAppend(&out, "Absolute magnitude breakdown of actual values:\n"); + CHECK_EQ(abs_value_buckets_.size() + 1, kAbsValueBucketBounds.size()); + for (int i = 0; i < abs_value_buckets_.size(); ++i) { + const int64 bucket_size = abs_value_buckets_[i].first; + const int64 bucket_mismatches = abs_value_buckets_[i].second; + string mismatch_str = bucket_mismatches > 0 + ? Printf(", mismatches %lld", bucket_mismatches) + : ""; + Appendf(&out, " %-6g <= x < %-6g : %7lld (%9s)%s\n", + kAbsValueBucketBounds[i], kAbsValueBucketBounds[i + 1], + bucket_size, percent_string(bucket_size, element_count).c_str(), + mismatch_str.c_str()); + } - // Number of element miscomparisons encountered so far. - int64 num_miscompares_; + auto print_accum_buckets = [&](const string& header, int64 total, + tensorflow::gtl::ArraySlice buckets) { + StrAppend(&out, header, ":\n"); + Appendf(&out, " < %-6g : %7lld (%s)\n", kErrorBucketBounds[0], + total - buckets[0], + percent_string(total - buckets[0], total).c_str()); + CHECK_EQ(buckets.size(), kErrorBucketBounds.size()); + for (int i = 0; i < kErrorBucketBounds.size(); ++i) { + Appendf(&out, " >= %-6g : %7lld (%s)\n", kErrorBucketBounds[i], + buckets[i], percent_string(buckets[i], total).c_str()); + } + }; + Appendf(&out, "Elements exceeding abs error bound %g: %lld (%s)\n", + error_.abs, num_abs_mismatches_, + percent_string(num_abs_mismatches_, element_count).c_str()); + print_accum_buckets( + "Relative error breakdown of elements exceeding abs error bound", + num_abs_mismatches_, rel_error_buckets_); + Appendf(&out, "Elements exceeding rel error bound %g: %lld (%s)\n", + error_.rel, num_rel_mismatches_, + percent_string(num_rel_mismatches_, element_count).c_str()); + print_accum_buckets( + "Absolute error breakdown of elements exceeding rel error bound", + num_rel_mismatches_, abs_error_buckets_); + return out; + } - // A Literal containing which elements did not match in the expected and - // actual literals. miscompares_ contains PREDs and is of the same sizes as - // the comparison literals. - Literal miscompares_; - - // A multidimensional index used when performing the recursive comparison. - std::vector multi_index_; - - // Aggregated Statistics on input. - double abs_diff_sum_; - double abs_expected_sum_; - double abs_diff_miscompare_sum_; - double abs_expected_miscompare_sum_; - float max_rel_err_; - float max_abs_err_; - int64 first_linear_index_; - int64 last_linear_index_; - int64 max_rel_linear_index_; - int64 max_abs_linear_index_; -}; + // 'actual' and 'expected' literals being compared. + const Literal& expected_; + const Literal& actual_; -template <> -bool NearComparator::NanMismatch(complex64 expected, - complex64 actual, - bool relaxed_nans) { - return NanMismatch(expected.real(), actual.real(), relaxed_nans) || - NanMismatch(expected.imag(), actual.imag(), relaxed_nans); -} + // The error bounds of the comparison. + ErrorSpec error_; -template <> -void NearComparator::ExpectNear(complex64 expected, complex64 actual, - const ::testing::Message& message) { - EXPECT_NEAR(expected.real(), actual.real(), error_.abs) - << "expected:\n " << expected << "\n\tvs actual:\n " << actual << "\n" - << message; - EXPECT_NEAR(expected.imag(), actual.imag(), error_.abs) - << "expected:\n " << expected << "\n\tvs actual:\n " << actual << "\n" - << message; -} + // Whether to include detailed breakdown of mismatches in the error message. + bool detailed_message_; -template <> -bool NearComparator::ExpectValuesNear(bfloat16 expected, - bfloat16 actual) { - return ExpectValuesNear(static_cast(expected), - static_cast(actual)); -} + // Number of element element mismatches encountered so far. + int64 num_mismatches_ = 0; -template <> -bool NearComparator::ExpectValuesNear(half expected, half actual) { - return ExpectValuesNear(static_cast(std::move(expected)), - static_cast(std::move(actual))); -} + // Number of elements with a nan mismatch. + int64 num_nan_mismatches_ = 0; -template <> -void NearComparator::UpdateAndLogMiscompares( - const bfloat16 expected, const bfloat16 actual, const Shape& shape, - const int64 linear_index) { - UpdateAndLogMiscompares(static_cast(expected), - static_cast(actual), shape, linear_index); -} + // Number of elements which exceed the absolute/relative error bound. + int64 num_abs_mismatches_ = 0; + int64 num_rel_mismatches_ = 0; -template <> -void NearComparator::UpdateAndLogMiscompares(half expected, half actual, - const Shape& shape, - const int64 linear_index) { - UpdateAndLogMiscompares(static_cast(std::move(expected)), - static_cast(std::move(actual)), shape, - linear_index); -} - -} // namespace + // A Literal containing which elements did not match in the expected and + // actual literals. mismatches_ contains PREDs and is of the same sizes as + // the comparison literals. + Literal mismatches_; + + // The number of mismatches to report in the output, sorted by relative error + // magnitude. + static constexpr int64 kTopRelativeErrorCount = 5; + + // The set of mismatches with the largest relative error. The size of this set + // is bounded by kTopRelativeErrorCount. + std::multiset top_rel_mismatches_; + + // Actual values are bucketed by absolute value. kAbsValueBucketBounds is the + // bounds of these buckets. abs_value_buckets_ contains a pair for each + // bucket: the element count and failure count. + static constexpr std::array kAbsValueBucketBounds = { + 0.0, 0.0001, 0.001, 0.01, 0.1, 1, std::numeric_limits::infinity()}; + std::vector> abs_value_buckets_; + + // Buckets for relative and absolute errors. The relative error buckets only + // contains those elements which exceed the *absolute* error bound, and vice + // versa. This makes it easy to see the effect of adjusting the relative (or + // absolute) error bound on the success of the comparison. kErrorBucketBounds + // are the lower bounds of the buckets in both vectors. The error buckets are + // a cumulative distribution so an error value may appear in more than one + // bucket. For example an error value of 0.003 may appear in the buckets + // bounded by 0.01, 0.1, and 1.0. + static constexpr std::array kErrorBucketBounds = {0.0001, 0.001, + 0.01, 0.1, 1}; + std::vector abs_error_buckets_; + std::vector rel_error_buckets_; +}; -/* static */ ::testing::AssertionResult LiteralTestUtil::Near( - const Literal& expected, const Literal& actual, const ErrorSpec& error) { +template +constexpr std::array NearComparator::kAbsValueBucketBounds; +template +constexpr std::array NearComparator::kErrorBucketBounds; + +// Helper function for comparing two literals for nearness. Handles tuple-shapes +// via recursion. shape_index is the ShapeIndex of expected (or actual) +// currently being compared. +::testing::AssertionResult NearHelper(const Literal& expected, + const Literal& actual, + const ErrorSpec& error, + bool detailed_message, + const ShapeIndex& shape_index) { ::testing::AssertionResult err = - EqualShapes(expected.shape(), actual.shape()); + LiteralTestUtil::EqualShapes(expected.shape(), actual.shape()); if (!err) { return err; } if (ShapeUtil::IsTuple(expected.shape())) { for (int64 i = 0; i < ShapeUtil::TupleElementCount(expected.shape()); ++i) { - SCOPED_TRACE(tensorflow::strings::StrCat( - "Tuple index ", i, " in ", ShapeUtil::HumanString(expected.shape()))); const auto expected_element = LiteralView::Create(expected, {i}); const auto actual_element = LiteralView::Create(actual, {i}); - + ShapeIndex element_index = shape_index; + element_index.push_back(i); ::testing::AssertionResult res = - Near(expected_element, actual_element, error); - if (err && !res) { - err = res; + NearHelper(expected_element, actual_element, error, detailed_message, + element_index); + if (!res) { + string err_message = + Printf("\nArray at shape index %s%s", + element_index.ToString().c_str(), res.message()); + if (err) { + err = ::testing::AssertionFailure() << err_message; + } else { + err << err_message; + } } } + if (!err && shape_index.empty()) { + // Emit a top-level error message containing the top-level shape in case + // of mismatch. + int64 total_elements = RecursiveElementCount(actual.shape()); + err = ::testing::AssertionFailure() + << Printf("\nMismatches in shape %s (%lld elements):\n%s", + ShapeUtil::HumanString(actual.shape()).c_str(), + total_elements, err.message()); + } return err; } if (ShapeUtil::ElementIsFloating(expected.shape()) || ShapeUtil::ElementIsComplex(expected.shape())) { - NearComparator comparator(error); - return comparator.ExpectNear(expected, actual) - ? ::testing::AssertionSuccess() - : ::testing::AssertionFailure() << "values were not near"; + switch (expected.shape().element_type()) { + case BF16: + return NearComparator::Compare(expected, actual, error, + detailed_message); + break; + case F16: + return NearComparator::Compare(expected, actual, error, + detailed_message); + break; + case F32: + return NearComparator::Compare(expected, actual, error, + detailed_message); + break; + case F64: + return NearComparator::Compare(expected, actual, error, + detailed_message); + break; + case C64: + return NearComparator::Compare(expected, actual, error, + detailed_message); + break; + default: + LOG(FATAL) << "Unsupported primitive type in near comparator: " + << PrimitiveType_Name(expected.shape().element_type()) + << ". Must be floating-point type."; + } } - return Equal(expected, actual); + // Non-floating point literal. + return LiteralTestUtil::Equal(expected, actual); +} + +} // namespace + +/* static */ ::testing::AssertionResult LiteralTestUtil::Near( + const Literal& expected, const Literal& actual, const ErrorSpec& error, + bool detailed_message) { + return NearHelper(expected, actual, error, detailed_message, + /*shape_index=*/{}); } /* static */ void LiteralTestUtil::ExpectNear(const Literal& expected, const Literal& actual, const ErrorSpec& error, const string& message) { - EXPECT_TRUE(Near(expected, actual, error)) - << (message.empty() - ? "" - : tensorflow::strings::StrCat("\nmessage: ", message)); + ::testing::AssertionResult res = + Near(expected, actual, error, /*detailed_message=*/false); + if (!res) { + res << "Expected: " << TruncateHugeLiteral(expected) << "\n"; + res << "Actual: " << TruncateHugeLiteral(actual) << "\n"; + if (!message.empty()) { + res << StrCat("\nmessage: ", message); + } + } + EXPECT_TRUE(res); } /*static*/ ::testing::AssertionResult LiteralTestUtil::NearOrEqual( @@ -754,8 +915,7 @@ void NearComparator::UpdateAndLogMiscompares(half expected, half actual, /* static */ string LiteralTestUtil::MultiIndexAsString( tensorflow::gtl::ArraySlice multi_index) { - return tensorflow::strings::StrCat( - "{", tensorflow::str_util::Join(multi_index, ","), "}"); + return StrCat("{", tensorflow::str_util::Join(multi_index, ","), "}"); } /* static */ std::unique_ptr LiteralTestUtil::Reshape( diff --git a/tensorflow/compiler/xla/tests/literal_test_util.h b/tensorflow/compiler/xla/tests/literal_test_util.h index 7b757a4bd7..a755568c0f 100644 --- a/tensorflow/compiler/xla/tests/literal_test_util.h +++ b/tensorflow/compiler/xla/tests/literal_test_util.h @@ -122,16 +122,19 @@ class LiteralTestUtil { // bounds are equivalent. // // Tuples are matched recursively. When comparing tensors of - // non-floating-point type, checks for exact equality, ignoring the ErroSpec. + // non-floating-point type, checks for exact equality, ignoring the ErrorSpec. // // If the shape of the literals is neither a complex/floating-point tensor nor // a tuple which contains a complex/floating-point tensor, Near() is // equivalent to Equal(). We don't raise an error in this case, because we // want to allow callers to call Near() even if they have no preconceptions // about the shapes being compared. + // + // If detailed_message is true, then the error message in the assertion result + // will contain a more detailed breakdown of mismatches. static ::testing::AssertionResult Near( - const Literal& expected, const Literal& actual, - const ErrorSpec& error) TF_MUST_USE_RESULT; + const Literal& expected, const Literal& actual, const ErrorSpec& error, + bool detailed_message = false) TF_MUST_USE_RESULT; // Expects expected and actual to be Near with the given error. static void ExpectNear(const Literal& expected, const Literal& actual, diff --git a/tensorflow/compiler/xla/tests/literal_test_util_test.cc b/tensorflow/compiler/xla/tests/literal_test_util_test.cc index 3a421f8458..9d619a77c7 100644 --- a/tensorflow/compiler/xla/tests/literal_test_util_test.cc +++ b/tensorflow/compiler/xla/tests/literal_test_util_test.cc @@ -89,7 +89,7 @@ TEST(LiteralTestUtilTest, ExpectNearFailurePlacesResultsInTemporaryDirectory) { EXPECT_EQ("2", literal->ToString()); } else if (result.find("actual") != string::npos) { EXPECT_EQ("4", literal->ToString()); - } else if (result.find("miscompares") != string::npos) { + } else if (result.find("mismatches") != string::npos) { EXPECT_EQ("true", literal->ToString()); } else { FAIL() << "unknown file in temporary directory: " << result; -- GitLab From 35543d5777b87c18b47eb73e83af41240a022e26 Mon Sep 17 00:00:00 2001 From: joel-shor Date: Fri, 20 Apr 2018 02:49:58 +0300 Subject: [PATCH 697/791] [tf.data] Correct / clarify docstring for `weights` as a dataset. This is a noop. --- tensorflow/contrib/data/python/ops/interleave_ops.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tensorflow/contrib/data/python/ops/interleave_ops.py b/tensorflow/contrib/data/python/ops/interleave_ops.py index 5ae1fa9e9e..812a50ecbf 100644 --- a/tensorflow/contrib/data/python/ops/interleave_ops.py +++ b/tensorflow/contrib/data/python/ops/interleave_ops.py @@ -200,10 +200,11 @@ def sample_from_datasets(datasets, weights=None, seed=None): Args: datasets: A list of @{tf.data.Dataset} objects with compatible structure. - weights: (Optional.) A list of `len(datasets)` floating-point values or a - @{tf.data.Dataset} object, where `weights[i]` represents the probability - with which an element should be sampled from `datasets[i]`. Defaults to a - uniform distribution across `datasets`. + weights: (Optional.) A list of `len(datasets)` floating-point values where + `weights[i]` represents the probability with which an element should be + sampled from `datasets[i]`, or a @{tf.data.Dataset} object where each + element is such a list. Defaults to a uniform distribution across + `datasets`. seed: (Optional.) A `tf.int64` scalar `tf.Tensor`, representing the random seed that will be used to create the distribution. See @{tf.set_random_seed} for behavior. -- GitLab From e07c9e23a94866966aa7e336a519b55931d570e3 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 16:53:14 -0700 Subject: [PATCH 698/791] Run EvaluateNodes for ModelPruner test except for NoPruning. PiperOrigin-RevId: 193596812 --- tensorflow/core/grappler/optimizers/BUILD | 1 + .../grappler/optimizers/model_pruner_test.cc | 52 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD index 63492e1a7f..a371186fe6 100644 --- a/tensorflow/core/grappler/optimizers/BUILD +++ b/tensorflow/core/grappler/optimizers/BUILD @@ -365,6 +365,7 @@ tf_cuda_cc_test( "//tensorflow/core:test", "//tensorflow/core:test_main", "//tensorflow/core:testlib", + "//tensorflow/core/grappler:devices", "//tensorflow/core/grappler:grappler_item", "//tensorflow/core/grappler:utils", "//tensorflow/core/grappler/inputs:trivial_test_graph_input_yielder", diff --git a/tensorflow/core/grappler/optimizers/model_pruner_test.cc b/tensorflow/core/grappler/optimizers/model_pruner_test.cc index 2b12eadec9..cf5b990377 100644 --- a/tensorflow/core/grappler/optimizers/model_pruner_test.cc +++ b/tensorflow/core/grappler/optimizers/model_pruner_test.cc @@ -17,6 +17,7 @@ limitations under the License. #include "tensorflow/cc/ops/standard_ops.h" #include "tensorflow/core/framework/node_def.pb.h" #include "tensorflow/core/framework/tensor_testutil.h" +#include "tensorflow/core/grappler/devices.h" #include "tensorflow/core/grappler/grappler_item.h" #include "tensorflow/core/grappler/inputs/trivial_test_graph_input_yielder.h" #include "tensorflow/core/grappler/utils.h" @@ -133,6 +134,13 @@ TEST_F(ModelPrunerTest, IdentityPruning) { EXPECT_EQ(NodeName(b.name()), new_d.input(0)); EXPECT_EQ(1, new_c.input_size()); EXPECT_EQ(NodeName(b.name()), new_c.input(0)); + + std::vector fetch = {"e"}; + auto expected_tensors = EvaluateNodes(item.graph, fetch); + auto actual_tensors = EvaluateNodes(output, fetch); + EXPECT_EQ(1, expected_tensors.size()); + EXPECT_EQ(1, actual_tensors.size()); + test::ExpectTensorEqual(expected_tensors[0], actual_tensors[0]); } TEST_F(ModelPrunerTest, NoOpPruning) { @@ -171,6 +179,13 @@ TEST_F(ModelPrunerTest, NoOpPruning) { EXPECT_EQ("a", new_node.input(0)); } } + + std::vector fetch = {"e"}; + auto expected_tensors = EvaluateNodes(item.graph, fetch); + auto actual_tensors = EvaluateNodes(output, fetch); + EXPECT_EQ(1, expected_tensors.size()); + EXPECT_EQ(1, actual_tensors.size()); + test::ExpectTensorEqual(expected_tensors[0], actual_tensors[0]); } TEST_F(ModelPrunerTest, PreserveIdentities) { @@ -201,6 +216,19 @@ TEST_F(ModelPrunerTest, PreserveIdentities) { TF_EXPECT_OK(status); EXPECT_EQ(item.graph.node_size(), output.node_size()); + + auto v_in_t = GenerateRandomTensor(TensorShape({3})); + Tensor v_ctrl_t(DT_BOOL, TensorShape({})); + v_ctrl_t.flat()(0) = true; + auto expected_tensors = EvaluateNodes( + item.graph, {"merge", "id2"}, {{"v_in", v_in_t}, {"v_ctrl", v_ctrl_t}}); + auto actual_tensors = EvaluateNodes(output, {"merge", "id2"}, + {{"v_in", v_in_t}, {"v_ctrl", v_ctrl_t}}); + EXPECT_EQ(2, expected_tensors.size()); + EXPECT_EQ(2, actual_tensors.size()); + for (int i = 0; i < expected_tensors.size(); i++) { + test::ExpectTensorEqual(expected_tensors[i], actual_tensors[i]); + } } TEST_F(ModelPrunerTest, PruningSkipsRefOutputs) { @@ -241,6 +269,14 @@ TEST_F(ModelPrunerTest, PruningSkipsRefOutputs) { EXPECT_EQ("b", new_c.input(0)); EXPECT_EQ("b", new_d.input(0)); EXPECT_EQ("b", new_e.input(0)); + + std::vector fetch = {"e"}; + auto a_t = GenerateRandomTensor(TensorShape({})); + auto expected_tensors = EvaluateNodes(item.graph, fetch, {{"a", a_t}}); + auto actual_tensors = EvaluateNodes(output, fetch, {{"a", a_t}}); + EXPECT_EQ(1, expected_tensors.size()); + EXPECT_EQ(1, actual_tensors.size()); + test::ExpectTensorEqual(expected_tensors[0], actual_tensors[0]); } // TODO(rmlarsen): Reenable this test when the issues with @@ -316,6 +352,12 @@ TEST_F(ModelPrunerTest, PruningPerservesFetch) { EXPECT_EQ(NodeName(b.name()), new_b.name()); const NodeDef& new_c = output.node(2); EXPECT_EQ(NodeName(c.name()), new_c.name()); + + auto expected_tensors = EvaluateNodes(item.graph, item.fetch); + auto actual_tensors = EvaluateNodes(output, item.fetch); + EXPECT_EQ(1, expected_tensors.size()); + EXPECT_EQ(1, actual_tensors.size()); + test::ExpectTensorEqual(expected_tensors[0], actual_tensors[0]); } TEST_F(ModelPrunerTest, PruningPerservesCrossDeviceIdentity) { @@ -348,6 +390,16 @@ TEST_F(ModelPrunerTest, PruningPerservesCrossDeviceIdentity) { EXPECT_EQ("c", node.input(0)); } } + if (GetNumAvailableGPUs() > 0) { + auto expected_tensors = EvaluateNodes(item.graph, item.fetch); + auto actual_tensors = EvaluateNodes(output, item.fetch); + EXPECT_EQ(4, expected_tensors.size()); + EXPECT_EQ(4, actual_tensors.size()); + for (int i = 0; i < expected_tensors.size(); i++) { + test::ExpectTensorNear(expected_tensors[i], actual_tensors[i], + 1e-6); + } + } } } // namespace -- GitLab From 2d8da1d12a5fbeaa99e1cdd761b735a02020611b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 17:17:05 -0700 Subject: [PATCH 699/791] Removed deprecated methods from tensorflow::StringPiece. This will allow tensorflow::StringPiece to be more easily replaced with absl::string_view as absl::string_view does not contain those methods. PiperOrigin-RevId: 193599651 --- tensorflow/core/lib/core/stringpiece.cc | 4 --- tensorflow/core/lib/core/stringpiece.h | 26 -------------------- tensorflow/core/lib/core/stringpiece_test.cc | 10 -------- 3 files changed, 40 deletions(-) diff --git a/tensorflow/core/lib/core/stringpiece.cc b/tensorflow/core/lib/core/stringpiece.cc index 0b006fa2b4..4c488066e4 100644 --- a/tensorflow/core/lib/core/stringpiece.cc +++ b/tensorflow/core/lib/core/stringpiece.cc @@ -25,10 +25,6 @@ std::ostream& operator<<(std::ostream& o, StringPiece piece) { return o; } -bool StringPiece::contains(StringPiece s) const { - return std::search(begin(), end(), s.begin(), s.end()) != end(); -} - size_t StringPiece::find(char c, size_t pos) const { if (pos >= size_) { return npos; diff --git a/tensorflow/core/lib/core/stringpiece.h b/tensorflow/core/lib/core/stringpiece.h index 835b938cbf..0cf6c24850 100644 --- a/tensorflow/core/lib/core/stringpiece.h +++ b/tensorflow/core/lib/core/stringpiece.h @@ -88,20 +88,6 @@ class StringPiece { size_t find(char c, size_t pos = 0) const; size_t rfind(char c, size_t pos = npos) const; - // DEPRECATED: Use tensorflow::str_util::StrContains instead. - bool contains(StringPiece s) const; - - // Checks whether StringPiece starts with x and if so advances the beginning - // of it to past the match. It's basically a shortcut for starts_with - // followed by remove_prefix. - // DEPRECATED: Use tensorflow::str_util::ConsumePrefix instead. - bool Consume(StringPiece x) { - if (starts_with(x)) { - remove_prefix(x.size_); - return true; - } - return false; - } StringPiece substr(size_t pos, size_t n = npos) const; @@ -114,18 +100,6 @@ class StringPiece { // > 0 iff "*this" > "b" int compare(StringPiece b) const; - // Return true iff "x" is a prefix of "*this" - // DEPRECATED: Use tensorflow::str_util::StartsWith instead. - bool starts_with(StringPiece x) const { - return ((size_ >= x.size_) && (memcmp(data_, x.data_, x.size_) == 0)); - } - // Return true iff "x" is a suffix of "*this" - // DEPRECATED: Use tensorflow::str_util::EndsWith instead. - bool ends_with(StringPiece x) const { - return ((size_ >= x.size_) && - (memcmp(data_ + (size_ - x.size_), x.data_, x.size_) == 0)); - } - private: const char* data_; size_t size_; diff --git a/tensorflow/core/lib/core/stringpiece_test.cc b/tensorflow/core/lib/core/stringpiece_test.cc index d0dbeb6072..de35d6eac6 100644 --- a/tensorflow/core/lib/core/stringpiece_test.cc +++ b/tensorflow/core/lib/core/stringpiece_test.cc @@ -55,14 +55,4 @@ TEST(StringPiece, Ctor) { } } -TEST(StringPiece, Contains) { - StringPiece a("abcdefg"); - StringPiece b("abcd"); - StringPiece c("efg"); - StringPiece d("gh"); - EXPECT_TRUE(a.contains(b)); - EXPECT_TRUE(a.contains(c)); - EXPECT_TRUE(!a.contains(d)); -} - } // namespace tensorflow -- GitLab From 4e17a3f1496b398afe632b002b0589b7346b2e3f Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Thu, 19 Apr 2018 17:18:10 -0700 Subject: [PATCH 700/791] [XLA] De-unique_ptr-ify ShapedBuffer and ScopedShapedBuffer. These are already notionally equivalent to T* and unique_ptr, so having a unique_ptr of a {Scoped,}ShapedBuffer is pretty redundant. Also clean up the ScopedShapedBuffer API a bit. PiperOrigin-RevId: 193599773 --- tensorflow/compiler/jit/xla_launch_util.cc | 47 ++--- tensorflow/compiler/jit/xla_launch_util.h | 2 +- tensorflow/compiler/jit/xla_tensor.cc | 6 +- tensorflow/compiler/jit/xla_tensor.h | 6 +- .../compiler/xla/client/local_client.cc | 23 ++- tensorflow/compiler/xla/client/local_client.h | 6 +- .../xla/python/local_computation_builder.cc | 46 ++--- .../xla/python/local_computation_builder.h | 6 +- .../xla/service/allocation_tracker.cc | 33 ++-- .../compiler/xla/service/allocation_tracker.h | 14 +- .../xla/service/cpu/cpu_executable.cc | 14 +- .../compiler/xla/service/cpu/cpu_executable.h | 8 +- .../service/cpu/parallel_cpu_executable.cc | 9 +- .../xla/service/cpu/parallel_cpu_executable.h | 4 +- tensorflow/compiler/xla/service/executable.cc | 16 +- tensorflow/compiler/xla/service/executable.h | 8 +- .../xla/service/gpu/gpu_executable.cc | 10 +- .../compiler/xla/service/gpu/gpu_executable.h | 4 +- tensorflow/compiler/xla/service/hlo_runner.cc | 45 +++-- .../xla/service/interpreter/executable.cc | 9 +- .../xla/service/interpreter/executable.h | 4 +- tensorflow/compiler/xla/service/service.cc | 14 +- .../compiler/xla/service/shaped_buffer.cc | 36 ++-- .../compiler/xla/service/shaped_buffer.h | 64 ++++--- .../compiler/xla/service/transfer_manager.cc | 21 ++- .../compiler/xla/service/transfer_manager.h | 8 +- .../compiler/xla/tests/dynamic_ops_test.cc | 8 +- tensorflow/compiler/xla/tests/fusion_test.cc | 16 +- .../xla/tests/local_client_allocation_test.cc | 7 +- .../xla/tests/local_client_execute_test.cc | 170 ++++++++---------- .../xla/tests/local_client_test_base.cc | 12 +- .../xla/tests/local_client_test_base.h | 11 +- .../xla/tests/transfer_manager_test.cc | 46 ++--- .../xla/tests/xla_hlo_profile_test.cc | 10 +- 34 files changed, 373 insertions(+), 370 deletions(-) diff --git a/tensorflow/compiler/jit/xla_launch_util.cc b/tensorflow/compiler/jit/xla_launch_util.cc index 50b0061d69..3520501c1a 100644 --- a/tensorflow/compiler/jit/xla_launch_util.cc +++ b/tensorflow/compiler/jit/xla_launch_util.cc @@ -32,10 +32,13 @@ limitations under the License. #include "tensorflow/core/framework/types.h" #include "tensorflow/core/util/stream_executor_util.h" +namespace { namespace gpu = perftools::gputools; +using xla::ScopedShapedBuffer; +using xla::ShapedBuffer; +} // anonymous namespace namespace tensorflow { - std::map SnapshotResourceVariables(OpKernelContext* ctx, int num_variables) { std::map snapshot; @@ -80,17 +83,17 @@ namespace { // Return the 'index''th subtree of the given ShapedBuffer as a // ScopedShapedBuffer. The returned ScopedShapedBuffer takes ownership of the // subtree, and sets the input's buffer pointers to nullptr for the subtree. -std::unique_ptr ExtractSubShapedBuffer( - xla::ShapedBuffer* shaped_buffer, int index, +ScopedShapedBuffer ExtractSubShapedBuffer( + ShapedBuffer* shaped_buffer, int index, xla::DeviceMemoryAllocator* allocator) { xla::Shape on_host_shape = xla::ShapeUtil::GetTupleElementShape( shaped_buffer->on_host_shape(), index); xla::Shape on_device_shape = xla::ShapeUtil::GetTupleElementShape( shaped_buffer->on_device_shape(), index); - xla::ShapedBuffer sub_shaped_buffer(on_host_shape, on_device_shape, - shaped_buffer->platform(), - shaped_buffer->device_ordinal()); + ShapedBuffer sub_shaped_buffer(on_host_shape, on_device_shape, + shaped_buffer->platform(), + shaped_buffer->device_ordinal()); auto& shape_tree = shaped_buffer->buffers(); auto& sub_shape_tree = sub_shaped_buffer.buffers(); @@ -102,8 +105,7 @@ std::unique_ptr ExtractSubShapedBuffer( index_to_buffer.second = gpu::DeviceMemoryBase(nullptr, 0); } } - return xla::ScopedShapedBuffer::MakeScoped(&sub_shaped_buffer, allocator) - .ValueOrDie(); + return ScopedShapedBuffer(std::move(sub_shaped_buffer), allocator); } } // namespace @@ -118,10 +120,10 @@ XlaComputationLaunchContext::XlaComputationLaunchContext( void XlaComputationLaunchContext::PopulateInputs( OpKernelContext* ctx, const XlaCompiler::CompilationResult* kernel, const std::map& variables) { - // Build xla::ShapedBuffers that point directly to the Tensor buffers. + // Build ShapedBuffers that point directly to the Tensor buffers. arg_buffers_.reserve(kernel->xla_input_shapes.size() + 1); arg_buffers_.resize(kernel->xla_input_shapes.size()); - arg_ptrs_ = std::vector(arg_buffers_.size()); + arg_ptrs_ = std::vector(arg_buffers_.size()); // Pass remaining parameters. const Tensor* t; @@ -140,8 +142,7 @@ void XlaComputationLaunchContext::PopulateInputs( if (xla::ShapeUtil::IsTuple(on_device_shape)) { const XlaTensor* xla_tensor = XlaTensor::FromTensor(t); CHECK(xla_tensor && xla_tensor->has_shaped_buffer()); - arg_ptrs_[i] = - const_cast(&xla_tensor->shaped_buffer()); + arg_ptrs_[i] = const_cast(&xla_tensor->shaped_buffer()); } else { CHECK(xla::ShapeUtil::Equal(shape, on_device_shape)) << "On-device shape " @@ -149,7 +150,7 @@ void XlaComputationLaunchContext::PopulateInputs( << " not the same as on-host shape " << xla::ShapeUtil::HumanStringWithLayout(shape); gpu::DeviceMemoryBase dmem = XlaTensor::DeviceMemoryFromTensor(*t); - arg_buffers_[i] = xla::MakeUnique( + arg_buffers_[i] = xla::MakeUnique( /*on_host_shape=*/shape, /*on_device_shape=*/shape, client_->platform(), client_->default_device_ordinal()); arg_buffers_[i]->set_buffer(dmem, /*index=*/{}); @@ -160,15 +161,15 @@ void XlaComputationLaunchContext::PopulateInputs( void XlaComputationLaunchContext::PopulateOutputs( OpKernelContext* ctx, const XlaCompiler::CompilationResult* kernel, - std::unique_ptr output) { + ScopedShapedBuffer output) { gpu::Stream* stream = ctx->op_device_context() ? ctx->op_device_context()->stream() : nullptr; // Computation output should always be a tuple. if (VLOG_IS_ON(2)) { - VLOG(2) << "Result tuple shape: " << output->on_host_shape().DebugString(); + VLOG(2) << "Result tuple shape: " << output.on_host_shape().DebugString(); VLOG(2) << "Result tuple shape (on device): " - << output->on_device_shape().DebugString(); + << output.on_device_shape().DebugString(); } CHECK_EQ(ctx->num_outputs(), kernel->outputs.size()); @@ -226,18 +227,18 @@ void XlaComputationLaunchContext::PopulateOutputs( const TensorShape& shape = kernel->outputs[i].shape; VLOG(2) << "Retval " << i << " shape " << shape.DebugString(); - gpu::DeviceMemoryBase buffer = output->buffer({output_num}); + gpu::DeviceMemoryBase buffer = output.buffer({output_num}); if (allocate_xla_tensors_) { Tensor* output_tensor; OP_REQUIRES_OK(ctx, ctx->allocate_output(i, shape, &output_tensor)); XlaTensor* xla_tensor = XlaTensor::FromTensor(output_tensor); CHECK(xla_tensor); - xla_tensor->set_shaped_buffer( - ExtractSubShapedBuffer(output.get(), output_num, xla_allocator_)); + xla_tensor->set_shaped_buffer(ScopedShapedBuffer( + ExtractSubShapedBuffer(&output, output_num, xla_allocator_))); } else { Tensor output_tensor = XlaTensorBuffer::MakeTensor( ctx->expected_output_dtype(i), shape, buffer, allocator); - output->set_buffer(gpu::DeviceMemoryBase(nullptr, 0), {output_num}); + output.set_buffer(gpu::DeviceMemoryBase(nullptr, 0), {output_num}); ctx->set_output(i, output_tensor); } ++output_num; @@ -257,7 +258,7 @@ void XlaComputationLaunchContext::PopulateOutputs( write.input_index >= 0 && write.input_index < ctx->num_inputs(), errors::Internal("Invalid input index for variable write.")); - gpu::DeviceMemoryBase buffer = output->buffer({output_num}); + gpu::DeviceMemoryBase buffer = output.buffer({output_num}); Var* variable = nullptr; // TODO(b/35625933): tensorflow::Var should contain a PersistentTensor, @@ -282,12 +283,12 @@ void XlaComputationLaunchContext::PopulateOutputs( XlaTensor* xla_tensor = XlaTensor::FromTensor(&output_tensor); CHECK(xla_tensor); xla_tensor->set_shaped_buffer( - ExtractSubShapedBuffer(output.get(), output_num, xla_allocator_)); + ExtractSubShapedBuffer(&output, output_num, xla_allocator_)); *variable->tensor() = output_tensor; } else { Tensor output_tensor = XlaTensorBuffer::MakeTensor( write.type, write.shape, buffer, allocator); - output->set_buffer(gpu::DeviceMemoryBase(nullptr, 0), {output_num}); + output.set_buffer(gpu::DeviceMemoryBase(nullptr, 0), {output_num}); *variable->tensor() = output_tensor; } ++output_num; diff --git a/tensorflow/compiler/jit/xla_launch_util.h b/tensorflow/compiler/jit/xla_launch_util.h index 14f70fe358..26dcaa8a51 100644 --- a/tensorflow/compiler/jit/xla_launch_util.h +++ b/tensorflow/compiler/jit/xla_launch_util.h @@ -87,7 +87,7 @@ class XlaComputationLaunchContext { // Given the XLA output in `output`, populate all outputs of `ctx`. void PopulateOutputs(OpKernelContext* ctx, const XlaCompiler::CompilationResult* kernel, - std::unique_ptr output); + xla::ScopedShapedBuffer output); // Return the argument list. Only valid after PopulateInputs() has been // called. diff --git a/tensorflow/compiler/jit/xla_tensor.cc b/tensorflow/compiler/jit/xla_tensor.cc index 956328e675..84b2835c40 100644 --- a/tensorflow/compiler/jit/xla_tensor.cc +++ b/tensorflow/compiler/jit/xla_tensor.cc @@ -65,10 +65,8 @@ Status XlaTensor::AllocateShapedBuffer(DataType dtype, const TensorShape& shape, device_ordinal, size, /*retry_on_failure=*/false)); } - TF_ASSIGN_OR_RETURN(auto scoped_buffer, - xla::ScopedShapedBuffer::MakeScoped( - &buffer, client->backend().memory_allocator())); - set_shaped_buffer(std::move(scoped_buffer)); + set_shaped_buffer(xla::ScopedShapedBuffer( + std::move(buffer), client->backend().memory_allocator())); return Status::OK(); } diff --git a/tensorflow/compiler/jit/xla_tensor.h b/tensorflow/compiler/jit/xla_tensor.h index 5ff2fb08f0..2334fd272b 100644 --- a/tensorflow/compiler/jit/xla_tensor.h +++ b/tensorflow/compiler/jit/xla_tensor.h @@ -64,9 +64,9 @@ class XlaTensor { return *shaped_buffer_; } // Mutates the TensorInfo to set the ShapedBuffer. - void set_shaped_buffer( - std::unique_ptr shaped_buffer) { - shaped_buffer_ = std::move(shaped_buffer); + void set_shaped_buffer(xla::ScopedShapedBuffer shaped_buffer) { + shaped_buffer_ = + xla::MakeUnique(std::move(shaped_buffer)); } // Some tensors on the device may have known values on the host. We use these diff --git a/tensorflow/compiler/xla/client/local_client.cc b/tensorflow/compiler/xla/client/local_client.cc index d951c44cb9..d0e945b70f 100644 --- a/tensorflow/compiler/xla/client/local_client.cc +++ b/tensorflow/compiler/xla/client/local_client.cc @@ -134,7 +134,7 @@ tensorflow::Status LocalExecutable::ValidateExecutionOptions( return Status::OK(); } -StatusOr> LocalExecutable::Run( +StatusOr LocalExecutable::Run( const tensorflow::gtl::ArraySlice arguments, ExecutableRunOptions run_options) { TF_RETURN_IF_ERROR( @@ -167,27 +167,26 @@ StatusOr> LocalExecutable::Run( return ExecuteAndDump(&service_options, arguments); } TF_ASSIGN_OR_RETURN( - std::unique_ptr result, + ShapedBuffer result, executable_->ExecuteOnStreamWrapper( &service_options, run_options.execution_profile(), arguments)); - return MakeUnique(std::move(*result), - run_options.allocator()); + return ScopedShapedBuffer(std::move(result), run_options.allocator()); } -StatusOr> LocalExecutable::ExecuteAndDump( +StatusOr LocalExecutable::ExecuteAndDump( const ServiceExecutableRunOptions* run_options, const tensorflow::gtl::ArraySlice arguments) { executable_->session_module()->set_execution_platform( backend_->platform()->Name()); TF_RETURN_IF_ERROR(RecordArguments(arguments, executable_->session_module())); TF_ASSIGN_OR_RETURN( - std::unique_ptr result, + ShapedBuffer result, executable_->ExecuteOnStream(run_options, arguments, /*hlo_execution_profile=*/nullptr)); - TF_RETURN_IF_ERROR(RecordResult(result.get(), executable_->session_module())); + TF_RETURN_IF_ERROR(RecordResult(&result, executable_->session_module())); TF_RETURN_IF_ERROR(executable_->DumpSessionModule()); - return ScopedShapedBuffer::MakeScoped(result.get(), run_options->allocator()); + return ScopedShapedBuffer(std::move(result), run_options->allocator()); } tensorflow::Status LocalExecutable::RecordArguments( @@ -281,9 +280,9 @@ StatusOr> LocalClient::Compile( updated_options)); } -StatusOr> -LocalClient::LiteralToShapedBuffer(const Literal& literal, int device_ordinal, - DeviceMemoryAllocator* allocator) { +StatusOr LocalClient::LiteralToShapedBuffer( + const Literal& literal, int device_ordinal, + DeviceMemoryAllocator* allocator) { if (allocator == nullptr) { allocator = backend().memory_allocator(); } @@ -293,7 +292,7 @@ LocalClient::LiteralToShapedBuffer(const Literal& literal, int device_ordinal, TF_ASSIGN_OR_RETURN(se::StreamExecutor * executor, backend().stream_executor(device_ordinal)); TF_RETURN_IF_ERROR(backend().transfer_manager()->TransferLiteralToDevice( - executor, literal, *scoped_buffer)); + executor, literal, scoped_buffer)); return std::move(scoped_buffer); } diff --git a/tensorflow/compiler/xla/client/local_client.h b/tensorflow/compiler/xla/client/local_client.h index 42812b936f..f306c520ed 100644 --- a/tensorflow/compiler/xla/client/local_client.h +++ b/tensorflow/compiler/xla/client/local_client.h @@ -38,7 +38,7 @@ class LocalExecutable { public: // Run the compiled computation with the given arguments and options and // return the result. - StatusOr> Run( + StatusOr Run( const tensorflow::gtl::ArraySlice arguments, ExecutableRunOptions run_options); @@ -73,7 +73,7 @@ class LocalExecutable { // Records the computation in a SessionModule proto with the arguments used to // invoke it, and the result. Enabled by flag: --tla_dump_executions_to. - StatusOr> ExecuteAndDump( + StatusOr ExecuteAndDump( const ServiceExecutableRunOptions* run_options, const tensorflow::gtl::ArraySlice arguments); @@ -136,7 +136,7 @@ class LocalClient : public Client { // ScopedShapedBuffer. If non-null the given memory allocator is used for // device memory allocation. If null, the default memory allocator for the // device is used. - StatusOr> LiteralToShapedBuffer( + StatusOr LiteralToShapedBuffer( const Literal& literal, int device_ordinal, DeviceMemoryAllocator* allocator = nullptr); diff --git a/tensorflow/compiler/xla/python/local_computation_builder.cc b/tensorflow/compiler/xla/python/local_computation_builder.cc index 2bacc6a914..24e17abbe0 100644 --- a/tensorflow/compiler/xla/python/local_computation_builder.cc +++ b/tensorflow/compiler/xla/python/local_computation_builder.cc @@ -89,17 +89,16 @@ StatusOr> TransferFromOutfeedLocalReplica( return client->TransferFromOutfeedLocal(shape, device_ordinal); } -LocalShapedBuffer::LocalShapedBuffer( - std::unique_ptr shaped_buffer) +LocalShapedBuffer::LocalShapedBuffer(ScopedShapedBuffer shaped_buffer) : shaped_buffer_(std::move(shaped_buffer)) {} -const std::unique_ptr& LocalShapedBuffer::shaped_buffer() - const { - return shaped_buffer_; +const ScopedShapedBuffer* LocalShapedBuffer::shaped_buffer() const { + return &shaped_buffer_; } -static StatusOr> ToBuffer( - LocalClient* client, int device_ordinal, const Literal& arg) { +static StatusOr ToBuffer(LocalClient* client, + int device_ordinal, + const Literal& arg) { return client->LiteralToShapedBuffer(arg, device_ordinal, client->backend().memory_allocator()); } @@ -109,14 +108,15 @@ LocalShapedBuffer* LocalShapedBuffer::FromLiteral( const Literal& argument, const tensorflow::gtl::optional& shape_with_layout) { LocalClient* client = GetOrCreateLocalClient(); - std::unique_ptr buf; - if (shape_with_layout) { - std::unique_ptr relaid = - argument.Relayout(shape_with_layout.value()); - buf = ToBuffer(client, /*device_ordinal=*/0, *relaid).ConsumeValueOrDie(); - } else { - buf = ToBuffer(client, /*device_ordinal=*/0, argument).ConsumeValueOrDie(); - } + ScopedShapedBuffer buf = [&] { + if (shape_with_layout) { + std::unique_ptr relaid = + argument.Relayout(shape_with_layout.value()); + return ToBuffer(client, /*device_ordinal=*/0, *relaid) + .ConsumeValueOrDie(); + } + return ToBuffer(client, /*device_ordinal=*/0, argument).ConsumeValueOrDie(); + }(); return new LocalShapedBuffer(std::move(buf)); } @@ -158,14 +158,14 @@ StatusOr> CompiledLocalComputation::Execute( << device_ordinal; // Transfer arguments in - std::vector> scoped_buffers; + std::vector scoped_buffers; scoped_buffers.reserve(arguments.size()); for (int i = 0; i < arguments.size(); ++i) { const Literal& argument = arguments[i]; const tensorflow::gtl::optional& shape_with_layout = shapes_with_layout[i]; - StatusOr> pushed; + StatusOr pushed; if (shape_with_layout) { std::unique_ptr relaid = argument.Relayout(shape_with_layout.value()); @@ -185,7 +185,7 @@ StatusOr> CompiledLocalComputation::Execute( std::vector argument_buffers; argument_buffers.reserve(scoped_buffers.size()); for (auto& buffer : scoped_buffers) { - argument_buffers.push_back(buffer.get()); + argument_buffers.push_back(&buffer); } DeviceAssignment device_assignment = @@ -202,7 +202,7 @@ StatusOr> CompiledLocalComputation::Execute( options.set_intra_op_thread_pool( client->backend().eigen_intra_op_thread_pool_device()); options.set_device_assignment(&device_assignment); - StatusOr> result_buffer_status = + StatusOr result_buffer_status = executable_->Run(argument_buffers, options); if (!result_buffer_status.ok()) { results[replica] = result_buffer_status.status(); @@ -210,8 +210,8 @@ StatusOr> CompiledLocalComputation::Execute( } // Transfer result out - results[replica] = - client->ShapedBufferToLiteral(*result_buffer_status.ValueOrDie()); + results[replica] = client->ShapedBufferToLiteral( + std::move(result_buffer_status).ValueOrDie()); }); } } @@ -236,7 +236,7 @@ LocalShapedBuffer* CompiledLocalComputation::ExecuteWithShapedBuffers( std::vector argument_buffers; argument_buffers.reserve(argument_handles.size()); for (auto& handle : argument_handles) { - argument_buffers.push_back(handle->shaped_buffer().get()); + argument_buffers.push_back(handle->shaped_buffer()); } // Execute @@ -245,7 +245,7 @@ LocalShapedBuffer* CompiledLocalComputation::ExecuteWithShapedBuffers( options.set_inter_op_thread_pool(client->backend().inter_op_thread_pool()); options.set_intra_op_thread_pool( client->backend().eigen_intra_op_thread_pool_device()); - std::unique_ptr result_buffer = + ScopedShapedBuffer result_buffer = executable_->Run(argument_buffers, options).ConsumeValueOrDie(); return new LocalShapedBuffer(std::move(result_buffer)); diff --git a/tensorflow/compiler/xla/python/local_computation_builder.h b/tensorflow/compiler/xla/python/local_computation_builder.h index 31046e60f1..e1048909ab 100644 --- a/tensorflow/compiler/xla/python/local_computation_builder.h +++ b/tensorflow/compiler/xla/python/local_computation_builder.h @@ -62,12 +62,12 @@ class LocalShapedBuffer { static LocalShapedBuffer* FromLiteral( const Literal& argument, const tensorflow::gtl::optional& shape_with_layout); - LocalShapedBuffer(std::unique_ptr shaped_buffer); - const std::unique_ptr& shaped_buffer() const; + LocalShapedBuffer(ScopedShapedBuffer shaped_buffer); + const ScopedShapedBuffer* shaped_buffer() const; std::unique_ptr ToLiteral() const; private: - std::unique_ptr shaped_buffer_; + ScopedShapedBuffer shaped_buffer_; }; // Wraps a LocalExecutable produced by compiling a diff --git a/tensorflow/compiler/xla/service/allocation_tracker.cc b/tensorflow/compiler/xla/service/allocation_tracker.cc index 359582a78c..6bf65825cd 100644 --- a/tensorflow/compiler/xla/service/allocation_tracker.cc +++ b/tensorflow/compiler/xla/service/allocation_tracker.cc @@ -31,52 +31,51 @@ limitations under the License. namespace xla { StatusOr AllocationTracker::Register( - std::unique_ptr shaped_buffer, const string& tag) { + ShapedBuffer shaped_buffer, const string& tag) { tensorflow::mutex_lock lock(mutex_); VLOG(2) << "Register"; - std::vector> replicated_buffers; + std::vector replicated_buffers; replicated_buffers.emplace_back(std::move(shaped_buffer)); return RegisterInternal(std::move(replicated_buffers), tag); } StatusOr AllocationTracker::RegisterReplicatedBuffers( - std::vector> replicated_buffers, - const string& tag) { + std::vector replicated_buffers, const string& tag) { tensorflow::mutex_lock lock(mutex_); VLOG(2) << "RegisterReplicatedBuffers"; return RegisterInternal(std::move(replicated_buffers), tag); } StatusOr AllocationTracker::RegisterInternal( - std::vector> replicated_buffers, - const string& tag) { + std::vector replicated_buffers, const string& tag) { VLOG(2) << "RegisterInternal(" << "tag: \"" << tag << "\" with " << replicated_buffers.size() << " shaped_buffers."; for (const auto& shaped_buffer : replicated_buffers) { - VLOG(2) << "shaped_buffer:" << *shaped_buffer; - if (shaped_buffer->platform() != backend_->platform()) { + VLOG(2) << "shaped_buffer:" << shaped_buffer; + if (shaped_buffer.platform() != backend_->platform()) { return InvalidArgument( "AllocationTracker for platform %s cannot register buffer from " "platform %s", backend_->platform()->Name().c_str(), - shaped_buffer->platform()->Name().c_str()); + shaped_buffer.platform()->Name().c_str()); } } int64 handle = next_handle_++; for (auto& shaped_buffer : replicated_buffers) { std::vector shape_indices; - ShapeUtil::ForEachSubshape(shaped_buffer->on_device_shape(), + ShapeUtil::ForEachSubshape(shaped_buffer.on_device_shape(), [this, &shape_indices](const Shape& /*subshape*/, const ShapeIndex& index) { shape_indices.push_back(index); }); for (const ShapeIndex& index : shape_indices) { - AddAllocationOrIncrementRefCount(shaped_buffer->buffer(index), - shaped_buffer->device_ordinal()); + AddAllocationOrIncrementRefCount(shaped_buffer.buffer(index), + shaped_buffer.device_ordinal()); } - handle_to_shaped_buffers_[handle].emplace_back(std::move(shaped_buffer)); + handle_to_shaped_buffers_[handle].emplace_back( + MakeUnique(std::move(shaped_buffer))); } GlobalDataHandle result; @@ -146,13 +145,13 @@ StatusOr> AllocationTracker::DeconstructTuple( for (int i = 0; i < ShapeUtil::TupleElementCount(shaped_buffer->on_device_shape()); ++i) { - auto element_buffer = MakeUnique( + auto element_buffer = ShapedBuffer( ShapeUtil::GetTupleElementShape(shaped_buffer->on_host_shape(), i), ShapeUtil::GetTupleElementShape(shaped_buffer->on_device_shape(), i), shaped_buffer->platform(), shaped_buffer->device_ordinal()); - element_buffer->set_buffer(shaped_buffer->buffer(/*index=*/{i}), - /*index=*/{}); - std::vector> replicated_buffers; + element_buffer.set_buffer(shaped_buffer->buffer(/*index=*/{i}), + /*index=*/{}); + std::vector replicated_buffers; replicated_buffers.emplace_back(std::move(element_buffer)); TF_ASSIGN_OR_RETURN( GlobalDataHandle element_handle, diff --git a/tensorflow/compiler/xla/service/allocation_tracker.h b/tensorflow/compiler/xla/service/allocation_tracker.h index 60e93358ef..2bfcd53712 100644 --- a/tensorflow/compiler/xla/service/allocation_tracker.h +++ b/tensorflow/compiler/xla/service/allocation_tracker.h @@ -45,14 +45,13 @@ class AllocationTracker { // Registers a shaped buffer of device memory, and returns a corresponding // handle that can be used for talking to XLA clients. The given shaped buffer // will be treated as the buffer corresponding to the only replica. - StatusOr Register( - std::unique_ptr shaped_buffer, const string& tag); + StatusOr Register(ShapedBuffer shaped_buffer, + const string& tag); // Registers a vector of shaped buffers of device memory, one per replica, and // returns a corresponding handle that can be used for talking to XLA clients. StatusOr RegisterReplicatedBuffers( - std::vector> replicated_buffers, - const string& tag); + std::vector replicated_buffers, const string& tag); // Unregister the allocation for the given data handle. Status Unregister(const GlobalDataHandle& data); @@ -95,8 +94,8 @@ class AllocationTracker { // Internal helper which registers a vector of shaped buffers, one per // replica. StatusOr RegisterInternal( - std::vector> replicated_buffers, - const string& tag) EXCLUSIVE_LOCKS_REQUIRED(mutex_); + std::vector replicated_buffers, const string& tag) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); // Resets the shaped buffers corresponding to the given handle. Status Reset(const GlobalDataHandle& data) EXCLUSIVE_LOCKS_REQUIRED(mutex_); @@ -132,6 +131,9 @@ class AllocationTracker { // A map from data handle to a vector of shaped buffers that represent the // buffers for different replicas. + // + // The ShapedBuffers in this map's vectors need to be unique_ptrs, because our + // public API returns pointers to them. tensorflow::gtl::FlatMap>> handle_to_shaped_buffers_ GUARDED_BY(mutex_); diff --git a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc index aee62a4935..97e550abe4 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc @@ -243,18 +243,18 @@ static Status DeallocateTempBuffers( return Status::OK(); } -StatusOr> CpuExecutable::CreateResultShapedBuffer( +StatusOr CpuExecutable::CreateResultShapedBuffer( const ServiceExecutableRunOptions* run_options, tensorflow::gtl::ArraySlice allocated_buffers, std::vector* buffers_in_result) { se::Stream* stream = run_options->stream(); - auto result_buffer = MakeUnique( + ShapedBuffer result_buffer( /*on_host_shape=*/result_shape(), /*on_device_shape=*/result_shape(), stream->parent()->platform(), stream->parent()->device_ordinal()); // Copy DeviceMemoryBase values which contain the array(s) of the result into // the respective location in ShapedBuffer which is returned to the caller. - TF_RETURN_IF_ERROR(result_buffer->buffers().ForEachMutableElementWithStatus( + TF_RETURN_IF_ERROR(result_buffer.buffers().ForEachMutableElementWithStatus( [&](const ShapeIndex& index, se::DeviceMemoryBase* device_memory) { const auto& sources = this->GetRootPointsToSet().element(index); // The points to set is unambiguous so the set should be a @@ -281,7 +281,7 @@ StatusOr> CpuExecutable::CreateResultShapedBuffer( return std::move(result_buffer); } -StatusOr> CpuExecutable::ExecuteOnStream( +StatusOr CpuExecutable::ExecuteOnStream( const ServiceExecutableRunOptions* run_options, tensorflow::gtl::ArraySlice arguments, HloExecutionProfile* hlo_execution_profile) { @@ -300,7 +300,7 @@ StatusOr> CpuExecutable::ExecuteOnStream( std::vector buffers_in_result(assignment_->Allocations().size(), false); TF_ASSIGN_OR_RETURN( - std::unique_ptr result_buffer, + ShapedBuffer result_buffer, CreateResultShapedBuffer(run_options, buffers, &buffers_in_result)); // Free all buffers not in the result. @@ -310,7 +310,7 @@ StatusOr> CpuExecutable::ExecuteOnStream( return std::move(result_buffer); } -StatusOr> CpuExecutable::ExecuteAsyncOnStream( +StatusOr CpuExecutable::ExecuteAsyncOnStream( const ServiceExecutableRunOptions* run_options, tensorflow::gtl::ArraySlice arguments) { if (hlo_profiling_enabled()) { @@ -330,7 +330,7 @@ StatusOr> CpuExecutable::ExecuteAsyncOnStream( std::vector buffers_in_result(assignment_->Allocations().size(), false); TF_ASSIGN_OR_RETURN( - std::unique_ptr result_buffer, + ShapedBuffer result_buffer, CreateResultShapedBuffer(run_options, buffers, &buffers_in_result)); LogLiveAddresses(buffers, buffers_in_result); diff --git a/tensorflow/compiler/xla/service/cpu/cpu_executable.h b/tensorflow/compiler/xla/service/cpu/cpu_executable.h index c3c2820c26..06b6943cb5 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_executable.h +++ b/tensorflow/compiler/xla/service/cpu/cpu_executable.h @@ -55,12 +55,12 @@ class CpuExecutable : public Executable { std::unique_ptr hlo_profile_index_map); ~CpuExecutable() override {} - StatusOr> ExecuteOnStream( + StatusOr ExecuteOnStream( const ServiceExecutableRunOptions* run_options, tensorflow::gtl::ArraySlice arguments, HloExecutionProfile* hlo_execution_profile) override; - StatusOr> ExecuteAsyncOnStream( + StatusOr ExecuteAsyncOnStream( const ServiceExecutableRunOptions* run_options, tensorflow::gtl::ArraySlice arguments) override; @@ -102,13 +102,13 @@ class CpuExecutable : public Executable { tensorflow::gtl::ArraySlice buffers, HloExecutionProfile* hlo_execution_profile); - // Create a ShapedBuffer for holding the result of the computation. The + // Creates a ShapedBuffer for holding the result of the computation. The // addresses (DeviceMemoryBases) are set according to buffer assignment. // 'buffers_in_result' should point to a vector of the same size as // 'allocated_buffers'. An element in buffers_in_result is set to true if the // corresponding buffer is live out of the computation (and thus contained in // the returned ShapedBuffer). - StatusOr> CreateResultShapedBuffer( + StatusOr CreateResultShapedBuffer( const ServiceExecutableRunOptions* run_options, tensorflow::gtl::ArraySlice allocated_buffers, std::vector* buffers_in_result); diff --git a/tensorflow/compiler/xla/service/cpu/parallel_cpu_executable.cc b/tensorflow/compiler/xla/service/cpu/parallel_cpu_executable.cc index 2d0f1d0be5..a2bd4fa195 100644 --- a/tensorflow/compiler/xla/service/cpu/parallel_cpu_executable.cc +++ b/tensorflow/compiler/xla/service/cpu/parallel_cpu_executable.cc @@ -447,7 +447,7 @@ Status ParallelCpuExecutable::ExecuteComputeFunctions( return Status::OK(); } -StatusOr> ParallelCpuExecutable::ExecuteOnStream( +StatusOr ParallelCpuExecutable::ExecuteOnStream( const ServiceExecutableRunOptions* run_options, tensorflow::gtl::ArraySlice arguments, HloExecutionProfile* hlo_execution_profile) { @@ -459,7 +459,7 @@ StatusOr> ParallelCpuExecutable::ExecuteOnStream( DeviceMemoryAllocator* memory_allocator = run_options->allocator(); std::vector buffers(assignment_->Allocations().size()); - auto result_buffer = MakeUnique( + ShapedBuffer result_buffer( /*on_host_shape=*/result_shape(), /*on_device_shape=*/result_shape(), stream->parent()->platform(), stream->parent()->device_ordinal()); @@ -472,7 +472,7 @@ StatusOr> ParallelCpuExecutable::ExecuteOnStream( // Copy DeviceMemoryBase values which into the respective location in // ShapedBuffer which is returned to the caller. std::vector buffers_in_result(assignment_->Allocations().size(), false); - TF_RETURN_IF_ERROR(result_buffer->buffers().ForEachMutableElementWithStatus( + TF_RETURN_IF_ERROR(result_buffer.buffers().ForEachMutableElementWithStatus( [&](const ShapeIndex& index, se::DeviceMemoryBase* device_memory) { const auto& sources = this->GetRootPointsToSet().element(index); @@ -511,8 +511,7 @@ StatusOr> ParallelCpuExecutable::ExecuteOnStream( return std::move(result_buffer); } -StatusOr> -ParallelCpuExecutable::ExecuteAsyncOnStream( +StatusOr ParallelCpuExecutable::ExecuteAsyncOnStream( const ServiceExecutableRunOptions* run_options, tensorflow::gtl::ArraySlice arguments) { // TODO(b/30671675): Implement asynchronous execution mode. diff --git a/tensorflow/compiler/xla/service/cpu/parallel_cpu_executable.h b/tensorflow/compiler/xla/service/cpu/parallel_cpu_executable.h index d87ba57a1e..5ce84fa996 100644 --- a/tensorflow/compiler/xla/service/cpu/parallel_cpu_executable.h +++ b/tensorflow/compiler/xla/service/cpu/parallel_cpu_executable.h @@ -59,12 +59,12 @@ class ParallelCpuExecutable : public Executable { std::unique_ptr hlo_profile_index_map); ~ParallelCpuExecutable() override {} - StatusOr> ExecuteOnStream( + StatusOr ExecuteOnStream( const ServiceExecutableRunOptions* run_options, tensorflow::gtl::ArraySlice arguments, HloExecutionProfile* hlo_execution_profile) override; - StatusOr> ExecuteAsyncOnStream( + StatusOr ExecuteAsyncOnStream( const ServiceExecutableRunOptions* run_options, tensorflow::gtl::ArraySlice arguments) override; diff --git a/tensorflow/compiler/xla/service/executable.cc b/tensorflow/compiler/xla/service/executable.cc index caa46686be..b097ef79cc 100644 --- a/tensorflow/compiler/xla/service/executable.cc +++ b/tensorflow/compiler/xla/service/executable.cc @@ -29,18 +29,19 @@ using tensorflow::gtl::ArraySlice; namespace xla { -StatusOr>> -Executable::ExecuteOnStreams( +StatusOr> Executable::ExecuteOnStreams( ArraySlice run_options, ArraySlice> arguments) { TF_RET_CHECK(run_options.size() == arguments.size()); - std::vector> return_values(run_options.size()); + std::vector return_values; + return_values.reserve(run_options.size()); if (run_options.size() == 1) { - TF_ASSIGN_OR_RETURN(return_values[0], + TF_ASSIGN_OR_RETURN(auto rv, ExecuteOnStream(&run_options[0], arguments[0], /*hlo_execution_profile=*/nullptr)); + return_values.push_back(std::move(rv)); return std::move(return_values); } @@ -48,8 +49,9 @@ Executable::ExecuteOnStreams( // We cannot BlockHostUntilDone() on the already-launched executions in case // of error, since if the executions communicate, the initially launched // executions may never complete if not all executions are running. - TF_ASSIGN_OR_RETURN(return_values[i], + TF_ASSIGN_OR_RETURN(auto rv, ExecuteAsyncOnStream(&run_options[i], arguments[i])); + return_values.push_back(std::move(rv)); } for (const auto& options : run_options) { TF_RET_CHECK(options.stream() != nullptr); @@ -58,7 +60,7 @@ Executable::ExecuteOnStreams( return std::move(return_values); } -StatusOr> Executable::ExecuteOnStreamWrapper( +StatusOr Executable::ExecuteOnStreamWrapper( const ServiceExecutableRunOptions* run_options, ExecutionProfile* profile, ArraySlice arguments) { se::Stream* stream = run_options->stream(); @@ -78,7 +80,7 @@ StatusOr> Executable::ExecuteOnStreamWrapper( &hlo_profile_index_map()) : nullptr; - StatusOr> return_value = + StatusOr return_value = ExecuteOnStream(run_options, arguments, profile_ptr.get()); TF_RETURN_IF_ERROR(return_value.status()); diff --git a/tensorflow/compiler/xla/service/executable.h b/tensorflow/compiler/xla/service/executable.h index 6f4cd99767..9c725f21d8 100644 --- a/tensorflow/compiler/xla/service/executable.h +++ b/tensorflow/compiler/xla/service/executable.h @@ -62,14 +62,14 @@ class Executable { // enabled. // // Returns a shaped buffer containing the result of the computation. - virtual StatusOr> ExecuteOnStream( + virtual StatusOr ExecuteOnStream( const ServiceExecutableRunOptions* run_options, tensorflow::gtl::ArraySlice arguments, HloExecutionProfile* hlo_execution_profile) = 0; // Same as ExecuteOnStream(), but this call is non-blocking and returns as // soon as all of the operations are enqueued for launch on the stream. - virtual StatusOr> ExecuteAsyncOnStream( + virtual StatusOr ExecuteAsyncOnStream( const ServiceExecutableRunOptions* run_options, tensorflow::gtl::ArraySlice arguments) = 0; @@ -77,7 +77,7 @@ class Executable { // streams. arguments[i] contains the arguments to the execution on // run_options[i]->stream() and the returned value is at index i of the // returned vector. - virtual StatusOr>> ExecuteOnStreams( + virtual StatusOr> ExecuteOnStreams( tensorflow::gtl::ArraySlice run_options, tensorflow::gtl::ArraySlice< @@ -97,7 +97,7 @@ class Executable { // Convenience wrapper for calling Executable::ExecuteOnStream. Sets up a // timer for the execution, sets up HLO profiling if enabled, and fills in the // given ExecutionProfile if non-null. - StatusOr> ExecuteOnStreamWrapper( + StatusOr ExecuteOnStreamWrapper( const ServiceExecutableRunOptions* run_options, ExecutionProfile* profile, tensorflow::gtl::ArraySlice arguments); diff --git a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc index 5676d4de8e..62ce15bc59 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc @@ -250,7 +250,7 @@ Status GpuExecutable::ExecuteThunks( return Status::OK(); } -StatusOr> GpuExecutable::ExecuteOnStream( +StatusOr GpuExecutable::ExecuteOnStream( const ServiceExecutableRunOptions* run_options, tensorflow::gtl::ArraySlice arguments, HloExecutionProfile* hlo_execution_profile) { @@ -297,13 +297,13 @@ StatusOr> GpuExecutable::ExecuteOnStream( HloInstruction* root = hlo_module_->entry_computation()->root_instruction(); auto device_ordinal = executor->device_ordinal(); - auto shaped_buffer = MakeUnique( - root->shape(), root->shape(), executor->platform(), device_ordinal); + auto shaped_buffer = ShapedBuffer(root->shape(), root->shape(), + executor->platform(), device_ordinal); // Copy DeviceMemoryBase values which contain the array(s) of the result into // the respective location in ShapedBuffer. std::set buffers_in_result; - TF_RETURN_IF_ERROR(shaped_buffer->buffers().ForEachMutableElementWithStatus( + TF_RETURN_IF_ERROR(shaped_buffer.buffers().ForEachMutableElementWithStatus( [&buffer_allocations, &buffers_in_result, &shaped_buffer, this]( const ShapeIndex& index, se::DeviceMemoryBase* device_memory) { const auto& sources = this->GetRootPointsToSet().element(index); @@ -335,7 +335,7 @@ StatusOr> GpuExecutable::ExecuteOnStream( return std::move(shaped_buffer); } -StatusOr> GpuExecutable::ExecuteAsyncOnStream( +StatusOr GpuExecutable::ExecuteAsyncOnStream( const ServiceExecutableRunOptions* run_options, tensorflow::gtl::ArraySlice arguments) { // TODO(b/30671675): Implement asynchronous execution mode. diff --git a/tensorflow/compiler/xla/service/gpu/gpu_executable.h b/tensorflow/compiler/xla/service/gpu/gpu_executable.h index dcb3991f41..361bc30b2f 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_executable.h +++ b/tensorflow/compiler/xla/service/gpu/gpu_executable.h @@ -74,12 +74,12 @@ class GpuExecutable : public Executable { // ExecuteOnStream will fail if the compute capability of the stream doesn't // match the compute capability passed to this object's constructor. - StatusOr> ExecuteOnStream( + StatusOr ExecuteOnStream( const ServiceExecutableRunOptions* run_options, tensorflow::gtl::ArraySlice arguments, HloExecutionProfile* hlo_execution_profile) override; - StatusOr> ExecuteAsyncOnStream( + StatusOr ExecuteAsyncOnStream( const ServiceExecutableRunOptions* run_options, tensorflow::gtl::ArraySlice arguments) override; diff --git a/tensorflow/compiler/xla/service/hlo_runner.cc b/tensorflow/compiler/xla/service/hlo_runner.cc index 171477299e..df5ffd0b7d 100644 --- a/tensorflow/compiler/xla/service/hlo_runner.cc +++ b/tensorflow/compiler/xla/service/hlo_runner.cc @@ -107,33 +107,35 @@ StatusOr> HloRunner::Execute( const ExecutableRunOptions& run_options = service_run_options.run_options(); // Copy arguments to device. - std::vector> argument_buffers; - std::vector argument_buffer_ptrs; + std::vector argument_buffers; for (Literal* argument : arguments) { TF_ASSIGN_OR_RETURN( - std::unique_ptr argument_buffer, + ScopedShapedBuffer argument_buffer, backend().transfer_manager()->AllocateScopedShapedBuffer( argument->shape(), run_options.allocator(), run_options.device_ordinal())); TF_RETURN_IF_ERROR(backend().transfer_manager()->TransferLiteralToDevice( - stream.parent(), *argument, *argument_buffer)); + stream.parent(), *argument, argument_buffer)); argument_buffers.push_back(std::move(argument_buffer)); - argument_buffer_ptrs.push_back(argument_buffers.back().get()); + } + + std::vector argument_buffer_ptrs; + argument_buffer_ptrs.reserve(argument_buffers.size()); + for (const auto& buf : argument_buffers) { + argument_buffer_ptrs.push_back(&buf); } TF_ASSIGN_OR_RETURN( - std::unique_ptr result, + ShapedBuffer result, executable->ExecuteOnStreamWrapper( &service_run_options, /*profile=*/nullptr, argument_buffer_ptrs)); // Create a ScopedShapedBuffer of the result to manage deallocation. This will // deallocate all the device memory when it goes out of scope. - TF_ASSIGN_OR_RETURN( - std::unique_ptr scoped_result, - ScopedShapedBuffer::MakeScoped(result.get(), run_options.allocator())); + ScopedShapedBuffer scoped_result(std::move(result), run_options.allocator()); auto result_literal = backend().transfer_manager()->TransferLiteralFromDevice( - stream.parent(), *scoped_result); + stream.parent(), scoped_result); if (result_literal.ok()) { VLOG(4) << "Executed binary and got result: " << result_literal.ValueOrDie()->ToString(); @@ -155,7 +157,13 @@ StatusOr>> HloRunner::ExecuteReplicated( backend().computation_placer()->AssignDevices(options.num_replicas, 1)); std::vector> streams; std::vector service_run_options; - std::vector> argument_buffers; + + std::vector argument_buffers; + // This reserve() call is necessary for correctness, because + // argument_buffer_ptrs contains pointers into the elements of + // argument_buffers. + argument_buffers.reserve(options.num_replicas * options.arguments.size()); + // Plus one so we can safely get &argument_buffer_ptrs[0] in case there are // no arguments. std::vector argument_buffer_ptrs( @@ -175,13 +183,13 @@ StatusOr>> HloRunner::ExecuteReplicated( // Copy arguments to device. for (const Literal* argument : options.arguments) { TF_ASSIGN_OR_RETURN( - std::unique_ptr argument_buffer, + ScopedShapedBuffer argument_buffer, backend().transfer_manager()->AllocateScopedShapedBuffer( argument->shape(), backend().memory_allocator(), device)); TF_RETURN_IF_ERROR(backend().transfer_manager()->TransferLiteralToDevice( - executor, *argument, *argument_buffer)); + executor, *argument, argument_buffer)); argument_buffers.push_back(std::move(argument_buffer)); - argument_buffer_ptrs[index++] = argument_buffers.back().get(); + argument_buffer_ptrs[index++] = &argument_buffers.back(); } argument_buffer_slices.emplace_back( &argument_buffer_ptrs[index - options.arguments.size()], @@ -240,19 +248,18 @@ StatusOr>> HloRunner::ExecuteReplicated( } LOG(INFO) << "Replicated execution started"; - TF_ASSIGN_OR_RETURN(std::vector> results, + TF_ASSIGN_OR_RETURN(std::vector results, executable->ExecuteOnStreams(service_run_options, argument_buffer_slices)); LOG(INFO) << "Replicated execution terminated"; std::vector> exec_results; for (int64 i = 0; i < options.num_replicas; ++i) { - TF_ASSIGN_OR_RETURN(std::unique_ptr result, - ScopedShapedBuffer::MakeScoped( - results[i].get(), backend().memory_allocator())); + ScopedShapedBuffer result(std::move(results[i]), + backend().memory_allocator()); TF_ASSIGN_OR_RETURN(std::unique_ptr literal, backend().transfer_manager()->TransferLiteralFromDevice( - streams[i]->parent(), *result)); + streams[i]->parent(), result)); exec_results.push_back(std::move(literal)); } return std::move(exec_results); diff --git a/tensorflow/compiler/xla/service/interpreter/executable.cc b/tensorflow/compiler/xla/service/interpreter/executable.cc index acfa79ea75..6553000336 100644 --- a/tensorflow/compiler/xla/service/interpreter/executable.cc +++ b/tensorflow/compiler/xla/service/interpreter/executable.cc @@ -45,7 +45,7 @@ InterpreterExecutable::InterpreterExecutable( InterpreterExecutable::~InterpreterExecutable() {} -StatusOr> InterpreterExecutable::ExecuteOnStream( +StatusOr InterpreterExecutable::ExecuteOnStream( const ServiceExecutableRunOptions* run_options, tensorflow::gtl::ArraySlice arguments, HloExecutionProfile* hlo_execution_profile) { @@ -88,12 +88,12 @@ StatusOr> InterpreterExecutable::ExecuteOnStream( evaluator.Evaluate>(*computation, arg_literals)); // Transform the result literal back into a ShapedBuffer. - TF_ASSIGN_OR_RETURN(std::unique_ptr result, + TF_ASSIGN_OR_RETURN(ShapedBuffer result, transfer_manager->AllocateShapedBuffer( result_literal->shape(), run_options->allocator(), executor->device_ordinal())); TF_RETURN_IF_ERROR(transfer_manager->TransferLiteralToDevice( - executor, *result_literal, *result)); + executor, *result_literal, result)); uint64 end_micros = tensorflow::Env::Default()->NowMicros(); @@ -106,8 +106,7 @@ StatusOr> InterpreterExecutable::ExecuteOnStream( return std::move(result); } -StatusOr> -InterpreterExecutable::ExecuteAsyncOnStream( +StatusOr InterpreterExecutable::ExecuteAsyncOnStream( const ServiceExecutableRunOptions* run_options, tensorflow::gtl::ArraySlice arguments) { return tensorflow::errors::Unimplemented( diff --git a/tensorflow/compiler/xla/service/interpreter/executable.h b/tensorflow/compiler/xla/service/interpreter/executable.h index 410110a1ad..c825a9a368 100644 --- a/tensorflow/compiler/xla/service/interpreter/executable.h +++ b/tensorflow/compiler/xla/service/interpreter/executable.h @@ -43,12 +43,12 @@ class InterpreterExecutable : public Executable { InterpreterExecutable(std::unique_ptr hlo_module); ~InterpreterExecutable() override; - StatusOr> ExecuteOnStream( + StatusOr ExecuteOnStream( const ServiceExecutableRunOptions* run_options, tensorflow::gtl::ArraySlice arguments, HloExecutionProfile* hlo_execution_profile) override; - StatusOr> ExecuteAsyncOnStream( + StatusOr ExecuteAsyncOnStream( const ServiceExecutableRunOptions* run_options, tensorflow::gtl::ArraySlice arguments) override; diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc index 2df59c3556..39f3aefdf8 100644 --- a/tensorflow/compiler/xla/service/service.cc +++ b/tensorflow/compiler/xla/service/service.cc @@ -550,7 +550,7 @@ Service::ExecuteParallelAndRegisterResult( // Stream executors for the replicas of the current computation. TF_ASSIGN_OR_RETURN(auto replicas, Replicas(*backend, device_handles[i])); CHECK_EQ(replicas.size(), arguments[i].size()); - std::vector> result_buffers; + std::vector result_buffers; for (int64 replica = 0; replica < replicas.size(); ++replica) { TF_ASSIGN_OR_RETURN(Pool::SmartPtr stream, backend->BorrowStream(replicas[replica])); @@ -582,7 +582,7 @@ Service::ExecuteParallelAndRegisterResult( backend->StreamBorrower()); // Asynchronously launch the computation. - TF_ASSIGN_OR_RETURN(std::unique_ptr result, + TF_ASSIGN_OR_RETURN(ShapedBuffer result, executables[i]->ExecuteAsyncOnStream( &run_options, arguments[i][replica])); @@ -1234,7 +1234,7 @@ tensorflow::Status Service::ExecuteAsync(const ExecuteAsyncRequest* arg, streams.push_back(std::move(stream)); } - std::vector> result_buffers; + std::vector result_buffers; for (size_t i = 0; i < streams.size(); ++i) { const auto& stream = streams[i]; ExecutableRunOptions options; @@ -1247,7 +1247,7 @@ tensorflow::Status Service::ExecuteAsync(const ExecuteAsyncRequest* arg, ServiceExecutableRunOptions service_options( options, execute_backend_->StreamBorrower()); - TF_ASSIGN_OR_RETURN(std::unique_ptr this_result_buffer, + TF_ASSIGN_OR_RETURN(ShapedBuffer this_result_buffer, executable->ExecuteAsyncOnStream( &service_options, replicated_arguments[i])); @@ -1347,16 +1347,16 @@ tensorflow::Status Service::TransferToServer(const TransferToServerRequest* arg, } // Allocate memory in each replica and transfer the data to all replicas. - std::vector> replicated_buffers; + std::vector replicated_buffers; for (se::StreamExecutor* executor : replicas) { TF_ASSIGN_OR_RETURN( - std::unique_ptr shaped_buffer, + ShapedBuffer shaped_buffer, execute_backend_->transfer_manager()->AllocateShapedBuffer( shape, execute_backend_->memory_allocator(), executor->device_ordinal())); TF_RETURN_IF_ERROR( execute_backend_->transfer_manager()->TransferLiteralToDevice( - executor, *literal, *shaped_buffer)); + executor, *literal, shaped_buffer)); replicated_buffers.emplace_back(std::move(shaped_buffer)); } TF_ASSIGN_OR_RETURN(*result->mutable_data(), diff --git a/tensorflow/compiler/xla/service/shaped_buffer.cc b/tensorflow/compiler/xla/service/shaped_buffer.cc index 10a2aa2b30..0b5a383f6f 100644 --- a/tensorflow/compiler/xla/service/shaped_buffer.cc +++ b/tensorflow/compiler/xla/service/shaped_buffer.cc @@ -66,6 +66,8 @@ ShapedBuffer& ShapedBuffer::operator=(ShapedBuffer&& s) { return *this; } +ShapedBuffer::~ShapedBuffer() {} + void ShapedBuffer::clear() { for (auto& pair : buffers_) { // A default constructed DeviceMemoryBase is a null pointer. @@ -102,18 +104,6 @@ std::ostream& operator<<(std::ostream& out, const ShapedBuffer& buffer) { return out; } -/* static */ -StatusOr> ScopedShapedBuffer::MakeScoped( - ShapedBuffer* shaped_buffer, DeviceMemoryAllocator* allocator) { - auto scoped_buffer = WrapUnique(new ScopedShapedBuffer( - shaped_buffer->on_host_shape(), shaped_buffer->on_device_shape(), - allocator, shaped_buffer->device_ordinal())); - scoped_buffer->buffers_ = shaped_buffer->buffers(); - shaped_buffer->clear(); - - return std::move(scoped_buffer); -} - ScopedShapedBuffer::ScopedShapedBuffer(const Shape& on_host_shape, const Shape& on_device_shape, DeviceMemoryAllocator* allocator, @@ -126,7 +116,25 @@ ScopedShapedBuffer::ScopedShapedBuffer(ShapedBuffer shaped_buffer, DeviceMemoryAllocator* allocator) : ShapedBuffer(std::move(shaped_buffer)), allocator_(allocator) {} +ScopedShapedBuffer::ScopedShapedBuffer(ScopedShapedBuffer&& s) + : ShapedBuffer(std::move(s)), allocator_(s.allocator_) { + // Null out s.allocator_ so it doesn't try to free anything in its destructor. + s.allocator_ = nullptr; +} + +ScopedShapedBuffer& ScopedShapedBuffer::operator=(ScopedShapedBuffer&& s) { + *static_cast(this) = std::move(static_cast(s)); + allocator_ = s.allocator_; + // Null out s.allocator_ so it doesn't try to free anything in its destructor. + s.allocator_ = nullptr; + return *this; +} + ScopedShapedBuffer::~ScopedShapedBuffer() { + // allocator_ will be null if we were moved-from. + if (allocator_ == nullptr) { + return; + } // Deallocate all non-null buffers. A buffer may appear in more than one spot // in the shape (eg, a tuple with a repeated element) so keep track of what // has been deallocated. @@ -142,8 +150,8 @@ ScopedShapedBuffer::~ScopedShapedBuffer() { } } -std::unique_ptr ScopedShapedBuffer::release() { - auto shaped_buffer = MakeUnique(std::move(*this)); +ShapedBuffer ScopedShapedBuffer::release() { + ShapedBuffer shaped_buffer(std::move(*this)); buffers_ = ShapeTree(); return shaped_buffer; } diff --git a/tensorflow/compiler/xla/service/shaped_buffer.h b/tensorflow/compiler/xla/service/shaped_buffer.h index 62ba8f2734..f1b0527474 100644 --- a/tensorflow/compiler/xla/service/shaped_buffer.h +++ b/tensorflow/compiler/xla/service/shaped_buffer.h @@ -43,6 +43,14 @@ class ShapedBuffer { ShapedBuffer(const Shape& on_host_shape, const Shape& on_device_shape, const se::Platform* platform, int device_ordinal); + // Movable, but not copyable. + ShapedBuffer(ShapedBuffer&& s); + ShapedBuffer& operator=(ShapedBuffer&&); + ShapedBuffer(const ShapedBuffer&) = delete; + ShapedBuffer& operator=(const ShapedBuffer&) = delete; + + virtual ~ShapedBuffer(); + // Returns the shape of the on-host representation of the data held by this // ShapedBuffer. const Shape& on_host_shape() const { return on_host_shape_; } @@ -80,13 +88,7 @@ class ShapedBuffer { string ToString() const; - ShapedBuffer(ShapedBuffer&& s); - ShapedBuffer& operator=(ShapedBuffer&&); - protected: - ShapedBuffer(const ShapedBuffer&) = delete; - ShapedBuffer& operator=(const ShapedBuffer&) = delete; - // The shape of the data when represented on the host. Shape on_host_shape_; @@ -108,41 +110,45 @@ std::ostream& operator<<(std::ostream& out, const ShapedBuffer& buffer); // ShapedBuffer derived class which allocates all internal buffers on // construction and deallocates the memory when the object is // destructed. +// +// TODO(timshen): Remove inheritance between ScopedShapedBuffer and +// ShapedBuffer. There should never be a need to consider a ScopedShapedBuffer +// as a ShapedBuffer, because in that case we should just be able to pass around +// our ShapeTree. Inheritance only adds complexity. See +// discussion in cl/192849370. class ScopedShapedBuffer : public ShapedBuffer { public: - // Takes a ShapedBuffer and returns a ScopedShapedBuffer which manages the - // deallocation of the device memory held in the shaped buffer. All device - // memory pointers in the given ShapedBuffer are set to null. - static StatusOr> MakeScoped( - ShapedBuffer* shaped_buffer, DeviceMemoryAllocator* allocator); - - // Create a ScopedShapedBuffer with null DeviceMemoryBases at each index. - ScopedShapedBuffer(const Shape& on_host_shape, const Shape& on_device_shape, - DeviceMemoryAllocator* allocator, int device_ordinal); + // Creates a ScopedShapedBuffer with null DeviceMemoryBases at each index. + explicit ScopedShapedBuffer(const Shape& on_host_shape, + const Shape& on_device_shape, + DeviceMemoryAllocator* allocator, + int device_ordinal); // Create a ScopedShapedBuffer by taking over the memory from the incoming // ShapedBuffer. - ScopedShapedBuffer(ShapedBuffer shaped_buffer, - DeviceMemoryAllocator* allocator); + explicit ScopedShapedBuffer(ShapedBuffer shaped_buffer, + DeviceMemoryAllocator* allocator); + + // Movable, but not copyable. + ScopedShapedBuffer(ScopedShapedBuffer&& s); + ScopedShapedBuffer& operator=(ScopedShapedBuffer&&); + ScopedShapedBuffer(const ScopedShapedBuffer&) = delete; + ScopedShapedBuffer& operator=(const ScopedShapedBuffer&) = delete; + + // All buffers in the shape are deallocated on destruction. + ~ScopedShapedBuffer() override; // Return the allocator used to allocate the device memory held in this // ScopedShapedBuffer. DeviceMemoryAllocator* memory_allocator() const { return allocator_; } - // Release all device memory owned by this ScopedShapedBuffer and - // return the device memory pointers in the form of a - // ShapedBuffer. The returned ShapedBuffer takes over the memory - // from the ScopedShapedBuffer. The resulting ScopedShapedBuffer can - // only be destroyed. - std::unique_ptr release(); - - // All buffers in the shape are deallocated on destruction. - virtual ~ScopedShapedBuffer(); + // Releases all device memory owned by this ScopedShapedBuffer and returns the + // device memory pointers in the form of a ShapedBuffer. The returned + // ShapedBuffer takes over the memory from the ScopedShapedBuffer. The + // resulting ScopedShapedBuffer can only be destroyed. + ShapedBuffer release(); protected: - ScopedShapedBuffer(const ScopedShapedBuffer&) = delete; - void operator=(const ScopedShapedBuffer&) = delete; - DeviceMemoryAllocator* allocator_; }; diff --git a/tensorflow/compiler/xla/service/transfer_manager.cc b/tensorflow/compiler/xla/service/transfer_manager.cc index be8231b73c..98d0111d04 100644 --- a/tensorflow/compiler/xla/service/transfer_manager.cc +++ b/tensorflow/compiler/xla/service/transfer_manager.cc @@ -175,7 +175,7 @@ Status TransferManager::TransferBufferToDevice( return Status::OK(); } -StatusOr> TransferManager::AllocateShapedBuffer( +StatusOr TransferManager::AllocateShapedBuffer( const Shape& on_host_shape, DeviceMemoryAllocator* allocator, int device_ordinal) { if (!LayoutUtil::HasLayout(on_host_shape)) { @@ -187,31 +187,30 @@ StatusOr> TransferManager::AllocateShapedBuffer( const Shape on_device_shape = HostShapeToDeviceShape(on_host_shape); TF_RET_CHECK(LayoutUtil::HasLayout(on_device_shape)); - auto shaped_buffer = WrapUnique(new ShapedBuffer( - on_host_shape, on_device_shape, allocator->platform(), device_ordinal)); + ShapedBuffer shaped_buffer(on_host_shape, on_device_shape, + allocator->platform(), device_ordinal); // Allocate an appropriate sized buffer for each element in the shape // including the tuple pointer arrays. - for (auto& pair : shaped_buffer->buffers()) { + for (auto& pair : shaped_buffer.buffers()) { const ShapeIndex& index = pair.first; se::DeviceMemoryBase& memory_base = pair.second; const Shape& subshape = ShapeUtil::GetSubshape(on_device_shape, index); TF_ASSIGN_OR_RETURN(memory_base, - allocator->Allocate(shaped_buffer->device_ordinal(), + allocator->Allocate(shaped_buffer.device_ordinal(), GetByteSizeRequirement(subshape))); } return std::move(shaped_buffer); } -StatusOr> -TransferManager::AllocateScopedShapedBuffer(const Shape& on_host_shape, - DeviceMemoryAllocator* allocator, - int device_ordinal) { +StatusOr TransferManager::AllocateScopedShapedBuffer( + const Shape& on_host_shape, DeviceMemoryAllocator* allocator, + int device_ordinal) { TF_ASSIGN_OR_RETURN( - std::unique_ptr unscoped_buffer, + ShapedBuffer unscoped_buffer, AllocateShapedBuffer(on_host_shape, allocator, device_ordinal)); - return ScopedShapedBuffer::MakeScoped(unscoped_buffer.get(), allocator); + return ScopedShapedBuffer(std::move(unscoped_buffer), allocator); } } // namespace xla diff --git a/tensorflow/compiler/xla/service/transfer_manager.h b/tensorflow/compiler/xla/service/transfer_manager.h index 410d2af7af..a6451c4bb1 100644 --- a/tensorflow/compiler/xla/service/transfer_manager.h +++ b/tensorflow/compiler/xla/service/transfer_manager.h @@ -107,10 +107,10 @@ class TransferManager { // Allocate a ShapedBuffer which can hold data with the given on-host // shape. The on-device shape may be different as indicated by // HostShapeToDeviceShape. - StatusOr> AllocateShapedBuffer( - const Shape& on_host_shape, DeviceMemoryAllocator* allocator, - int device_ordinal); - StatusOr> AllocateScopedShapedBuffer( + StatusOr AllocateShapedBuffer(const Shape& on_host_shape, + DeviceMemoryAllocator* allocator, + int device_ordinal); + StatusOr AllocateScopedShapedBuffer( const Shape& on_host_shape, DeviceMemoryAllocator* allocator, int device_ordinal); diff --git a/tensorflow/compiler/xla/tests/dynamic_ops_test.cc b/tensorflow/compiler/xla/tests/dynamic_ops_test.cc index 464b8cbebb..021fbcedb9 100644 --- a/tensorflow/compiler/xla/tests/dynamic_ops_test.cc +++ b/tensorflow/compiler/xla/tests/dynamic_ops_test.cc @@ -735,11 +735,11 @@ void BM_DynamicSlice(int num_iters) { auto start_indices_literal = Literal::CreateR1({0, 1, 2, 3}); ASSERT_IS_OK(transfer_manager->TransferLiteralToDevice( - executors[device_ordinal], *start_indices_literal, *buffer)); + executors[device_ordinal], *start_indices_literal, buffer)); std::unique_ptr executable = client - ->Compile(computation, {&buffer->on_host_shape()}, + ->Compile(computation, {&buffer.on_host_shape()}, ExecutableBuildOptions()) .ConsumeValueOrDie(); @@ -748,14 +748,14 @@ void BM_DynamicSlice(int num_iters) { options.set_allocator(&allocator); const int kWarmups = 2; for (int i = 0; i < kWarmups; ++i) { - auto result = executable->Run({buffer.get()}, options); + auto result = executable->Run({&buffer}, options); ASSERT_TRUE(result.ok()); } // Run benchmark. tensorflow::testing::StartTiming(); for (int i = 0; i < num_iters; ++i) { - auto result = executable->Run({buffer.get()}, options); + auto result = executable->Run({&buffer}, options); ASSERT_TRUE(result.ok()); } } diff --git a/tensorflow/compiler/xla/tests/fusion_test.cc b/tensorflow/compiler/xla/tests/fusion_test.cc index ed16963b40..c7f64d8560 100644 --- a/tensorflow/compiler/xla/tests/fusion_test.cc +++ b/tensorflow/compiler/xla/tests/fusion_test.cc @@ -794,19 +794,19 @@ void BM_ParallelFusion(int num_iters) { // Transfer literals to device. auto param0_literal = Literal::CreateR2F32Linspace(1.0, 2.0, param0_dim0, param0_dim1); - std::unique_ptr buffer0 = + ShapedBuffer buffer0 = client->LiteralToShapedBuffer(*param0_literal, device_ordinal) .ConsumeValueOrDie(); auto param1_literal = Literal::CreateR2F32Linspace(1.0, 2.0, param1_dim0, param1_dim1); - std::unique_ptr buffer1 = + ShapedBuffer buffer1 = client->LiteralToShapedBuffer(*param1_literal, device_ordinal) .ConsumeValueOrDie(); auto param2_literal = Literal::CreateR2F32Linspace(1.0, 2.0, param2_dim0, param2_dim1); - std::unique_ptr buffer2 = + ShapedBuffer buffer2 = client->LiteralToShapedBuffer(*param2_literal, device_ordinal) .ConsumeValueOrDie(); @@ -814,8 +814,8 @@ void BM_ParallelFusion(int num_iters) { std::unique_ptr executable = client ->Compile(computation, - {&buffer0->on_host_shape(), &buffer1->on_host_shape(), - &buffer2->on_host_shape()}, + {&buffer0.on_host_shape(), &buffer1.on_host_shape(), + &buffer2.on_host_shape()}, ExecutableBuildOptions()) .ConsumeValueOrDie(); @@ -836,8 +836,7 @@ void BM_ParallelFusion(int num_iters) { // Run some warm-up executions. const int kWarmups = 2; for (int i = 0; i < kWarmups; ++i) { - auto result = - executable->Run({buffer0.get(), buffer1.get(), buffer2.get()}, options); + auto result = executable->Run({&buffer0, &buffer1, &buffer2}, options); ASSERT_TRUE(result.ok()); } @@ -850,8 +849,7 @@ void BM_ParallelFusion(int num_iters) { tensorflow::testing::UseRealTime(); tensorflow::testing::StartTiming(); for (int i = 0; i < num_iters; ++i) { - auto result = - executable->Run({buffer0.get(), buffer1.get(), buffer2.get()}, options); + auto result = executable->Run({&buffer0, &buffer1, &buffer2}, options); ASSERT_TRUE(result.ok()); } } diff --git a/tensorflow/compiler/xla/tests/local_client_allocation_test.cc b/tensorflow/compiler/xla/tests/local_client_allocation_test.cc index 3d30ceeaf1..7209f91639 100644 --- a/tensorflow/compiler/xla/tests/local_client_allocation_test.cc +++ b/tensorflow/compiler/xla/tests/local_client_allocation_test.cc @@ -25,6 +25,7 @@ limitations under the License. #include "tensorflow/compiler/xla/tests/literal_test_util.h" #include "tensorflow/compiler/xla/tests/local_client_test_base.h" #include "tensorflow/compiler/xla/tests/test_macros.h" +#include "tensorflow/core/lib/gtl/optional.h" #include "tensorflow/core/platform/test.h" #include "tensorflow/core/platform/types.h" @@ -53,7 +54,7 @@ XLA_TEST_F(LocalClientAllocationTest, AddVectors) { // deallocation happen on the right allocator. ExecutableRunOptions options; options.set_allocator(allocator); - std::unique_ptr result = + tensorflow::gtl::optional result = ExecuteLocallyOrDie(builder.Build().ValueOrDie(), {}, DefaultExecutableBuildOptions(), options); @@ -66,7 +67,7 @@ XLA_TEST_F(LocalClientAllocationTest, AddVectors) { // Deallocate result and verify that deallocate was called once. int64 deallocation_count_before = allocator_->deallocation_count(); - result = nullptr; + result.reset(); EXPECT_EQ(deallocation_count_before + 1, allocator_->deallocation_count()); } @@ -92,7 +93,7 @@ XLA_TEST_F(LocalClientAllocationTest, RunOnDevices) { computation, {}, ExecutableBuildOptions().set_device_ordinal(d), ExecutableRunOptions().set_device_ordinal(d).set_allocator(allocator)); LiteralTestUtil::ExpectR1Near( - {2.0f, 4.0f, 6.0f}, *ShapedBufferToLiteral(*result), error_spec_); + {2.0f, 4.0f, 6.0f}, *ShapedBufferToLiteral(result), error_spec_); // At least one allocation should have been performed when executing the // computation. diff --git a/tensorflow/compiler/xla/tests/local_client_execute_test.cc b/tensorflow/compiler/xla/tests/local_client_execute_test.cc index 373dd3c5df..7e14e77366 100644 --- a/tensorflow/compiler/xla/tests/local_client_execute_test.cc +++ b/tensorflow/compiler/xla/tests/local_client_execute_test.cc @@ -57,10 +57,9 @@ XLA_TEST_F(LocalClientExecuteTest, Constant) { ComputationBuilder builder(local_client_, TestName()); auto y = builder.ConstantR0(123.0f); - std::unique_ptr result = + ScopedShapedBuffer result = ExecuteLocallyOrDie(builder.Build().ValueOrDie(), {}); - - LiteralTestUtil::ExpectR0Near(123.f, *ShapedBufferToLiteral(*result), + LiteralTestUtil::ExpectR0Near(123.f, *ShapedBufferToLiteral(result), error_spec_); } @@ -71,10 +70,9 @@ XLA_TEST_F(LocalClientExecuteTest, AddScalars) { builder.Add(x, y); auto x_value = LiteralToShapedBuffer(*Literal::CreateR0(42.0f)); - std::unique_ptr result = - ExecuteLocallyOrDie(builder.Build().ValueOrDie(), {x_value.get()}); - - LiteralTestUtil::ExpectR0Near(165.f, *ShapedBufferToLiteral(*result), + ScopedShapedBuffer result = + ExecuteLocallyOrDie(builder.Build().ValueOrDie(), {&x_value}); + LiteralTestUtil::ExpectR0Near(165.f, *ShapedBufferToLiteral(result), error_spec_); } @@ -85,10 +83,9 @@ XLA_TEST_F(LocalClientExecuteTest, AddZeroElementVectors) { builder.Add(x, y); auto x_array = LiteralToShapedBuffer(*Literal::CreateR1({})); - std::unique_ptr result = - ExecuteLocallyOrDie(builder.Build().ValueOrDie(), {x_array.get()}); - - LiteralTestUtil::ExpectR1Near({}, *ShapedBufferToLiteral(*result), + ScopedShapedBuffer result = + ExecuteLocallyOrDie(builder.Build().ValueOrDie(), {&x_array}); + LiteralTestUtil::ExpectR1Near({}, *ShapedBufferToLiteral(result), error_spec_); } @@ -100,11 +97,10 @@ XLA_TEST_F(LocalClientExecuteTest, AddVectors) { auto x_array = LiteralToShapedBuffer(*Literal::CreateR1({0.0f, 1.0f, 2.0f})); - std::unique_ptr result = - ExecuteLocallyOrDie(builder.Build().ValueOrDie(), {x_array.get()}); - + ScopedShapedBuffer result = + ExecuteLocallyOrDie(builder.Build().ValueOrDie(), {&x_array}); LiteralTestUtil::ExpectR1Near( - {2.0f, 4.0f, 6.0f}, *ShapedBufferToLiteral(*result), error_spec_); + {2.0f, 4.0f, 6.0f}, *ShapedBufferToLiteral(result), error_spec_); } XLA_TEST_F(LocalClientExecuteTest, AddVectorsWithProfile) { @@ -116,13 +112,12 @@ XLA_TEST_F(LocalClientExecuteTest, AddVectorsWithProfile) { auto x_array = LiteralToShapedBuffer(*Literal::CreateR1({0.0f, 1.0f, 2.0f})); ExecutionProfile profile; - std::unique_ptr result = ExecuteLocallyOrDie( - builder.Build().ValueOrDie(), {x_array.get()}, - DefaultExecutableBuildOptions(), + ScopedShapedBuffer result = ExecuteLocallyOrDie( + builder.Build().ValueOrDie(), {&x_array}, DefaultExecutableBuildOptions(), DefaultExecutableRunOptions().set_execution_profile(&profile)); LiteralTestUtil::ExpectR1Near( - {2.0f, 4.0f, 6.0f}, *ShapedBufferToLiteral(*result), error_spec_); + {2.0f, 4.0f, 6.0f}, *ShapedBufferToLiteral(result), error_spec_); EXPECT_GT(profile.compute_and_transfer_time_ns(), 0); } @@ -136,27 +131,27 @@ XLA_TEST_F(LocalClientExecuteTest, AddArraysWithDifferentInputLayouts) { // Create x as a col-major array. auto x_array = LiteralToShapedBuffer(*Literal::CreateR2WithLayout( {{1.0f, 2.0f}, {3.0f, 4.0f}}, LayoutUtil::MakeLayout({0, 1}))); - EXPECT_TRUE(LayoutUtil::Equal(x_array->on_device_shape().layout(), + EXPECT_TRUE(LayoutUtil::Equal(x_array.on_device_shape().layout(), LayoutUtil::MakeLayout({0, 1}))); // Create y as a row-major array. auto y_array = LiteralToShapedBuffer(*Literal::CreateR2WithLayout( {{10.0f, 20.0f}, {30.0f, 40.0f}}, LayoutUtil::MakeLayout({1, 0}))); - EXPECT_TRUE(LayoutUtil::Equal(y_array->on_device_shape().layout(), + EXPECT_TRUE(LayoutUtil::Equal(y_array.on_device_shape().layout(), LayoutUtil::MakeLayout({1, 0}))); - std::unique_ptr result_colmaj = - ExecuteLocallyOrDie(computation, {x_array.get(), y_array.get()}); + ScopedShapedBuffer result_colmaj = + ExecuteLocallyOrDie(computation, {&x_array, &y_array}); LiteralTestUtil::ExpectR2Near({{11.0f, 22.0f}, {33.0f, 44.0f}}, - *ShapedBufferToLiteral(*result_colmaj), + *ShapedBufferToLiteral(result_colmaj), error_spec_); // Run with the parameter values in a different order. - std::unique_ptr result_param_swap = - ExecuteLocallyOrDie(computation, {y_array.get(), x_array.get()}); + ScopedShapedBuffer result_param_swap = + ExecuteLocallyOrDie(computation, {&y_array, &x_array}); LiteralTestUtil::ExpectR2Near( {{11.0f, 22.0f}, {33.0f, 44.0f}}, - *ShapedBufferToLiteral(*result_param_swap), error_spec_); + *ShapedBufferToLiteral(result_param_swap), error_spec_); } XLA_TEST_F(LocalClientExecuteTest, AddArraysWithDifferentOutputLayouts) { @@ -172,27 +167,27 @@ XLA_TEST_F(LocalClientExecuteTest, AddArraysWithDifferentOutputLayouts) { *Literal::CreateR2({{10.0f, 20.0f}, {30.0f, 40.0f}})); // Run with col-major result layout. - std::unique_ptr result_colmaj = ExecuteLocallyOrDie( - computation, {x_array.get(), y_array.get()}, + ScopedShapedBuffer result_colmaj = ExecuteLocallyOrDie( + computation, {&x_array, &y_array}, DefaultExecutableBuildOptions().set_result_layout( ShapeUtil::MakeShapeWithLayout(F32, /*dimensions=*/{2, 2}, {0, 1})), DefaultExecutableRunOptions()); - EXPECT_TRUE(LayoutUtil::Equal(result_colmaj->on_device_shape().layout(), + EXPECT_TRUE(LayoutUtil::Equal(result_colmaj.on_device_shape().layout(), LayoutUtil::MakeLayout({0, 1}))); LiteralTestUtil::ExpectR2Near({{11.0f, 22.0f}, {33.0f, 44.0f}}, - *ShapedBufferToLiteral(*result_colmaj), + *ShapedBufferToLiteral(result_colmaj), error_spec_); // Run with row-major result layout. - std::unique_ptr result_rowmaj = ExecuteLocallyOrDie( - computation, {x_array.get(), y_array.get()}, + ScopedShapedBuffer result_rowmaj = ExecuteLocallyOrDie( + computation, {&x_array, &y_array}, DefaultExecutableBuildOptions().set_result_layout( ShapeUtil::MakeShapeWithLayout(F32, /*dimensions=*/{2, 2}, {1, 0})), DefaultExecutableRunOptions()); - EXPECT_TRUE(LayoutUtil::Equal(result_rowmaj->on_device_shape().layout(), + EXPECT_TRUE(LayoutUtil::Equal(result_rowmaj.on_device_shape().layout(), LayoutUtil::MakeLayout({1, 0}))); LiteralTestUtil::ExpectR2Near({{11.0f, 22.0f}, {33.0f, 44.0f}}, - *ShapedBufferToLiteral(*result_rowmaj), + *ShapedBufferToLiteral(result_rowmaj), error_spec_); } @@ -208,13 +203,13 @@ XLA_TEST_F(LocalClientExecuteTest, TupleResult) { auto y_array = LiteralToShapedBuffer( *Literal::CreateR2({{10.0f, 20.0f}, {30.0f, 40.0f}})); - std::unique_ptr result = - ExecuteLocallyOrDie(computation, {x_array.get(), y_array.get()}); + ScopedShapedBuffer result = + ExecuteLocallyOrDie(computation, {&x_array, &y_array}); - EXPECT_TRUE(ShapeUtil::IsTuple(result->on_host_shape())); - EXPECT_EQ(3, ShapeUtil::TupleElementCount(result->on_host_shape())); + EXPECT_TRUE(ShapeUtil::IsTuple(result.on_host_shape())); + EXPECT_EQ(3, ShapeUtil::TupleElementCount(result.on_host_shape())); - std::unique_ptr result_literal = ShapedBufferToLiteral(*result); + std::unique_ptr result_literal = ShapedBufferToLiteral(result); LiteralTestUtil::ExpectR2Equal( {{1.0f, 2.0f}, {3.0f, 4.0f}}, LiteralView::Create(*result_literal, {0})); LiteralTestUtil::ExpectR2Equal( @@ -237,13 +232,13 @@ XLA_TEST_F(LocalClientExecuteTest, NestedTupleResult) { auto y_array = LiteralToShapedBuffer( *Literal::CreateR2({{10.0f, 20.0f}, {30.0f, 40.0f}})); - std::unique_ptr result = - ExecuteLocallyOrDie(computation, {x_array.get(), y_array.get()}); + ScopedShapedBuffer result = + ExecuteLocallyOrDie(computation, {&x_array, &y_array}); - EXPECT_TRUE(ShapeUtil::IsTuple(result->on_host_shape())); - EXPECT_EQ(2, ShapeUtil::TupleElementCount(result->on_host_shape())); + EXPECT_TRUE(ShapeUtil::IsTuple(result.on_host_shape())); + EXPECT_EQ(2, ShapeUtil::TupleElementCount(result.on_host_shape())); - std::unique_ptr result_literal = ShapedBufferToLiteral(*result); + std::unique_ptr result_literal = ShapedBufferToLiteral(result); LiteralTestUtil::ExpectR2Equal( {{1.0f, 2.0f}, {3.0f, 4.0f}}, LiteralView::Create(*result_literal, {1})); LiteralTestUtil::ExpectR2Equal( @@ -274,11 +269,11 @@ XLA_TEST_F(LocalClientExecuteTest, TupleResultWithLayout) { ShapeUtil::MakeShapeWithLayout(F32, /*dimensions=*/{2, 2}, /*minor_to_major=*/{1, 0})}); options.set_result_layout(shape_with_layout); - std::unique_ptr result = ExecuteLocallyOrDie( - builder.Build().ValueOrDie(), {array.get(), array.get()}, options, - DefaultExecutableRunOptions()); + ScopedShapedBuffer result = + ExecuteLocallyOrDie(builder.Build().ValueOrDie(), {&array, &array}, + options, DefaultExecutableRunOptions()); - std::unique_ptr result_literal = ShapedBufferToLiteral(*result); + std::unique_ptr result_literal = ShapedBufferToLiteral(result); LiteralTestUtil::ExpectR2Equal( {{1.0f, 2.0f}, {3.0f, 4.0f}}, LiteralView::Create(*result_literal, {0})); LiteralTestUtil::ExpectR2Equal( @@ -318,13 +313,13 @@ XLA_TEST_F(LocalClientExecuteTest, TupleArguments) { auto x_buffer = LiteralToShapedBuffer(*x_literal); auto y_buffer = LiteralToShapedBuffer(*y_literal); - std::unique_ptr result = - ExecuteLocallyOrDie(computation, {x_buffer.get(), y_buffer.get()}); + ScopedShapedBuffer result = + ExecuteLocallyOrDie(computation, {&x_buffer, &y_buffer}); - EXPECT_TRUE(ShapeUtil::IsTuple(result->on_host_shape())); - EXPECT_EQ(2, ShapeUtil::TupleElementCount(result->on_host_shape())); + EXPECT_TRUE(ShapeUtil::IsTuple(result.on_host_shape())); + EXPECT_EQ(2, ShapeUtil::TupleElementCount(result.on_host_shape())); - std::unique_ptr result_literal = ShapedBufferToLiteral(*result); + std::unique_ptr result_literal = ShapedBufferToLiteral(result); LiteralTestUtil::ExpectR2Equal( {{56.0f, 46.0f}, {36.0f, 26.0f}}, LiteralView::Create(*result_literal, {0})); @@ -363,10 +358,9 @@ XLA_TEST_F(LocalClientExecuteTest, NestedTupleArgument) { Literal::CreateR1({222.0, -2.0, 10.0}).get()}); auto arg_buffer = LiteralToShapedBuffer(*arg_literal); - std::unique_ptr result = - ExecuteLocallyOrDie(computation, {arg_buffer.get()}); + ScopedShapedBuffer result = ExecuteLocallyOrDie(computation, {&arg_buffer}); - std::unique_ptr result_literal = ShapedBufferToLiteral(*result); + std::unique_ptr result_literal = ShapedBufferToLiteral(result); LiteralTestUtil::ExpectR2Equal( {{-1.0, -2.0}, {-3.0, -4}}, LiteralView::Create(*result_literal, {0})); LiteralTestUtil::ExpectR1Equal( @@ -394,18 +388,16 @@ XLA_TEST_F(LocalClientExecuteTest, PassingTupleResultBackIntoComputation) { Literal::CreateR2({{11.0, 3.0}, {4.0, 5.0}}).get()}); auto arg_buffer = LiteralToShapedBuffer(*arg_literal); - std::unique_ptr result_0 = - ExecuteLocallyOrDie(computation, {arg_buffer.get()}); - std::unique_ptr result_0_literal = ShapedBufferToLiteral(*result_0); + ScopedShapedBuffer result_0 = ExecuteLocallyOrDie(computation, {&arg_buffer}); + std::unique_ptr result_0_literal = ShapedBufferToLiteral(result_0); LiteralTestUtil::ExpectR2Equal( {{-1.0, -2.0}, {-3.0, -4.0}}, LiteralView::Create(*result_0_literal, {0})); LiteralTestUtil::ExpectR2Equal( {{22.0, 6.0}, {8.0, 10}}, LiteralView::Create(*result_0_literal, {1})); - std::unique_ptr result_1 = - ExecuteLocallyOrDie(computation, {result_0.get()}); - std::unique_ptr result_1_literal = ShapedBufferToLiteral(*result_1); + ScopedShapedBuffer result_1 = ExecuteLocallyOrDie(computation, {&result_0}); + std::unique_ptr result_1_literal = ShapedBufferToLiteral(result_1); LiteralTestUtil::ExpectR2Equal( {{1.0, 2.0}, {3.0, 4.0}}, LiteralView::Create(*result_1_literal, {0})); LiteralTestUtil::ExpectR2Equal( @@ -451,10 +443,8 @@ XLA_TEST_F(LocalClientExecuteTest, LargeTuple) { Literal::MakeTupleOwned(std::move(arg_elements)); auto arg_buffer = LiteralToShapedBuffer(*arg_literal); - std::unique_ptr result = - ExecuteLocallyOrDie(computation, {arg_buffer.get()}); - - std::unique_ptr result_literal = ShapedBufferToLiteral(*result); + ScopedShapedBuffer result = ExecuteLocallyOrDie(computation, {&arg_buffer}); + std::unique_ptr result_literal = ShapedBufferToLiteral(result); for (int i = 0; i < kElementCount; ++i) { LiteralTestUtil::ExpectR1Near( @@ -509,9 +499,8 @@ XLA_TEST_F(LocalClientExecuteTest, DISABLED_ON_CPU_PARALLEL(LargeNestedTuple)) { auto arg_literal = Literal::MakeTupleOwned(std::move(outer_tuple_elements)); auto arg_buffer = LiteralToShapedBuffer(*arg_literal); - std::unique_ptr result = - ExecuteLocallyOrDie(computation, {arg_buffer.get()}); - std::unique_ptr result_literal = ShapedBufferToLiteral(*result); + ScopedShapedBuffer result = ExecuteLocallyOrDie(computation, {&arg_buffer}); + std::unique_ptr result_literal = ShapedBufferToLiteral(result); for (int i = 0; i < kFanout; ++i) { for (int j = 0; j < kFanout; ++j) { @@ -554,9 +543,8 @@ XLA_TEST_F(LocalClientExecuteTest, DeepTuple) { } auto arg_buffer = LiteralToShapedBuffer(*arg_literal); - std::unique_ptr result = - ExecuteLocallyOrDie(computation, {arg_buffer.get()}); - std::unique_ptr result_literal = ShapedBufferToLiteral(*result); + ScopedShapedBuffer result = ExecuteLocallyOrDie(computation, {&arg_buffer}); + std::unique_ptr result_literal = ShapedBufferToLiteral(result); ShapeIndex index; for (int i = 0; i < kTupleDepth; ++i) { @@ -576,7 +564,7 @@ XLA_TEST_F(LocalClientExecuteTest, InvalidNumberOfArguments) { auto x_array = LiteralToShapedBuffer(*Literal::CreateR1({1.0f, 2.0f, 3.0f})); auto execute_status = - ExecuteLocally(builder.Build().ValueOrDie(), {x_array.get()}); + ExecuteLocally(builder.Build().ValueOrDie(), {&x_array}); EXPECT_FALSE(execute_status.ok()); EXPECT_THAT(execute_status.status().error_message(), @@ -592,7 +580,7 @@ XLA_TEST_F(LocalClientExecuteTest, IncorrectArgumentShape) { auto x_array = LiteralToShapedBuffer( *Literal::CreateR2({{0.0f, 1.0f}, {2.0f, 3.0f}})); auto execute_status = - ExecuteLocally(builder.Build().ValueOrDie(), {x_array.get()}); + ExecuteLocally(builder.Build().ValueOrDie(), {&x_array}); EXPECT_FALSE(execute_status.ok()); EXPECT_THAT(execute_status.status().error_message(), @@ -609,7 +597,7 @@ XLA_TEST_F(LocalClientExecuteTest, InvalidResultLayout) { auto x_array = LiteralToShapedBuffer( *Literal::CreateR2({{0.0f, 1.0f}, {2.0f, 3.0f}})); auto execute_status = ExecuteLocally( - builder.Build().ValueOrDie(), {x_array.get()}, + builder.Build().ValueOrDie(), {&x_array}, DefaultExecutableBuildOptions().set_result_layout( ShapeUtil::MakeShapeWithLayout(F32, /*dimensions=*/{1, 2, 3, 4}, @@ -642,9 +630,9 @@ XLA_TEST_F(LocalClientExecuteTest, RunOnAllDeviceOrdinals) { computation, {}, DefaultExecutableBuildOptions().set_device_ordinal(d), DefaultExecutableRunOptions().set_device_ordinal(d)); - EXPECT_EQ(d, result->device_ordinal()); + EXPECT_EQ(d, result.device_ordinal()); LiteralTestUtil::ExpectR0Equal(42.0f, - *ShapedBufferToLiteral(*result)); + *ShapedBufferToLiteral(result)); } } } @@ -687,9 +675,9 @@ XLA_TEST_F(LocalClientExecuteTest, RunOnStream) { DefaultExecutableRunOptions().set_stream(&stream)); // As a check to verify that the computation ran of the device associated // with the stream. This is a weak check, but stronger verification is hard. - EXPECT_EQ(d, result->device_ordinal()); + EXPECT_EQ(d, result.device_ordinal()); LiteralTestUtil::ExpectR0Equal(42.0f, - *ShapedBufferToLiteral(*result)); + *ShapedBufferToLiteral(result)); } } @@ -765,9 +753,9 @@ XLA_TEST_F(LocalClientExecuteTest, SelectBetweenTuples) { {builder.ConstantR1(vec2), builder.ConstantR1(vec1)}); builder.Select(builder.ConstantR0(false), tuple12, tuple21); - std::unique_ptr result = + ScopedShapedBuffer result = ExecuteLocallyOrDie(builder.Build().ValueOrDie(), {}); - std::unique_ptr tuple_literal = ShapedBufferToLiteral(*result); + std::unique_ptr tuple_literal = ShapedBufferToLiteral(result); LiteralTestUtil::ExpectR1Equal( {2.0f, 4.0f, 6.0f}, LiteralView::Create(*tuple_literal, {0})); LiteralTestUtil::ExpectR1Equal( @@ -791,12 +779,12 @@ XLA_TEST_F(LocalClientExecuteTest, CompileExecutable) { auto x_array = LiteralToShapedBuffer(*Literal::CreateR1({0.0f, 1.0f, 2.0f})); - std::unique_ptr result = - executable->Run({x_array.get()}, DefaultExecutableRunOptions()) + ScopedShapedBuffer result = + executable->Run({&x_array}, DefaultExecutableRunOptions()) .ConsumeValueOrDie(); LiteralTestUtil::ExpectR1Near( - {2.0f, 4.0f, 6.0f}, *ShapedBufferToLiteral(*result), error_spec_); + {2.0f, 4.0f, 6.0f}, *ShapedBufferToLiteral(result), error_spec_); } XLA_TEST_F(LocalClientExecuteTest, ShapeBufferToLiteralConversion) { @@ -809,7 +797,7 @@ XLA_TEST_F(LocalClientExecuteTest, ShapeBufferToLiteralConversion) { literal, local_client_->default_device_ordinal(), allocator_)); TF_ASSERT_OK_AND_ASSIGN( auto transferred_literal, - local_client_->ShapedBufferToLiteral(*shaped_buffer)); + local_client_->ShapedBufferToLiteral(shaped_buffer)); EXPECT_EQ(literal, *transferred_literal); }; @@ -849,7 +837,7 @@ XLA_TEST_F(LocalClientExecuteTest, ShapeBufferToLiteralConversion64bit) { literal, local_client_->default_device_ordinal(), allocator_)); TF_ASSERT_OK_AND_ASSIGN( auto transferred_literal, - local_client_->ShapedBufferToLiteral(*shaped_buffer)); + local_client_->ShapedBufferToLiteral(shaped_buffer)); EXPECT_EQ(literal, *transferred_literal); }; @@ -917,12 +905,12 @@ void BM_LocalClientOverhead(int num_iters) { .ConsumeValueOrDie(); auto literal = Literal::CreateR2({{0, 0, 0}, {0, 0, 0}}); ASSERT_IS_OK(transfer_manager->TransferLiteralToDevice( - executors[device_ordinal], *literal, *buffer)); + executors[device_ordinal], *literal, buffer)); const int kWarmups = 2; auto executable_status = client->Compile( - computation, {&buffer->on_host_shape()}, ExecutableBuildOptions()); + computation, {&buffer.on_host_shape()}, ExecutableBuildOptions()); ASSERT_IS_OK(executable_status); std::unique_ptr executable = executable_status.ConsumeValueOrDie(); @@ -934,13 +922,13 @@ void BM_LocalClientOverhead(int num_iters) { run_options.set_allocator(&allocator).set_stream(&stream); for (int i = 0; i < kWarmups; ++i) { - auto result = executable->Run({buffer.get()}, run_options); + auto result = executable->Run({&buffer}, run_options); ASSERT_IS_OK(result); } tensorflow::testing::StartTiming(); for (int i = 0; i < num_iters; ++i) { - auto result = executable->Run({buffer.get()}, run_options); + auto result = executable->Run({&buffer}, run_options); ASSERT_IS_OK(result); } } diff --git a/tensorflow/compiler/xla/tests/local_client_test_base.cc b/tensorflow/compiler/xla/tests/local_client_test_base.cc index 29fd985acf..c60ba2422f 100644 --- a/tensorflow/compiler/xla/tests/local_client_test_base.cc +++ b/tensorflow/compiler/xla/tests/local_client_test_base.cc @@ -128,7 +128,7 @@ LocalClientTestBase::LocalClientTestBase(se::Platform* platform) LocalClientTestBase::~LocalClientTestBase() {} -std::unique_ptr LocalClientTestBase::LiteralToShapedBuffer( +ScopedShapedBuffer LocalClientTestBase::LiteralToShapedBuffer( const Literal& literal) { return local_client_ ->LiteralToShapedBuffer(literal, local_client_->default_device_ordinal()) @@ -155,7 +155,7 @@ ExecutableRunOptions LocalClientTestBase::DefaultExecutableRunOptions() const { return run_options; } -std::unique_ptr LocalClientTestBase::ExecuteLocallyOrDie( +ScopedShapedBuffer LocalClientTestBase::ExecuteLocallyOrDie( const Computation& computation, tensorflow::gtl::ArraySlice arguments) { return ExecuteLocally(computation, arguments, DefaultExecutableBuildOptions(), @@ -163,7 +163,7 @@ std::unique_ptr LocalClientTestBase::ExecuteLocallyOrDie( .ConsumeValueOrDie(); } -std::unique_ptr LocalClientTestBase::ExecuteLocallyOrDie( +ScopedShapedBuffer LocalClientTestBase::ExecuteLocallyOrDie( const Computation& computation, tensorflow::gtl::ArraySlice arguments, const ExecutableBuildOptions& build_options, @@ -172,16 +172,14 @@ std::unique_ptr LocalClientTestBase::ExecuteLocallyOrDie( .ConsumeValueOrDie(); } -StatusOr> -LocalClientTestBase::ExecuteLocally( +StatusOr LocalClientTestBase::ExecuteLocally( const Computation& computation, tensorflow::gtl::ArraySlice arguments) { return ExecuteLocally(computation, arguments, DefaultExecutableBuildOptions(), DefaultExecutableRunOptions()); } -StatusOr> -LocalClientTestBase::ExecuteLocally( +StatusOr LocalClientTestBase::ExecuteLocally( const Computation& computation, tensorflow::gtl::ArraySlice arguments, const ExecutableBuildOptions& build_options, diff --git a/tensorflow/compiler/xla/tests/local_client_test_base.h b/tensorflow/compiler/xla/tests/local_client_test_base.h index 7555d5e893..4ee56a05ec 100644 --- a/tensorflow/compiler/xla/tests/local_client_test_base.h +++ b/tensorflow/compiler/xla/tests/local_client_test_base.h @@ -83,8 +83,7 @@ class LocalClientTestBase : public ::testing::Test { // Copy the given literal onto the default device and return a // ScopedShapedBuffer. Convenience wrapper around // LocalClient::LiteralToShapedBuffer. - std::unique_ptr LiteralToShapedBuffer( - const Literal& literal); + ScopedShapedBuffer LiteralToShapedBuffer(const Literal& literal); // Construct and return a literal containing the array represented by // shaped_buffer. @@ -93,19 +92,19 @@ class LocalClientTestBase : public ::testing::Test { // Execute the given computation on the local client. With and without // options. - StatusOr> ExecuteLocally( + StatusOr ExecuteLocally( const Computation& computation, tensorflow::gtl::ArraySlice arguments); - StatusOr> ExecuteLocally( + StatusOr ExecuteLocally( const Computation& computation, tensorflow::gtl::ArraySlice arguments, const ExecutableBuildOptions& build_options, const ExecutableRunOptions& run_options); - std::unique_ptr ExecuteLocallyOrDie( + ScopedShapedBuffer ExecuteLocallyOrDie( const Computation& computation, tensorflow::gtl::ArraySlice arguments); - std::unique_ptr ExecuteLocallyOrDie( + ScopedShapedBuffer ExecuteLocallyOrDie( const Computation& computation, tensorflow::gtl::ArraySlice arguments, const ExecutableBuildOptions& build_options, diff --git a/tensorflow/compiler/xla/tests/transfer_manager_test.cc b/tensorflow/compiler/xla/tests/transfer_manager_test.cc index 268ba338f2..e2067bc1b8 100644 --- a/tensorflow/compiler/xla/tests/transfer_manager_test.cc +++ b/tensorflow/compiler/xla/tests/transfer_manager_test.cc @@ -45,7 +45,7 @@ class TransferManagerTest : public LocalClientTestBase { ~TransferManagerTest() override = default; - std::unique_ptr AllocateDeviceBuffer(const Shape& shape) { + ScopedShapedBuffer AllocateDeviceBuffer(const Shape& shape) { return transfer_manager_ ->AllocateScopedShapedBuffer( shape, GetOrCreateAllocator(local_client_->platform()), @@ -64,10 +64,10 @@ XLA_TEST_F(TransferManagerTest, TransferR0U32) { // Round trip literal through device. ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice( - stream_executor_, *literal, *device_buffer)); + stream_executor_, *literal, device_buffer)); TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr result, transfer_manager_->TransferLiteralFromDevice( - stream_executor_, *device_buffer)); + stream_executor_, device_buffer)); LiteralTestUtil::ExpectR0Equal(42, *result); } @@ -80,10 +80,10 @@ XLA_TEST_F(TransferManagerTest, TransferR1F32) { // Round trip literal through device. ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice( - stream_executor_, *literal, *device_buffer)); + stream_executor_, *literal, device_buffer)); TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr result, transfer_manager_->TransferLiteralFromDevice( - stream_executor_, *device_buffer)); + stream_executor_, device_buffer)); LiteralTestUtil::ExpectR1Equal({1.25f, 2.5f, -17.0f, -20.125f}, *result); @@ -98,10 +98,10 @@ XLA_TEST_F(TransferManagerTest, TransferR1LargeF32) { // Round trip literal through device. ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice( - stream_executor_, *literal, *device_buffer)); + stream_executor_, *literal, device_buffer)); TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr result, transfer_manager_->TransferLiteralFromDevice( - stream_executor_, *device_buffer)); + stream_executor_, device_buffer)); LiteralTestUtil::ExpectR1Equal(test_vector, *result); } @@ -114,10 +114,10 @@ XLA_TEST_F(TransferManagerTest, TransferR1U8) { // Round trip literal through device. ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice( - stream_executor_, *literal, *device_buffer)); + stream_executor_, *literal, device_buffer)); TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr result, transfer_manager_->TransferLiteralFromDevice( - stream_executor_, *device_buffer)); + stream_executor_, device_buffer)); EXPECT_EQ(result->GetR1U8AsString(), test_string); } @@ -130,10 +130,10 @@ XLA_TEST_F(TransferManagerTest, TransferR2F32) { // Round trip literal through device. ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice( - stream_executor_, *literal, *device_buffer)); + stream_executor_, *literal, device_buffer)); TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr result, transfer_manager_->TransferLiteralFromDevice( - stream_executor_, *device_buffer)); + stream_executor_, device_buffer)); LiteralTestUtil::ExpectR2Equal( {{1.0f, 2.0f, 3.0f}, {4.0f, 5.0f, 6.0f}}, *result); @@ -150,10 +150,10 @@ XLA_TEST_F(TransferManagerTest, // Round trip literal through device. Set the on-device layout to something // different than the literal layout. ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice( - stream_executor_, *literal, *device_buffer)); + stream_executor_, *literal, device_buffer)); TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr result, transfer_manager_->TransferLiteralFromDevice( - stream_executor_, *device_buffer)); + stream_executor_, device_buffer)); EXPECT_FALSE( LayoutUtil::Equal(result->shape().layout(), literal->shape().layout())); @@ -170,10 +170,10 @@ XLA_TEST_F(TransferManagerTest, TransferTuple) { // Round trip literal through device. ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice( - stream_executor_, *literal, *device_buffer)); + stream_executor_, *literal, device_buffer)); TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr result, transfer_manager_->TransferLiteralFromDevice( - stream_executor_, *device_buffer)); + stream_executor_, device_buffer)); LiteralTestUtil::ExpectEqual(*literal, *result); } @@ -184,10 +184,10 @@ XLA_TEST_F(TransferManagerTest, TransferEmptyTuple) { // Round trip literal through device. ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice( - stream_executor_, *literal, *device_buffer)); + stream_executor_, *literal, device_buffer)); TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr result, transfer_manager_->TransferLiteralFromDevice( - stream_executor_, *device_buffer)); + stream_executor_, device_buffer)); LiteralTestUtil::ExpectEqual(*literal, *result); } @@ -204,10 +204,10 @@ XLA_TEST_F(TransferManagerTest, TransferNestedTuple) { // Round trip literal through device. ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice( - stream_executor_, *literal, *device_buffer)); + stream_executor_, *literal, device_buffer)); TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr result, transfer_manager_->TransferLiteralFromDevice( - stream_executor_, *device_buffer)); + stream_executor_, device_buffer)); LiteralTestUtil::ExpectEqual(*literal, *result); } @@ -219,10 +219,10 @@ XLA_TEST_F(TransferManagerTest, TransferComplexValue) { // Round trip literal through device. ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice( - stream_executor_, *literal, *device_buffer)); + stream_executor_, *literal, device_buffer)); TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr result, transfer_manager_->TransferLiteralFromDevice( - stream_executor_, *device_buffer)); + stream_executor_, device_buffer)); LiteralTestUtil::ExpectEqual(*literal, *result); } @@ -238,10 +238,10 @@ XLA_TEST_F(TransferManagerTest, TransferComplexValueInTuple) { // Round trip literal through device. ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice( - stream_executor_, *literal, *device_buffer)); + stream_executor_, *literal, device_buffer)); TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr result, transfer_manager_->TransferLiteralFromDevice( - stream_executor_, *device_buffer)); + stream_executor_, device_buffer)); LiteralTestUtil::ExpectEqual(*literal, *result); } diff --git a/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc b/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc index efb00d56c5..837a01e873 100644 --- a/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc +++ b/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc @@ -129,18 +129,18 @@ void ExecuteAndFetchProfile(string* profile_output, LocalClient* client, auto* transfer_manager = backend->transfer_manager(); TF_ASSERT_OK_AND_ASSIGN( - std::unique_ptr lhs_arg, + ScopedShapedBuffer lhs_arg, transfer_manager->AllocateScopedShapedBuffer( lhs_arg_shape, allocator, backend->default_device_ordinal())); TF_ASSERT_OK(transfer_manager->TransferLiteralToDevice( - executor, *Literal::CreateFromShape(lhs_arg_shape), *lhs_arg)); + executor, *Literal::CreateFromShape(lhs_arg_shape), lhs_arg)); TF_ASSERT_OK_AND_ASSIGN( - std::unique_ptr rhs_arg, + ScopedShapedBuffer rhs_arg, transfer_manager->AllocateScopedShapedBuffer( rhs_arg_shape, allocator, backend->default_device_ordinal())); TF_ASSERT_OK(transfer_manager->TransferLiteralToDevice( - executor, *Literal::CreateFromShape(rhs_arg_shape), *rhs_arg)); + executor, *Literal::CreateFromShape(rhs_arg_shape), rhs_arg)); TF_ASSERT_OK_AND_ASSIGN( std::unique_ptr local_executable, @@ -165,7 +165,7 @@ void ExecuteAndFetchProfile(string* profile_output, LocalClient* client, backend->eigen_intra_op_thread_pool()); TF_ASSERT_OK_AND_ASSIGN( auto execution_result, - executable->ExecuteOnStream(&run_options, {lhs_arg.get(), rhs_arg.get()}, + executable->ExecuteOnStream(&run_options, {&lhs_arg, &rhs_arg}, &hlo_execution_profile)); (void)execution_result; -- GitLab From d710d01a015fda65348ac0e5c25be3747624a779 Mon Sep 17 00:00:00 2001 From: Jiri Simsa Date: Thu, 19 Apr 2018 17:21:50 -0700 Subject: [PATCH 701/791] Minor code refactoring. PiperOrigin-RevId: 193600173 --- tensorflow/core/kernels/data/BUILD | 3 ++- tensorflow/core/kernels/data/dataset_utils.cc | 13 +++++++++++++ tensorflow/core/kernels/data/dataset_utils.h | 2 ++ tensorflow/core/kernels/data/iterator_ops.cc | 13 ++----------- tensorflow/core/kernels/data/writer_ops.cc | 15 ++------------- 5 files changed, 21 insertions(+), 25 deletions(-) diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD index 667a6967a8..c78e0aff83 100644 --- a/tensorflow/core/kernels/data/BUILD +++ b/tensorflow/core/kernels/data/BUILD @@ -515,6 +515,7 @@ tf_kernel_library( srcs = ["iterator_ops.cc"], deps = [ ":dataset", + ":dataset_utils", "//tensorflow/core:core_cpu_internal", "//tensorflow/core:dataset_ops_op_lib", "//tensorflow/core:framework", @@ -586,7 +587,7 @@ tf_kernel_library( srcs = ["writer_ops.cc"], deps = [ ":dataset", - "//tensorflow/core:core_cpu_internal", + ":dataset_utils", "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", diff --git a/tensorflow/core/kernels/data/dataset_utils.cc b/tensorflow/core/kernels/data/dataset_utils.cc index e3a3601ee8..67ddb52d57 100644 --- a/tensorflow/core/kernels/data/dataset_utils.cc +++ b/tensorflow/core/kernels/data/dataset_utils.cc @@ -14,6 +14,7 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/kernels/data/dataset_utils.h" +#include "tensorflow/core/common_runtime/device.h" namespace tensorflow { @@ -45,6 +46,18 @@ Status MakeIteratorFromInputElement( return Status::OK(); } +IteratorContext MakeIteratorContext(OpKernelContext* ctx) { + IteratorContext::Params params; + params.env = ctx->env(); + params.runner = *(ctx->runner()); + params.lib = ctx->function_library(); + DeviceBase* device = ctx->function_library()->device(); + params.allocator_getter = [device](AllocatorAttributes attrs) { + return device->GetAllocator(attrs); + }; + return IteratorContext(params); +} + } // namespace dataset } // namespace tensorflow diff --git a/tensorflow/core/kernels/data/dataset_utils.h b/tensorflow/core/kernels/data/dataset_utils.h index 6c4191c2be..e5ca71dd99 100644 --- a/tensorflow/core/kernels/data/dataset_utils.h +++ b/tensorflow/core/kernels/data/dataset_utils.h @@ -28,6 +28,8 @@ Status MakeIteratorFromInputElement( int64 thread_index, CapturedFunction* captured_func, StringPiece prefix, std::unique_ptr* out_iterator); +IteratorContext MakeIteratorContext(OpKernelContext* ctx); + } // namespace dataset } // namespace tensorflow diff --git a/tensorflow/core/kernels/data/iterator_ops.cc b/tensorflow/core/kernels/data/iterator_ops.cc index 4e4997d7b3..f5db97fd59 100644 --- a/tensorflow/core/kernels/data/iterator_ops.cc +++ b/tensorflow/core/kernels/data/iterator_ops.cc @@ -24,6 +24,7 @@ limitations under the License. #include "tensorflow/core/framework/variant_op_registry.h" #include "tensorflow/core/graph/graph_constructor.h" #include "tensorflow/core/kernels/data/dataset.h" +#include "tensorflow/core/kernels/data/dataset_utils.h" #include "tensorflow/core/kernels/ops_util.h" #include "tensorflow/core/lib/core/threadpool.h" #include "tensorflow/core/lib/gtl/cleanup.h" @@ -609,17 +610,7 @@ class ToSingleElementOp : public AsyncOpKernel { ctx, GetDatasetFromVariantTensor(ctx->input(0), &dataset), done); auto iterator = dataset->MakeIterator("SingleElementIterator"); - IteratorContext::Params params; - params.env = ctx->env(); - params.runner = *(ctx->runner()); - params.lib = ctx->function_library(); - DeviceBase* device = ctx->function_library()->device(); - params.allocator_getter = [device](AllocatorAttributes attrs) { - return device->GetAllocator(attrs); - }; - - IteratorContext iter_ctx(std::move(params)); - + IteratorContext iter_ctx = dataset::MakeIteratorContext(ctx); std::vector components; components.reserve(dataset->output_dtypes().size()); bool end_of_sequence; diff --git a/tensorflow/core/kernels/data/writer_ops.cc b/tensorflow/core/kernels/data/writer_ops.cc index 46821fd7b3..656fee1e85 100644 --- a/tensorflow/core/kernels/data/writer_ops.cc +++ b/tensorflow/core/kernels/data/writer_ops.cc @@ -13,9 +13,9 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/common_runtime/device.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/kernels/data/dataset.h" +#include "tensorflow/core/kernels/data/dataset_utils.h" #include "tensorflow/core/kernels/ops_util.h" #include "tensorflow/core/lib/core/threadpool.h" #include "tensorflow/core/lib/io/record_writer.h" @@ -72,21 +72,10 @@ class ToTFRecordOp : public AsyncOpKernel { ctx, GetDatasetFromVariantTensor(ctx->input(0), &dataset), done); auto iterator = dataset->MakeIterator("ToTFRecordOpIterator"); - IteratorContext::Params params; // TODO(b/78245447) - params.env = ctx->env(); - params.runner = *(ctx->runner()); - params.lib = ctx->function_library(); - DeviceBase* device = ctx->function_library()->device(); - params.allocator_getter = [device](AllocatorAttributes attrs) { - return device->GetAllocator(attrs); - }; - - IteratorContext iter_ctx(std::move(params)); - + IteratorContext iter_ctx = dataset::MakeIteratorContext(ctx); std::vector components; components.reserve(dataset->output_dtypes().size()); bool end_of_sequence; - do { OP_REQUIRES_OK_ASYNC( ctx, iterator->GetNext(&iter_ctx, &components, &end_of_sequence), -- GitLab From c2905469335715929c630d2bd70068ccbc8eb2d1 Mon Sep 17 00:00:00 2001 From: manhyuk Date: Fri, 20 Apr 2018 09:28:37 +0900 Subject: [PATCH 702/791] fix typo --- tensorflow/core/grappler/costs/virtual_scheduler.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.h b/tensorflow/core/grappler/costs/virtual_scheduler.h index 5116c8183c..7edd10e3e8 100644 --- a/tensorflow/core/grappler/costs/virtual_scheduler.h +++ b/tensorflow/core/grappler/costs/virtual_scheduler.h @@ -212,7 +212,7 @@ class FirstReadyManager : public ReadyNodeManager { }; // CompositeNodeManager has a few other NodeManagers: per-device LIFO for normal -// ops (neither _Send nor _Recv) and FirstyReadyManagers for _Send ops and _Recv +// ops (neither _Send nor _Recv) and FirstReadyManagers for _Send ops and _Recv // ops, and then it chooses FirstReady among the ops chosen from each // internal NodeManagers. The objective is to maximize producer-consumer // locality within device, while processing nodes across devices, including -- GitLab From 28a95990bf9ff228abec6a52389a4244a17a9101 Mon Sep 17 00:00:00 2001 From: manhyuk Date: Fri, 20 Apr 2018 09:28:45 +0900 Subject: [PATCH 703/791] fix typo --- tensorflow/core/grappler/costs/virtual_scheduler.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.h b/tensorflow/core/grappler/costs/virtual_scheduler.h index 7edd10e3e8..67bf1e6980 100644 --- a/tensorflow/core/grappler/costs/virtual_scheduler.h +++ b/tensorflow/core/grappler/costs/virtual_scheduler.h @@ -199,7 +199,7 @@ class FirstReadyManager : public ReadyNodeManager { // current node. std::vector nodes_; // Newly added nodes are added to waiting_queue_. That way, GetCurrNode(), - // wihch returns the front of the nodes_, always returns the same node, + // which returns the front of the nodes_, always returns the same node, // even if any of new nodes has time_ready smaller than the current node's. std::vector waiting_queue_; // Comparator functor for heap; stl heap is max heap, so we use "greater than" -- GitLab From c18a80967e55350affafbf2ff562056d4bddf234 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 17:26:41 -0700 Subject: [PATCH 704/791] Add support for non-Tensor args in recompute_grad Previously, the function decorated by recompute_grad had to have a signature that contained only positional arguments, and all those arguments had to be Tensors. Most "layers" users define however have non-Tensor arguments (for example, various hyperparameters) and often have keyword arguments as well. This change allows a user to use whatever function signature they wish while being explicit about which arguments are Tensors. PiperOrigin-RevId: 193600682 --- .../layers/python/layers/rev_block_lib.py | 77 +++++++++++-- .../python/layers/rev_block_lib_test.py | 102 ++++++++++++++++++ 2 files changed, 168 insertions(+), 11 deletions(-) diff --git a/tensorflow/contrib/layers/python/layers/rev_block_lib.py b/tensorflow/contrib/layers/python/layers/rev_block_lib.py index 02d294c68f..9f904cc302 100644 --- a/tensorflow/contrib/layers/python/layers/rev_block_lib.py +++ b/tensorflow/contrib/layers/python/layers/rev_block_lib.py @@ -45,6 +45,7 @@ from tensorflow.python.ops import math_ops from tensorflow.python.ops import variable_scope from tensorflow.python.platform import tf_logging as logging from tensorflow.python.util import nest +from tensorflow.python.util import tf_inspect __all__ = ["rev_block", "RevBlock", "recompute_grad"] @@ -429,12 +430,13 @@ def enable_with_args(dec): @enable_with_args -def recompute_grad(fn, use_data_dep=_USE_DEFAULT, tupleize_grads=False): +def recompute_grad(fn, use_data_dep=_USE_DEFAULT, tupleize_grads=False, + tensor_arg_names=None): """Decorator that recomputes the function on the backwards pass. Args: - fn: a function that takes Tensors (all as positional arguments) and returns - a tuple of Tensors. + fn: the subgraph-producing function to wrap and recompute when computing + gradients. Provide `tensor_arg_names` if not all arguments are `Tensor`s. use_data_dep: `bool`, if `True` will use a dummy data dependency to force the recompute to happen. If `False` will use a control dependency. By default will be `True` if in an XLA context and `False` otherwise. XLA @@ -443,17 +445,25 @@ def recompute_grad(fn, use_data_dep=_USE_DEFAULT, tupleize_grads=False): that all gradients are produced before any are consumed by downstream ops. If `use_data_dep` is also `True`, will use a data dependency instead of a control dependency. + tensor_arg_names: `list`, names of the `Tensor` arguments to `fn`. If + `None`, assumes all arguments are `Tensor`s. Returns: A wrapped fn that is identical to fn when called, but its activations will be discarded and recomputed on the backwards pass (i.e. on a call to tf.gradients). """ + if tensor_arg_names: + if not isinstance(tensor_arg_names, (list, tuple)): + raise TypeError("tensor_arg_names must be a list") @functools.wraps(fn) - def wrapped(*args): + def wrapped(*args, **kwargs): + tensor_only_fn, tensor_args = _make_tensor_only(fn, args, kwargs, + tensor_arg_names) return _recompute_grad( - fn, args, use_data_dep=use_data_dep, tupleize_grads=tupleize_grads) + tensor_only_fn, tensor_args, use_data_dep=use_data_dep, + tupleize_grads=tupleize_grads) return wrapped @@ -463,11 +473,59 @@ def _is_on_tpu(): return control_flow_util.GetContainingXLAContext(ctxt) is not None -def _recompute_grad(fn, args, use_data_dep=_USE_DEFAULT, tupleize_grads=False): +def _make_tensor_only(fn, args, kwargs, tensor_arg_names): + """Return fn such that it only takes Tensor args for tensor_arg_names.""" + argspec = tf_inspect.getargspec(fn) + if argspec.varargs is not None or argspec.keywords is not None: + raise ValueError("Function decorated with recompute_grad must not use " + "*args or **kwargs.") + fn_arg_names = list(argspec.args) + + # name_to_arg is a dict of argument name to argument value, including both + # positional and keyword arguments passed. + name_to_arg = {} + # Populate positional arguments. + for name, arg in zip(fn_arg_names[:len(args)], args): + name_to_arg[name] = arg + # Populate keyword arguments. + name_to_arg.update(kwargs) + + # Separate the Tensor arguments from the non-Tensor arguments. + # The default is that all arguments are Tensor arguments. + tensor_arg_names = tensor_arg_names or fn_arg_names + for name in tensor_arg_names: + if name not in name_to_arg: + raise ValueError("Must provide Tensor argument %s" % name) + tensor_args = [name_to_arg[name] for name in tensor_arg_names] + non_tensor_kwargs = dict([(name, arg) for name, arg in name_to_arg.items() + if name not in tensor_arg_names]) + + # Check that Tensor arguments are in fact Tensors and that non-Tensor + # arguments are not. + for name, arg in zip(tensor_arg_names, tensor_args): + if not isinstance(arg, framework_ops.Tensor): + raise TypeError("Fn argument %s must be a Tensor." % name) + for name, arg in non_tensor_kwargs.items(): + if isinstance(arg, framework_ops.Tensor): + raise TypeError("Fn argument %s must not be a Tensor." % name) + + # Construct a Tensor-only wrapper function that will pass the non-Tensor + # arguments as well when called. + def tensor_only_fn(*tensors): + all_kwargs = dict(zip(tensor_arg_names, tensors)) + all_kwargs.update(non_tensor_kwargs) + return fn(**all_kwargs) + + return tensor_only_fn, tensor_args + + +def _recompute_grad(fn, args, use_data_dep=_USE_DEFAULT, + tupleize_grads=False): """See recompute_grad.""" for arg in args: if not isinstance(arg, framework_ops.Tensor): raise ValueError("All inputs to function must be Tensors") + use_data_dep_ = use_data_dep if use_data_dep_ == _USE_DEFAULT: use_data_dep_ = _is_on_tpu() @@ -501,14 +559,11 @@ def _recompute_grad(fn, args, use_data_dep=_USE_DEFAULT, tupleize_grads=False): grad_vars = grads[len(inputs):] return grad_inputs, grad_vars + # TODO(rsepassi): Replace with tf.custom_gradient @_fn_with_custom_grad(grad_fn) def fn_with_recompute(*args): cached_vs.append(variable_scope.get_variable_scope()) - # TODO(rsepassi): Rm conditional in TF 1.4 - if hasattr(contrib_framework_ops, "current_arg_scope"): - cached_arg_scope.append(contrib_framework_ops.current_arg_scope()) - else: - cached_arg_scope.append({}) + cached_arg_scope.append(contrib_framework_ops.current_arg_scope()) return fn(*args) return fn_with_recompute(*args) diff --git a/tensorflow/contrib/layers/python/layers/rev_block_lib_test.py b/tensorflow/contrib/layers/python/layers/rev_block_lib_test.py index 392a490be1..66ccc696f9 100644 --- a/tensorflow/contrib/layers/python/layers/rev_block_lib_test.py +++ b/tensorflow/contrib/layers/python/layers/rev_block_lib_test.py @@ -318,6 +318,108 @@ class RecomputeTest(test.TestCase): self.assertEqual(1, len(grads)) self.assertTrue(grads[0] is not None) + def testWithNontensorArgs(self): + @rev_block_lib.recompute_grad(tupleize_grads=True, + tensor_arg_names=["inputs"]) + def layer_with_recompute(inputs, plus=None): + var = variable_scope.get_variable("var", ()) + self.assertFalse(plus) # called with False below + if plus: + return var + inputs + else: + return var * inputs + + inputs = array_ops.ones((), dtypes.float32) + outputs = layer_with_recompute(inputs, plus=False) + loss = math_ops.square(outputs) + grads = gradients_impl.gradients(loss, variables.trainable_variables()) + self.assertEqual(1, len(grads)) + self.assertTrue(grads[0] is not None) + + +class MakeTensorOnlyTest(test.TestCase): + + def testMakeTensorOnly(self): + def fn(a, b, c, d=1, e=None, f=7): + return (a, b, c, d, e, f) + + t1 = array_ops.ones(()) + t2 = array_ops.ones(()) + t3 = array_ops.ones(()) + args = [1, t1, 3, t2] + kwargs = {"e": t3} + tensor_only_fn, tensor_args = rev_block_lib._make_tensor_only( + fn, args, kwargs, ["b", "d", "e"]) + self.assertAllEqual(tensor_args, [t1, t2, t3]) + out = tensor_only_fn(*tensor_args) + self.assertAllEqual(out, (1, t1, 3, t2, t3, 7)) + + def testMakeTensorOnlyPositionalArgsOnly(self): + def fn(a, b, c): + return (a, b, c) + + t1 = array_ops.ones(()) + t2 = array_ops.ones(()) + args = [t1, 3, t2] + tensor_only_fn, tensor_args = rev_block_lib._make_tensor_only( + fn, args, {}, ["a", "c"]) + self.assertAllEqual(tensor_args, [t1, t2]) + out = tensor_only_fn(*tensor_args) + self.assertAllEqual(out, (t1, 3, t2)) + + def testMakeTensorOnlyKwargsArgsOnly(self): + def fn(a=1, b=2, c=3): + return (a, b, c) + + t1 = array_ops.ones(()) + t2 = array_ops.ones(()) + args = [t1] + kwargs = {"c": t2} + tensor_only_fn, tensor_args = rev_block_lib._make_tensor_only( + fn, args, kwargs, ["a", "c"]) + self.assertAllEqual(tensor_args, [t1, t2]) + out = tensor_only_fn(*tensor_args) + self.assertAllEqual(out, (t1, 2, t2)) + + def testErrorOnMissingTensorArg(self): + def fn(a, b): + return (a, b) + + with self.assertRaisesWithPredicateMatch( + ValueError, "provide Tensor argument"): + rev_block_lib._make_tensor_only(fn, [], {"b": 2}, ["a"]) + + def testErrorOnSignatureSplats(self): + def fn1(a, *args): + return (a, args) + + err_msg = r"must not use \*args or \*\*kwargs" + with self.assertRaisesWithPredicateMatch(ValueError, err_msg): + rev_block_lib._make_tensor_only(fn1, [1, 2], {}, ["a"]) + + def fn2(a, **kwargs): + return (a, kwargs) + + with self.assertRaisesWithPredicateMatch(ValueError, err_msg): + rev_block_lib._make_tensor_only(fn2, [], {"a": 1, "b": 2}, ["a"]) + + def testErrorOnNonTensorForTensor(self): + def fn(a, b): + return (a, b) + + with self.assertRaisesWithPredicateMatch(TypeError, "must be a Tensor"): + rev_block_lib._make_tensor_only(fn, [2, 3], {}, ["a"]) + + def testErrorOnTensorForNonTensor(self): + def fn(a, b): + return (a, b) + + with self.assertRaisesWithPredicateMatch( + TypeError, "must not be a Tensor"): + t1 = array_ops.ones(()) + t2 = array_ops.ones(()) + rev_block_lib._make_tensor_only(fn, [t1, t2], {}, ["a"]) + class FnWithCustomGradTest(test.TestCase): -- GitLab From 13a7e9820a800cf3877e5a44b9f654f79808a2d4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 17:27:04 -0700 Subject: [PATCH 705/791] Update DecodeProtoOp so that it returns explicitly specified default values for missing fields. PiperOrigin-RevId: 193600735 --- .../kernel_tests/defaut_values.TestCase.pbtxt | 94 +++++++++ .../promote_unsigned.TestCase.pbtxt | 10 +- .../python/kernel_tests/test_example.proto | 33 +++ tensorflow/core/kernels/decode_proto_op.cc | 188 +++++++++++++++--- 4 files changed, 300 insertions(+), 25 deletions(-) create mode 100644 tensorflow/contrib/proto/python/kernel_tests/defaut_values.TestCase.pbtxt diff --git a/tensorflow/contrib/proto/python/kernel_tests/defaut_values.TestCase.pbtxt b/tensorflow/contrib/proto/python/kernel_tests/defaut_values.TestCase.pbtxt new file mode 100644 index 0000000000..4e31681907 --- /dev/null +++ b/tensorflow/contrib/proto/python/kernel_tests/defaut_values.TestCase.pbtxt @@ -0,0 +1,94 @@ +primitive { + # No fields specified, so we get all defaults +} +shape: 1 +sizes: 0 +field { + name: "double_default" + dtype: DT_DOUBLE + expected { double_value: 1.0 } +} +sizes: 0 +field { + name: "float_default" + dtype: DT_DOUBLE # Try casting the float field to double. + expected { double_value: 2.0 } +} +sizes: 0 +field { + name: "int64_default" + dtype: DT_INT64 + expected { int64_value: 3 } +} +sizes: 0 +field { + name: "uint64_default" + dtype: DT_INT64 + expected { int64_value: 4 } +} +sizes: 0 +field { + name: "int32_default" + dtype: DT_INT32 + expected { int32_value: 5 } +} +sizes: 0 +field { + name: "fixed64_default" + dtype: DT_INT64 + expected { int64_value: 6 } +} +sizes: 0 +field { + name: "fixed32_default" + dtype: DT_INT32 + expected { int32_value: 7 } +} +sizes: 0 +field { + name: "bool_default" + dtype: DT_BOOL + expected { bool_value: true } +} +sizes: 0 +field { + name: "string_default" + dtype: DT_STRING + expected { string_value: "a" } +} +sizes: 0 +field { + name: "bytes_default" + dtype: DT_STRING + expected { string_value: "a longer default string" } +} +sizes: 0 +field { + name: "uint32_default" + dtype: DT_INT32 + expected { int32_value: -1 } +} +sizes: 0 +field { + name: "sfixed32_default" + dtype: DT_INT32 + expected { int32_value: 10 } +} +sizes: 0 +field { + name: "sfixed64_default" + dtype: DT_INT64 + expected { int64_value: 11 } +} +sizes: 0 +field { + name: "sint32_default" + dtype: DT_INT32 + expected { int32_value: 12 } +} +sizes: 0 +field { + name: "sint64_default" + dtype: DT_INT64 + expected { int64_value: 13 } +} diff --git a/tensorflow/contrib/proto/python/kernel_tests/promote_unsigned.TestCase.pbtxt b/tensorflow/contrib/proto/python/kernel_tests/promote_unsigned.TestCase.pbtxt index db7555bf2d..bc07efc8f3 100644 --- a/tensorflow/contrib/proto/python/kernel_tests/promote_unsigned.TestCase.pbtxt +++ b/tensorflow/contrib/proto/python/kernel_tests/promote_unsigned.TestCase.pbtxt @@ -4,7 +4,6 @@ primitive { } shape: 1 sizes: 1 -sizes: 1 field { name: "fixed32_value" dtype: DT_INT64 @@ -12,6 +11,7 @@ field { int64_value: 4294967295 } } +sizes: 1 field { name: "uint32_value" dtype: DT_INT64 @@ -19,3 +19,11 @@ field { int64_value: 4294967295 } } +sizes: 0 +field { + name: "uint32_default" + dtype: DT_INT64 + expected { + int64_value: 4294967295 # Comes from an explicitly-specified default + } +} diff --git a/tensorflow/contrib/proto/python/kernel_tests/test_example.proto b/tensorflow/contrib/proto/python/kernel_tests/test_example.proto index dc495034ff..a2c88e372b 100644 --- a/tensorflow/contrib/proto/python/kernel_tests/test_example.proto +++ b/tensorflow/contrib/proto/python/kernel_tests/test_example.proto @@ -72,6 +72,23 @@ message RepeatedPrimitiveValue { repeated sint32 sint32_value = 17; repeated sint64 sint64_value = 18; repeated PrimitiveValue message_value = 19; + + // Optional fields with explicitly-specified defaults. + optional double double_default = 20 [default = 1.0]; + optional float float_default = 21 [default = 2.0]; + optional int64 int64_default = 22 [default = 3]; + optional uint64 uint64_default = 23 [default = 4]; + optional int32 int32_default = 24 [default = 5]; + optional fixed64 fixed64_default = 25 [default = 6]; + optional fixed32 fixed32_default = 26 [default = 7]; + optional bool bool_default = 27 [default = true]; + optional string string_default = 28 [default = "a"]; + optional bytes bytes_default = 29 [default = "a longer default string"]; + optional uint32 uint32_default = 30 [default = 4294967295]; + optional sfixed32 sfixed32_default = 31 [default = 10]; + optional sfixed64 sfixed64_default = 32 [default = 11]; + optional sint32 sint32_default = 33 [default = 12]; + optional sint64 sint64_default = 34 [default = 13]; } // A PackedPrimitiveValue looks exactly the same as a RepeatedPrimitiveValue @@ -97,6 +114,22 @@ message PackedPrimitiveValue { repeated sint32 sint32_value = 17 [packed = true]; repeated sint64 sint64_value = 18 [packed = true]; repeated PrimitiveValue message_value = 19; + + optional double double_default = 20 [default = 1.0]; + optional float float_default = 21 [default = 2.0]; + optional int64 int64_default = 22 [default = 3]; + optional uint64 uint64_default = 23 [default = 4]; + optional int32 int32_default = 24 [default = 5]; + optional fixed64 fixed64_default = 25 [default = 6]; + optional fixed32 fixed32_default = 26 [default = 7]; + optional bool bool_default = 27 [default = true]; + optional string string_default = 28 [default = "a"]; + optional bytes bytes_default = 29 [default = "a longer default string"]; + optional uint32 uint32_default = 30 [default = 4294967295]; + optional sfixed32 sfixed32_default = 31 [default = 10]; + optional sfixed64 sfixed64_default = 32 [default = 11]; + optional sint32 sint32_default = 33 [default = 12]; + optional sint64 sint64_default = 34 [default = 13]; } message EnumValue { diff --git a/tensorflow/core/kernels/decode_proto_op.cc b/tensorflow/core/kernels/decode_proto_op.cc index b4e5b776ed..24f8a4f72f 100644 --- a/tensorflow/core/kernels/decode_proto_op.cc +++ b/tensorflow/core/kernels/decode_proto_op.cc @@ -105,11 +105,137 @@ bool CheckOutputType(FieldDescriptor::Type field_type, DataType output_type) { } } +// Used to store the default value of a protocol message field, casted to the +// type of the output tensor. +// +// TODO(paskin): Use absl::variant once TensorFlow gets absl dependencies. +struct DefaultValue { + DataType dtype = DataType::DT_INVALID; + union Value { + bool v_bool; // DT_BOOL + uint8 v_uint8; // DT_UINT8 + int8 v_int8; // DT_INT8 + int32 v_int32; // DT_INT32 + int64 v_int64; // DT_INT64 + float v_float; // DT_FLOAT + double v_double; // DT_DOUBLE + const char* v_string; // DT_STRING + }; + Value value; +}; + +// Initializes a DefaultValue object. This generic template handles numeric +// types and strings are handled by a template specialization below. +// +// Args: +// dtype: the type of the output tensor +// value: the default value as obtained from the FieldDescriptor +// result: the object to initialize +template +Status InitDefaultValue(DataType dtype, const T value, DefaultValue* result) { + result->dtype = dtype; + switch (dtype) { + case DT_BOOL: + result->value.v_bool = static_cast(value); + break; + case DT_INT32: + result->value.v_int32 = static_cast(value); + break; + case DT_INT8: + result->value.v_int8 = static_cast(value); + break; + case DT_UINT8: + result->value.v_uint8 = static_cast(value); + break; + case DT_INT64: + result->value.v_int64 = static_cast(value); + break; + case DT_FLOAT: + result->value.v_float = static_cast(value); + break; + case DT_DOUBLE: + result->value.v_double = static_cast(value); + break; + default: + // We should never get here, given the type checking that occurs earlier. + return errors::Internal( + "Cannot initialize default value for unsupported type: ", + DataTypeString(dtype)); + } + return Status::OK(); +} + +template <> +Status InitDefaultValue(DataType dtype, const char* value, + DefaultValue* result) { + // These are sanity checks that should never trigger given the code that + // leads here. + if (TF_PREDICT_FALSE(dtype != DT_STRING)) { + return errors::InvalidArgument( + "Cannot cast field to anything but DT_STRING"); + } + if (TF_PREDICT_FALSE(value == nullptr)) { + return errors::InvalidArgument("Null default string value."); + } + result->dtype = DT_STRING; + result->value.v_string = value; + return Status::OK(); +} + +// Initializes a default value from the output data type and the field +// descriptor. +Status InitDefaultValueFromFieldDescriptor(DataType dtype, + const FieldDescriptor* field_desc, + DefaultValue* result) { + switch (field_desc->type()) { + case WireFormatLite::TYPE_DOUBLE: + return InitDefaultValue(dtype, field_desc->default_value_double(), + result); + case WireFormatLite::TYPE_FLOAT: + return InitDefaultValue(dtype, field_desc->default_value_float(), result); + case WireFormatLite::TYPE_INT64: + case WireFormatLite::TYPE_SINT64: + case WireFormatLite::TYPE_SFIXED64: + return InitDefaultValue(dtype, field_desc->default_value_int64(), result); + case WireFormatLite::TYPE_FIXED64: + case WireFormatLite::TYPE_UINT64: + return InitDefaultValue(dtype, field_desc->default_value_uint64(), + result); + case WireFormatLite::TYPE_ENUM: + case WireFormatLite::TYPE_INT32: + case WireFormatLite::TYPE_SINT32: + case WireFormatLite::TYPE_SFIXED32: + return InitDefaultValue(dtype, field_desc->default_value_int32(), result); + case WireFormatLite::TYPE_FIXED32: + case WireFormatLite::TYPE_UINT32: + return InitDefaultValue(dtype, field_desc->default_value_uint32(), + result); + case WireFormatLite::TYPE_BOOL: + return InitDefaultValue(dtype, field_desc->default_value_bool(), result); + case WireFormatLite::TYPE_BYTES: + case WireFormatLite::TYPE_STRING: + // Manipulating default string values as C-style pointers should be OK + // for typical code-generated protocol messages. It is possible in + // principle to register a message descriptor on the fly, and these + // pointers may not be stable if that descriptor has a weird + // implementation. (But the return type of default_value_string() is + // const string&, so it'd have to be very weird.) + return InitDefaultValue(dtype, field_desc->default_value_string().c_str(), + result); + case WireFormatLite::TYPE_GROUP: + case WireFormatLite::TYPE_MESSAGE: + return InitDefaultValue(dtype, "", result); + // default: intentionally omitted in order to enable static checking. + } + return Status::OK(); +} + // A FieldInfo holds a handful of information from the FieldDescriptor // and user attributes. struct FieldInfo { - FieldInfo(const FieldDescriptor* field_desc, int user_index) - : output_index(user_index) { + FieldInfo(const FieldDescriptor* field_desc, int user_index, + DefaultValue def_value) + : output_index(user_index), default_value(def_value) { // Without this intermediate data structure, the profile had hotspots // calling methods of FieldDescriptor. number = field_desc->number(); @@ -144,6 +270,7 @@ struct FieldInfo { WireFormatLite::FieldType type; int number; bool is_repeated; + DefaultValue default_value; }; // A CountCollector counts sizes of repeated and optional fields in a proto. @@ -394,8 +521,11 @@ class DenseCollector { DenseCollector() = default; // A DenseCollector applies to one field of a serialized message. - DenseCollector(uint8* datap, DataType dtype, int max_repeat_count) - : datap_(datap), dtype_(dtype), max_repeat_count_(max_repeat_count) {} + // Note that default_value.dtype is the type of the output tensor. + DenseCollector(uint8* datap, DefaultValue default_value, int max_repeat_count) + : datap_(datap), + default_value_(default_value), + max_repeat_count_(max_repeat_count) {} // Reads a value from the input stream and stores it. // @@ -415,8 +545,8 @@ class DenseCollector { } next_repeat_index_ = index + 1; - return internal::ReadValue(input, field.type, field.number, dtype_, index, - datap_); + return internal::ReadValue(input, field.type, field.number, + default_value_.dtype, index, datap_); } // Reads and stores a length-delimited list of values. @@ -445,8 +575,8 @@ class DenseCollector { field.number, ", Max entries allowed: ", max_repeat_count_); } else { return internal::ReadPackedFromArray(buf, buf_size, field.type, - field.number, dtype_, stride, - &next_repeat_index_, datap_); + field.number, default_value_.dtype, + stride, &next_repeat_index_, datap_); } } @@ -454,23 +584,23 @@ class DenseCollector { // Dispatches to the appropriately typed field default based on the // runtime type tag. Status FillWithDefaults() { - switch (dtype_) { + switch (default_value_.dtype) { case DataType::DT_FLOAT: - return FillDefault(); + return FillDefault(default_value_.value.v_float); case DataType::DT_DOUBLE: - return FillDefault(); + return FillDefault(default_value_.value.v_double); case DataType::DT_INT32: - return FillDefault(); + return FillDefault(default_value_.value.v_int32); case DataType::DT_UINT8: - return FillDefault(); + return FillDefault(default_value_.value.v_uint8); case DataType::DT_INT8: - return FillDefault(); + return FillDefault(default_value_.value.v_int8); case DataType::DT_STRING: - return FillDefault(); + return FillDefault(default_value_.value.v_string); case DataType::DT_INT64: - return FillDefault(); + return FillDefault(default_value_.value.v_int64); case DataType::DT_BOOL: - return FillDefault(); + return FillDefault(default_value_.value.v_bool); default: // There are many tensorflow dtypes not handled here, but they // should not come up unless type casting is added to the Op. @@ -485,9 +615,9 @@ class DenseCollector { // default value. This uses next_repeat_index_ which counts the number // of parsed values for the field. template - Status FillDefault() { + Status FillDefault(const T& default_value) { for (int i = next_repeat_index_; i < max_repeat_count_; i++) { - reinterpret_cast(datap_)[i] = T(); + reinterpret_cast(datap_)[i] = default_value; } return Status::OK(); } @@ -501,7 +631,7 @@ class DenseCollector { // for more items than we have allocated space. void* const datap_ = nullptr; - const DataType dtype_ = DataType::DT_INVALID; + const DefaultValue default_value_; const int max_repeat_count_ = 0; }; @@ -577,8 +707,14 @@ class DecodeProtoOp : public OpKernel { // Now store the fields in sorted order. for (int i = 0; i < field_names.size(); i++) { - fields_.push_back(MakeUnique(field_descs[output_indices[i]], - output_indices[i])); + const int output_index = output_indices[i]; + const DataType dtype = output_types[output_index]; + const FieldDescriptor* field_descriptor = field_descs[output_index]; + DefaultValue default_value; + OP_REQUIRES_OK(context, InitDefaultValueFromFieldDescriptor( + dtype, field_descriptor, &default_value)); + fields_.push_back( + MakeUnique(field_descriptor, output_index, default_value)); } message_prototype_ = message_factory_.GetPrototype(message_desc); @@ -805,9 +941,13 @@ class DecodeProtoOp : public OpKernel { std::vector collectors; collectors.reserve(field_count); - for (const TensorInfo& info : tensors) { + for (int output_index = 0; output_index < field_count; ++output_index) { + const TensorInfo& info = tensors[output_index]; + const FieldInfo* field_info = fields_[output_index].get(); + DCHECK(field_info != nullptr); + const DefaultValue default_value = field_info->default_value; collectors.emplace_back(info.data + message_index * info.stride, - info.dtype, info.last_dim_size); + default_value, info.last_dim_size); } // Fill in output tensors from the wire. -- GitLab From 976229dcbfde389864069433ebfc4085015df9c1 Mon Sep 17 00:00:00 2001 From: Austin Anderson Date: Thu, 19 Apr 2018 17:30:49 -0700 Subject: [PATCH 706/791] Internal testing changes PiperOrigin-RevId: 193601134 --- tensorflow/contrib/lite/kernels/BUILD | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tensorflow/contrib/lite/kernels/BUILD b/tensorflow/contrib/lite/kernels/BUILD index 8cfa7e53d1..80cefe83b2 100644 --- a/tensorflow/contrib/lite/kernels/BUILD +++ b/tensorflow/contrib/lite/kernels/BUILD @@ -212,6 +212,7 @@ tf_cc_test( name = "audio_spectrogram_test", size = "small", srcs = ["audio_spectrogram_test.cc"], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -225,6 +226,7 @@ tf_cc_test( name = "mfcc_test", size = "small", srcs = ["mfcc_test.cc"], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -346,6 +348,7 @@ tf_cc_test( name = "cast_test", size = "small", srcs = ["cast_test.cc"], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -398,6 +401,7 @@ tf_cc_test( name = "dequantize_test", size = "small", srcs = ["dequantize_test.cc"], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", @@ -504,6 +508,7 @@ tf_cc_test( name = "maximum_minimum_test", size = "small", srcs = ["maximum_minimum_test.cc"], + tags = ["tflite_not_portable_ios"], deps = [ ":builtin_ops", "//tensorflow/contrib/lite:framework", -- GitLab From 7f87125dceb3c69c5fd1d0712c6c93cc4ceaa854 Mon Sep 17 00:00:00 2001 From: Lukasz Kaiser Date: Thu, 19 Apr 2018 17:39:09 -0700 Subject: [PATCH 707/791] internal END_PUBLIC BEGIN_PUBLIC Automated g4 rollback of changelist 193571934 PiperOrigin-RevId: 193602050 --- tensorflow/core/lib/io/record_reader.cc | 147 ++++++++++---- tensorflow/core/lib/io/record_reader.h | 16 +- tensorflow/core/lib/io/recordio_test.cc | 212 +++++++-------------- tensorflow/core/lib/io/zlib_inputstream.cc | 9 +- tensorflow/core/lib/io/zlib_inputstream.h | 10 +- 5 files changed, 188 insertions(+), 206 deletions(-) diff --git a/tensorflow/core/lib/io/record_reader.cc b/tensorflow/core/lib/io/record_reader.cc index c24628be57..6de850bb20 100644 --- a/tensorflow/core/lib/io/record_reader.cc +++ b/tensorflow/core/lib/io/record_reader.cc @@ -56,55 +56,110 @@ RecordReaderOptions RecordReaderOptions::CreateRecordReaderOptions( RecordReader::RecordReader(RandomAccessFile* file, const RecordReaderOptions& options) - : options_(options), - input_stream_(new RandomAccessInputStream(file)), - last_read_failed_(false) { + : src_(file), options_(options) { if (options.buffer_size > 0) { - input_stream_.reset(new BufferedInputStream(input_stream_.release(), - options.buffer_size, true)); + input_stream_.reset(new BufferedInputStream(file, options.buffer_size)); + } else { + input_stream_.reset(new RandomAccessInputStream(file)); } if (options.compression_type == RecordReaderOptions::ZLIB_COMPRESSION) { // We don't have zlib available on all embedded platforms, so fail. #if defined(IS_SLIM_BUILD) LOG(FATAL) << "Zlib compression is unsupported on mobile platforms."; #else // IS_SLIM_BUILD - input_stream_.reset(new ZlibInputStream( - input_stream_.release(), options.zlib_options.input_buffer_size, - options.zlib_options.output_buffer_size, options.zlib_options, true)); + zlib_input_stream_.reset(new ZlibInputStream( + input_stream_.get(), options.zlib_options.input_buffer_size, + options.zlib_options.output_buffer_size, options.zlib_options)); #endif // IS_SLIM_BUILD } else if (options.compression_type == RecordReaderOptions::NONE) { // Nothing to do. } else { - LOG(FATAL) << "Unrecognized compression type :" << options.compression_type; + LOG(FATAL) << "Unspecified compression type :" << options.compression_type; } } // Read n+4 bytes from file, verify that checksum of first n bytes is // stored in the last 4 bytes and store the first n bytes in *result. -// -// offset corresponds to the user-provided value to ReadRecord() -// and is used only in error messages. -Status RecordReader::ReadChecksummed(uint64 offset, size_t n, string* result) { +// May use *storage as backing store. +Status RecordReader::ReadChecksummed(uint64 offset, size_t n, + StringPiece* result, string* storage) { if (n >= SIZE_MAX - sizeof(uint32)) { return errors::DataLoss("record size too large"); } const size_t expected = n + sizeof(uint32); - TF_RETURN_IF_ERROR(input_stream_->ReadNBytes(expected, result)); + storage->resize(expected); + +#if !defined(IS_SLIM_BUILD) + if (zlib_input_stream_) { + // If we have a zlib compressed buffer, we assume that the + // file is being read sequentially, and we use the underlying + // implementation to read the data. + // + // No checks are done to validate that the file is being read + // sequentially. At some point the zlib input buffer may support + // seeking, possibly inefficiently. + TF_RETURN_IF_ERROR(zlib_input_stream_->ReadNBytes(expected, storage)); + + if (storage->size() != expected) { + if (storage->empty()) { + return errors::OutOfRange("eof"); + } else { + return errors::DataLoss("truncated record at ", offset); + } + } - if (result->size() != expected) { - if (result->empty()) { - return errors::OutOfRange("eof"); + uint32 masked_crc = core::DecodeFixed32(storage->data() + n); + if (crc32c::Unmask(masked_crc) != crc32c::Value(storage->data(), n)) { + return errors::DataLoss("corrupted record at ", offset); + } + *result = StringPiece(storage->data(), n); + } else { +#endif // IS_SLIM_BUILD + if (options_.buffer_size > 0) { + // If we have a buffer, we assume that the file is being read + // sequentially, and we use the underlying implementation to read the + // data. + // + // No checks are done to validate that the file is being read + // sequentially. + TF_RETURN_IF_ERROR(input_stream_->ReadNBytes(expected, storage)); + + if (storage->size() != expected) { + if (storage->empty()) { + return errors::OutOfRange("eof"); + } else { + return errors::DataLoss("truncated record at ", offset); + } + } + + const uint32 masked_crc = core::DecodeFixed32(storage->data() + n); + if (crc32c::Unmask(masked_crc) != crc32c::Value(storage->data(), n)) { + return errors::DataLoss("corrupted record at ", offset); + } + *result = StringPiece(storage->data(), n); } else { - return errors::DataLoss("truncated record at ", offset); + // This version supports reading from arbitrary offsets + // since we are accessing the random access file directly. + StringPiece data; + TF_RETURN_IF_ERROR(src_->Read(offset, expected, &data, &(*storage)[0])); + if (data.size() != expected) { + if (data.empty()) { + return errors::OutOfRange("eof"); + } else { + return errors::DataLoss("truncated record at ", offset); + } + } + const uint32 masked_crc = core::DecodeFixed32(data.data() + n); + if (crc32c::Unmask(masked_crc) != crc32c::Value(data.data(), n)) { + return errors::DataLoss("corrupted record at ", offset); + } + *result = StringPiece(data.data(), n); } +#if !defined(IS_SLIM_BUILD) } +#endif // IS_SLIM_BUILD - const uint32 masked_crc = core::DecodeFixed32(result->data() + n); - if (crc32c::Unmask(masked_crc) != crc32c::Value(result->data(), n)) { - return errors::DataLoss("corrupted record at ", offset); - } - result->resize(n); return Status::OK(); } @@ -112,42 +167,50 @@ Status RecordReader::ReadRecord(uint64* offset, string* record) { static const size_t kHeaderSize = sizeof(uint64) + sizeof(uint32); static const size_t kFooterSize = sizeof(uint32); - // Position the input stream. - int64 curr_pos = input_stream_->Tell(); - int64 desired_pos = static_cast(*offset); - if (curr_pos > desired_pos || curr_pos < 0 /* EOF */ || - (curr_pos == desired_pos && last_read_failed_)) { - last_read_failed_ = false; - TF_RETURN_IF_ERROR(input_stream_->Reset()); - TF_RETURN_IF_ERROR(input_stream_->SkipNBytes(desired_pos)); - } else if (curr_pos < desired_pos) { - TF_RETURN_IF_ERROR(input_stream_->SkipNBytes(desired_pos - curr_pos)); - } - DCHECK_EQ(desired_pos, input_stream_->Tell()); - // Read header data. - Status s = ReadChecksummed(*offset, sizeof(uint64), record); + StringPiece lbuf; + Status s = ReadChecksummed(*offset, sizeof(uint64), &lbuf, record); if (!s.ok()) { - last_read_failed_ = true; return s; } - const uint64 length = core::DecodeFixed64(record->data()); + const uint64 length = core::DecodeFixed64(lbuf.data()); // Read data - s = ReadChecksummed(*offset + kHeaderSize, length, record); + StringPiece data; + s = ReadChecksummed(*offset + kHeaderSize, length, &data, record); if (!s.ok()) { - last_read_failed_ = true; if (errors::IsOutOfRange(s)) { s = errors::DataLoss("truncated record at ", *offset); } return s; } + if (record->data() != data.data()) { + // RandomAccessFile placed the data in some other location. + memmove(&(*record)[0], data.data(), data.size()); + } + + record->resize(data.size()); + *offset += kHeaderSize + length + kFooterSize; - DCHECK_EQ(*offset, input_stream_->Tell()); return Status::OK(); } +Status RecordReader::SkipNBytes(uint64 offset) { +#if !defined(IS_SLIM_BUILD) + if (zlib_input_stream_) { + TF_RETURN_IF_ERROR(zlib_input_stream_->SkipNBytes(offset)); + } else { +#endif + if (options_.buffer_size > 0) { + TF_RETURN_IF_ERROR(input_stream_->SkipNBytes(offset)); + } +#if !defined(IS_SLIM_BUILD) + } +#endif + return Status::OK(); +} // namespace io + SequentialRecordReader::SequentialRecordReader( RandomAccessFile* file, const RecordReaderOptions& options) : underlying_(file, options), offset_(0) {} diff --git a/tensorflow/core/lib/io/record_reader.h b/tensorflow/core/lib/io/record_reader.h index f6d587dfa0..26278e0328 100644 --- a/tensorflow/core/lib/io/record_reader.h +++ b/tensorflow/core/lib/io/record_reader.h @@ -69,14 +69,25 @@ class RecordReader { // Read the record at "*offset" into *record and update *offset to // point to the offset of the next record. Returns OK on success, // OUT_OF_RANGE for end of file, or something else for an error. + // + // Note: if buffering is used (with or without compression), access must be + // sequential. Status ReadRecord(uint64* offset, string* record); + // Skip the records till "offset". Returns OK on success, + // OUT_OF_RANGE for end of file, or something else for an error. + Status SkipNBytes(uint64 offset); + private: - Status ReadChecksummed(uint64 offset, size_t n, string* result); + Status ReadChecksummed(uint64 offset, size_t n, StringPiece* result, + string* storage); + RandomAccessFile* src_; RecordReaderOptions options_; std::unique_ptr input_stream_; - bool last_read_failed_; +#if !defined(IS_SLIM_BUILD) + std::unique_ptr zlib_input_stream_; +#endif // IS_SLIM_BUILD TF_DISALLOW_COPY_AND_ASSIGN(RecordReader); }; @@ -110,6 +121,7 @@ class SequentialRecordReader { return errors::InvalidArgument( "Trying to seek offset: ", offset, " which is less than the current offset: ", offset_); + TF_RETURN_IF_ERROR(underlying_.SkipNBytes(offset - offset_)); offset_ = offset; return Status::OK(); } diff --git a/tensorflow/core/lib/io/recordio_test.cc b/tensorflow/core/lib/io/recordio_test.cc index da514bd21c..63235761d9 100644 --- a/tensorflow/core/lib/io/recordio_test.cc +++ b/tensorflow/core/lib/io/recordio_test.cc @@ -26,11 +26,10 @@ limitations under the License. namespace tensorflow { namespace io { -namespace { // Construct a string of the specified length made out of the supplied // partial string. -string BigString(const string& partial_string, size_t n) { +static string BigString(const string& partial_string, size_t n) { string result; while (result.size() < n) { result.append(partial_string); @@ -40,66 +39,62 @@ string BigString(const string& partial_string, size_t n) { } // Construct a string from a number -string NumberString(int n) { +static string NumberString(int n) { char buf[50]; snprintf(buf, sizeof(buf), "%d.", n); return string(buf); } // Return a skewed potentially long string -string RandomSkewedString(int i, random::SimplePhilox* rnd) { +static string RandomSkewedString(int i, random::SimplePhilox* rnd) { return BigString(NumberString(i), rnd->Skewed(17)); } -class StringDest : public WritableFile { - public: - explicit StringDest(string* contents) : contents_(contents) {} - - Status Close() override { return Status::OK(); } - Status Flush() override { return Status::OK(); } - Status Sync() override { return Status::OK(); } - Status Append(const StringPiece& slice) override { - contents_->append(slice.data(), slice.size()); - return Status::OK(); - } - +class RecordioTest : public ::testing::Test { private: - string* contents_; -}; - -class StringSource : public RandomAccessFile { - public: - explicit StringSource(string* contents) - : contents_(contents), force_error_(false) {} - - Status Read(uint64 offset, size_t n, StringPiece* result, - char* scratch) const override { - if (force_error_) { - force_error_ = false; - return errors::DataLoss("read error"); + class StringDest : public WritableFile { + public: + string contents_; + + Status Close() override { return Status::OK(); } + Status Flush() override { return Status::OK(); } + Status Sync() override { return Status::OK(); } + Status Append(const StringPiece& slice) override { + contents_.append(slice.data(), slice.size()); + return Status::OK(); } - - if (offset >= contents_->size()) { - return errors::OutOfRange("end of file"); - } - - if (contents_->size() < offset + n) { - n = contents_->size() - offset; + }; + + class StringSource : public RandomAccessFile { + public: + StringPiece contents_; + mutable bool force_error_; + mutable bool returned_partial_; + StringSource() : force_error_(false), returned_partial_(false) {} + + Status Read(uint64 offset, size_t n, StringPiece* result, + char* scratch) const override { + EXPECT_FALSE(returned_partial_) << "must not Read() after eof/error"; + + if (force_error_) { + force_error_ = false; + returned_partial_ = true; + return errors::DataLoss("read error"); + } + + if (offset >= contents_.size()) { + return errors::OutOfRange("end of file"); + } + + if (contents_.size() < offset + n) { + n = contents_.size() - offset; + returned_partial_ = true; + } + *result = StringPiece(contents_.data() + offset, n); + return Status::OK(); } - *result = StringPiece(contents_->data() + offset, n); - return Status::OK(); - } - - void force_error() { force_error_ = true; } - - private: - string* contents_; - mutable bool force_error_; -}; + }; -class RecordioTest : public ::testing::Test { - private: - string contents_; StringDest dest_; StringSource source_; bool reading_; @@ -109,9 +104,7 @@ class RecordioTest : public ::testing::Test { public: RecordioTest() - : dest_(&contents_), - source_(&contents_), - reading_(false), + : reading_(false), readpos_(0), writer_(new RecordWriter(&dest_)), reader_(new RecordReader(&source_)) {} @@ -126,11 +119,12 @@ class RecordioTest : public ::testing::Test { TF_ASSERT_OK(writer_->WriteRecord(StringPiece(msg))); } - size_t WrittenBytes() const { return contents_.size(); } + size_t WrittenBytes() const { return dest_.contents_.size(); } string Read() { if (!reading_) { reading_ = true; + source_.contents_ = StringPiece(dest_.contents_); } string record; Status s = reader_->ReadRecord(&readpos_, &record); @@ -143,20 +137,26 @@ class RecordioTest : public ::testing::Test { } } - void IncrementByte(int offset, int delta) { contents_[offset] += delta; } + void IncrementByte(int offset, int delta) { + dest_.contents_[offset] += delta; + } - void SetByte(int offset, char new_byte) { contents_[offset] = new_byte; } + void SetByte(int offset, char new_byte) { + dest_.contents_[offset] = new_byte; + } - void ShrinkSize(int bytes) { contents_.resize(contents_.size() - bytes); } + void ShrinkSize(int bytes) { + dest_.contents_.resize(dest_.contents_.size() - bytes); + } void FixChecksum(int header_offset, int len) { // Compute crc of type/len/data - uint32_t crc = crc32c::Value(&contents_[header_offset + 6], 1 + len); + uint32_t crc = crc32c::Value(&dest_.contents_[header_offset + 6], 1 + len); crc = crc32c::Mask(crc); - core::EncodeFixed32(&contents_[header_offset], crc); + core::EncodeFixed32(&dest_.contents_[header_offset], crc); } - void ForceError() { source_.force_error(); } + void ForceError() { source_.force_error_ = true; } void StartReadingAt(uint64_t initial_offset) { readpos_ = initial_offset; } @@ -165,6 +165,7 @@ class RecordioTest : public ::testing::Test { Write("bar"); Write(BigString("x", 10000)); reading_ = true; + source_.contents_ = StringPiece(dest_.contents_); uint64 offset = WrittenBytes() + offset_past_end; string record; Status s = reader_->ReadRecord(&offset, &record); @@ -216,100 +217,16 @@ TEST_F(RecordioTest, RandomRead) { ASSERT_EQ("EOF", Read()); } -void TestNonSequentialReads(const RecordWriterOptions& writer_options, - const RecordReaderOptions& reader_options) { - string contents; - StringDest dst(&contents); - RecordWriter writer(&dst, writer_options); - for (int i = 0; i < 10; ++i) { - TF_ASSERT_OK(writer.WriteRecord(NumberString(i))) << i; - } - TF_ASSERT_OK(writer.Close()); - - StringSource file(&contents); - RecordReader reader(&file, reader_options); - - string record; - // First read sequentially to fill in the offsets table. - uint64 offsets[10] = {0}; - uint64 offset = 0; - for (int i = 0; i < 10; ++i) { - offsets[i] = offset; - TF_ASSERT_OK(reader.ReadRecord(&offset, &record)) << i; - } - - // Read randomly: First go back to record #3 then forward to #8. - offset = offsets[3]; - TF_ASSERT_OK(reader.ReadRecord(&offset, &record)); - EXPECT_EQ("3.", record); - EXPECT_EQ(offsets[4], offset); - - offset = offsets[8]; - TF_ASSERT_OK(reader.ReadRecord(&offset, &record)); - EXPECT_EQ("8.", record); - EXPECT_EQ(offsets[9], offset); -} - -TEST_F(RecordioTest, NonSequentialReads) { - TestNonSequentialReads(RecordWriterOptions(), RecordReaderOptions()); -} - -TEST_F(RecordioTest, NonSequentialReadsWithReadBuffer) { - RecordReaderOptions options; - options.buffer_size = 1 << 10; - TestNonSequentialReads(RecordWriterOptions(), options); -} - -TEST_F(RecordioTest, NonSequentialReadsWithCompression) { - TestNonSequentialReads( - RecordWriterOptions::CreateRecordWriterOptions("ZLIB"), - RecordReaderOptions::CreateRecordReaderOptions("ZLIB")); -} - // Tests of all the error paths in log_reader.cc follow: -void AssertHasSubstr(StringPiece s, StringPiece expected) { +static void AssertHasSubstr(StringPiece s, StringPiece expected) { EXPECT_TRUE(str_util::StrContains(s, expected)) << s << " does not contain " << expected; } -void TestReadError(const RecordWriterOptions& writer_options, - const RecordReaderOptions& reader_options) { - const string wrote = BigString("well hello there!", 100); - string contents; - StringDest dst(&contents); - TF_ASSERT_OK(RecordWriter(&dst, writer_options).WriteRecord(wrote)); - - StringSource file(&contents); - RecordReader reader(&file, reader_options); - - uint64 offset = 0; - string read; - file.force_error(); - Status status = reader.ReadRecord(&offset, &read); - ASSERT_TRUE(errors::IsDataLoss(status)); - ASSERT_EQ(0, offset); - - // A failed Read() shouldn't update the offset, and thus a retry shouldn't - // lose the record. - status = reader.ReadRecord(&offset, &read); - ASSERT_TRUE(status.ok()) << status; - EXPECT_GT(offset, 0); - EXPECT_EQ(wrote, read); -} - TEST_F(RecordioTest, ReadError) { - TestReadError(RecordWriterOptions(), RecordReaderOptions()); -} - -TEST_F(RecordioTest, ReadErrorWithBuffering) { - RecordReaderOptions options; - options.buffer_size = 1 << 20; - TestReadError(RecordWriterOptions(), options); -} - -TEST_F(RecordioTest, ReadErrorWithCompression) { - TestReadError(RecordWriterOptions::CreateRecordWriterOptions("ZLIB"), - RecordReaderOptions::CreateRecordReaderOptions("ZLIB")); + Write("foo"); + ForceError(); + AssertHasSubstr(Read(), "Data loss"); } TEST_F(RecordioTest, CorruptLength) { @@ -340,6 +257,5 @@ TEST_F(RecordioTest, ReadEnd) { CheckOffsetPastEndReturnsNoRecords(0); } TEST_F(RecordioTest, ReadPastEnd) { CheckOffsetPastEndReturnsNoRecords(5); } -} // namespace } // namespace io } // namespace tensorflow diff --git a/tensorflow/core/lib/io/zlib_inputstream.cc b/tensorflow/core/lib/io/zlib_inputstream.cc index bf8dcf0988..984fbc2810 100644 --- a/tensorflow/core/lib/io/zlib_inputstream.cc +++ b/tensorflow/core/lib/io/zlib_inputstream.cc @@ -25,9 +25,8 @@ ZlibInputStream::ZlibInputStream( InputStreamInterface* input_stream, size_t input_buffer_bytes, // size of z_stream.next_in buffer size_t output_buffer_bytes, // size of z_stream.next_out buffer - const ZlibCompressionOptions& zlib_options, bool owns_input_stream) - : owns_input_stream_(owns_input_stream), - input_stream_(input_stream), + const ZlibCompressionOptions& zlib_options) + : input_stream_(input_stream), input_buffer_capacity_(input_buffer_bytes), output_buffer_capacity_(output_buffer_bytes), z_stream_input_(new Bytef[input_buffer_capacity_]), @@ -42,14 +41,10 @@ ZlibInputStream::~ZlibInputStream() { if (z_stream_) { inflateEnd(z_stream_.get()); } - if (owns_input_stream_) { - delete input_stream_; - } } Status ZlibInputStream::Reset() { TF_RETURN_IF_ERROR(input_stream_->Reset()); - inflateEnd(z_stream_.get()); InitZlibBuffer(); bytes_read_ = 0; return Status::OK(); diff --git a/tensorflow/core/lib/io/zlib_inputstream.h b/tensorflow/core/lib/io/zlib_inputstream.h index 6099e2455d..9c7e14441c 100644 --- a/tensorflow/core/lib/io/zlib_inputstream.h +++ b/tensorflow/core/lib/io/zlib_inputstream.h @@ -40,13 +40,10 @@ class ZlibInputStream : public InputStreamInterface { // Create a ZlibInputStream for `input_stream` with a buffer of size // `input_buffer_bytes` bytes for reading contents from `input_stream` and // another buffer with size `output_buffer_bytes` for caching decompressed - // contents. - // - // Takes ownership of `input_stream` iff `owns_input_stream` is true. + // contents. Does *not* take ownership of "input_stream". ZlibInputStream(InputStreamInterface* input_stream, size_t input_buffer_bytes, size_t output_buffer_bytes, - const ZlibCompressionOptions& zlib_options, - bool owns_input_stream = false); + const ZlibCompressionOptions& zlib_options); ~ZlibInputStream(); @@ -68,8 +65,7 @@ class ZlibInputStream : public InputStreamInterface { private: void InitZlibBuffer(); - const bool owns_input_stream_; - InputStreamInterface* input_stream_; + InputStreamInterface* input_stream_; // Not owned size_t input_buffer_capacity_; // Size of z_stream_input_ size_t output_buffer_capacity_; // Size of z_stream_output_ char* next_unread_byte_; // Next unread byte in z_stream_output_ -- GitLab From b7cca088e90b4c2a28c1038980aa09240584e382 Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Thu, 19 Apr 2018 18:12:57 -0700 Subject: [PATCH 708/791] Respect any device filters in {Create,Delete}WorkerSessions(). This is another step towards enabling us to turn on explicit worker sessions for all master sessions. PiperOrigin-RevId: 193605565 --- tensorflow/core/distributed_runtime/master.cc | 6 +++++- tensorflow/core/distributed_runtime/master_env.h | 3 ++- tensorflow/core/distributed_runtime/master_session.cc | 9 +++++---- tensorflow/core/distributed_runtime/master_session.h | 6 +++++- .../core/distributed_runtime/rpc/grpc_server_lib.cc | 4 +++- 5 files changed, 20 insertions(+), 8 deletions(-) diff --git a/tensorflow/core/distributed_runtime/master.cc b/tensorflow/core/distributed_runtime/master.cc index f47502e844..288656e7f8 100644 --- a/tensorflow/core/distributed_runtime/master.cc +++ b/tensorflow/core/distributed_runtime/master.cc @@ -417,9 +417,13 @@ void Master::CreateSession(const CreateSessionRequest* req, SessionOptions options; options.config = req->config(); + std::vector filtered_worker_list; + DeviceFinder::GetRemoteWorkers(req->config().device_filters(), env_, + worker_cache, &filtered_worker_list); + MasterSession* session = env_->master_session_factory( options, env_, std::move(remote_devices), std::move(worker_cache_ptr), - std::move(device_set)); + std::move(device_set), std::move(filtered_worker_list)); GraphDef* gdef = const_cast(req)->mutable_graph_def(); diff --git a/tensorflow/core/distributed_runtime/master_env.h b/tensorflow/core/distributed_runtime/master_env.h index 178c5b40ee..16f4d93c8b 100644 --- a/tensorflow/core/distributed_runtime/master_env.h +++ b/tensorflow/core/distributed_runtime/master_env.h @@ -83,7 +83,8 @@ struct MasterEnv { SessionOptions, MasterEnv*, std::unique_ptr>>, std::unique_ptr, - std::unique_ptr device_set)> + std::unique_ptr device_set, + std::vector filtered_worker_list)> master_session_factory; std::functionReleaseWorker(part.name, part.worker); + part.worker = nullptr; } return s; } @@ -1119,6 +1120,7 @@ MasterSession::MasterSession( std::unique_ptr>> remote_devs, std::unique_ptr worker_cache, std::unique_ptr device_set, + std::vector filtered_worker_list, StatsPublisherFactory stats_publisher_factory) : session_opts_(opt), env_(env), @@ -1126,6 +1128,7 @@ MasterSession::MasterSession( remote_devs_(std::move(remote_devs)), worker_cache_(std::move(worker_cache)), devices_(std::move(device_set)), + filtered_worker_list_(std::move(filtered_worker_list)), stats_publisher_factory_(std::move(stats_publisher_factory)), graph_version_(0), run_graphs_(5), @@ -1183,9 +1186,8 @@ Status MasterSession::Create(GraphDef* graph_def, Status MasterSession::CreateWorkerSessions( const WorkerCacheFactoryOptions& options) { - std::vector worker_names; + const std::vector worker_names = filtered_worker_list_; WorkerCacheInterface* worker_cache = get_worker_cache(); - worker_cache->ListWorkers(&worker_names); struct WorkerGroup { // The worker name. (Not owned.) @@ -1263,8 +1265,7 @@ Status MasterSession::CreateWorkerSessions( Status MasterSession::DeleteWorkerSessions() { WorkerCacheInterface* worker_cache = get_worker_cache(); - std::vector worker_names; - worker_cache->ListWorkers(&worker_names); + const std::vector& worker_names = filtered_worker_list_; struct WorkerGroup { // The worker name. (Not owned.) diff --git a/tensorflow/core/distributed_runtime/master_session.h b/tensorflow/core/distributed_runtime/master_session.h index a05419904f..ec34e20b79 100644 --- a/tensorflow/core/distributed_runtime/master_session.h +++ b/tensorflow/core/distributed_runtime/master_session.h @@ -52,6 +52,7 @@ class MasterSession : public core::RefCounted { std::unique_ptr>> remote_devs, std::unique_ptr worker_cache, std::unique_ptr device_set, + std::vector filtered_worker_list, StatsPublisherFactory stats_publisher_factory); // Initialize the MasterSession for "def". Must be called before Extend(), @@ -130,6 +131,10 @@ class MasterSession : public core::RefCounted { // The device set used by this session. std::unique_ptr devices_; + // The (partial device) names of remote worker tasks that this + // session will contact. + const std::vector filtered_worker_list_; + StatsPublisherFactory stats_publisher_factory_; std::atomic_ulong last_access_time_usec_; @@ -212,7 +217,6 @@ class MasterSession : public core::RefCounted { // workers. Status CreateWorkerSessions(const WorkerCacheFactoryOptions& server_def); - // TODO(b/36574172): Always use Create/DeleteWorkerSession. bool should_delete_worker_sessions_ = false; Status DeleteWorkerSessions(); diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc b/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc index be19103582..488dcde9f5 100644 --- a/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc +++ b/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc @@ -222,10 +222,12 @@ Status GrpcServer::Init( SessionOptions options, const MasterEnv* env, std::unique_ptr>> remote_devs, std::unique_ptr worker_cache, - std::unique_ptr device_set) { + std::unique_ptr device_set, + std::vector filtered_worker_list) { options.config.MergeFrom(config); return new MasterSession(options, env, std::move(remote_devs), std::move(worker_cache), std::move(device_set), + std::move(filtered_worker_list), stats_factory); }; master_env_.worker_cache_factory = -- GitLab From 4f8768319cfa56c25973cc66d920146ad454bd97 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 18:17:02 -0700 Subject: [PATCH 709/791] Optimize Graph function library. PiperOrigin-RevId: 193605910 --- tensorflow/core/grappler/optimizers/BUILD | 4 + .../grappler/optimizers/function_optimizer.cc | 126 ++++++- .../grappler/optimizers/function_optimizer.h | 6 +- .../optimizers/function_optimizer_test.cc | 32 +- .../grappler/optimizers/meta_optimizer.cc | 326 +++++++++++------- .../core/grappler/optimizers/meta_optimizer.h | 33 +- .../optimizers/meta_optimizer_test.cc | 172 ++++++++- tensorflow/core/grappler/utils/functions.cc | 12 +- tensorflow/core/grappler/utils/functions.h | 40 ++- .../core/grappler/utils/functions_test.cc | 8 +- 10 files changed, 563 insertions(+), 196 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD index a371186fe6..3ab8d8f584 100644 --- a/tensorflow/core/grappler/optimizers/BUILD +++ b/tensorflow/core/grappler/optimizers/BUILD @@ -518,11 +518,13 @@ cc_library( ":loop_optimizer", ":memory_optimizer", ":model_pruner", + "//tensorflow/core:core_cpu_base", "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:protos_all_cc", "//tensorflow/core/grappler:grappler_item", "//tensorflow/core/grappler/utils:colocation", + "//tensorflow/core/grappler/utils:functions", "//tensorflow/core/grappler/utils:topological_sort", ], ) @@ -539,9 +541,11 @@ tf_cuda_cc_test( "//tensorflow/core:tensorflow", "//tensorflow/core:test", "//tensorflow/core:test_main", + "//tensorflow/core:testlib", "//tensorflow/core/grappler:grappler_item", "//tensorflow/core/grappler:utils", "//tensorflow/core/grappler/inputs:trivial_test_graph_input_yielder", + "//tensorflow/core/grappler/utils:grappler_test", ], ) diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc index d008a9719f..950933b933 100644 --- a/tensorflow/core/grappler/optimizers/function_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow/core/framework/function.pb.h" #include "tensorflow/core/framework/graph_def_util.h" #include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/framework/node_def_util.h" #include "tensorflow/core/framework/op_def.pb.h" #include "tensorflow/core/framework/versions.pb.h" #include "tensorflow/core/graph/graph_constructor.h" @@ -75,12 +76,10 @@ string UniqueSpecializedFunctionName(const FunctionDef& func, class FunctionOptimizerContext { public: - explicit FunctionOptimizerContext(const GrapplerItem& item, - RewriterConfig::Toggle opt_level) - : opt_level_(opt_level), - function_library_(FunctionLibraryDefinition(OpRegistry::Global(), - item.graph.library())) { - InitializeInlinedFunctions(item); + explicit FunctionOptimizerContext(RewriterConfig::Toggle opt_level, + const GrapplerItem& item) + : function_library_(OpRegistry::Global(), item.graph.library()) { + InitializeInlinedFunctions(opt_level, item); } const FunctionLibraryDefinition& function_library() const { @@ -101,8 +100,9 @@ class FunctionOptimizerContext { } private: - void InitializeInlinedFunctions(const GrapplerItem& item) { - bool aggressive = opt_level_ == RewriterConfig::AGGRESSIVE; + void InitializeInlinedFunctions(RewriterConfig::Toggle opt_level, + const GrapplerItem& item) { + bool aggressive = opt_level == RewriterConfig::AGGRESSIVE; for (const FunctionDef& func : item.graph.library().function()) { // Can't create IdentityN nodes with no input or output: skip these @@ -120,7 +120,6 @@ class FunctionOptimizerContext { } } - RewriterConfig::Toggle opt_level_; FunctionLibraryDefinition function_library_; // Functions that can be inlined into optimized graph. std::unordered_map inlined_functions_; @@ -128,9 +127,93 @@ class FunctionOptimizerContext { TF_DISALLOW_COPY_AND_ASSIGN(FunctionOptimizerContext); }; +// Return trimmed FunctionDefLibrary with functions that are reachable from +// the optimized graph. +FunctionDefLibrary TrimFunctionLibrary(const FunctionLibraryDefinition& flib, + const GraphDef& optimized_graph) { + // Functions that are reachable from the optimized graph. + std::unordered_set keep_funcs; + + std::vector func_queue; + func_queue.reserve(flib.num_functions()); + + // Add registered and not already processed functions to the queue by name. + const auto add_to_func_queue = [&](const string& func_name) { + const FunctionDef* func = flib.Find(func_name); + if (func && keep_funcs.find(func_name) == keep_funcs.end()) { + func_queue.push_back(func); + } + }; + + // Find all the functions that are reachable from the given node. + const auto add_node_to_func_queue = [&](const NodeDef& node) { + // Node itself can be a call to the function. + add_to_func_queue(node.op()); + + // Or node can have an attribute referencing a function. + for (const auto& attr : node.attr()) { + const auto& attr_value = attr.second; + + // 1. AttrValue.func + if (attr_value.has_func()) { + add_to_func_queue(attr_value.func().name()); + } + + // 2. AttrValue.ListValue.func + if (attr_value.has_list()) { + for (const auto& func : attr_value.list().func()) { + add_to_func_queue(func.name()); + } + } + } + }; + + // Add all functions that are directly called from the optimized graph. + const auto& graph_nodes = optimized_graph.node(); + std::for_each(graph_nodes.begin(), graph_nodes.end(), add_node_to_func_queue); + + // Process all reachable functions. + while (!func_queue.empty()) { + const FunctionDef* func = func_queue.back(); + func_queue.pop_back(); + + const string& func_name = func->signature().name(); + keep_funcs.insert(func_name); + + // Find all the functions that called from the function body. + const auto& func_body = func->node_def(); + std::for_each(func_body.begin(), func_body.end(), add_node_to_func_queue); + + // Check if the function has a registered gradient. + const string grad_func_name = flib.FindGradient(func_name); + if (!grad_func_name.empty()) add_to_func_queue(grad_func_name); + } + + FunctionDefLibrary lib; + for (const string& func_name : keep_funcs) { + const FunctionDef* func = CHECK_NOTNULL(flib.Find(func_name)); + *lib.add_function() = *func; + + const string grad_func_name = flib.FindGradient(func_name); + if (!grad_func_name.empty()) { + GradientDef* gd = lib.add_gradient(); + gd->set_function_name(func_name); + gd->set_gradient_func(grad_func_name); + } + } + + VLOG(3) << "Trimmed function library: " << keep_funcs.size() << " functions (" + << static_cast(keep_funcs.size() - flib.num_functions()) << ")"; + + return lib; +} + Status SpecializeFunction(const NodeDef& func_node, const FunctionDef& func, FunctionOptimizerContext* ctx, GraphDef* optimized_graph) { + VLOG(2) << "Specialize function instantiation: " + << SummarizeNodeDef(func_node); + const std::unordered_map func_attr( func_node.attr().begin(), func_node.attr().end()); @@ -141,20 +224,20 @@ Status SpecializeFunction(const NodeDef& func_node, const FunctionDef& func, TF_RETURN_IF_ERROR(MakeGrapplerFunctionItem(func, func_attr, flib, &item)); // TODO(ezhulenev): Push down const inputs and known input shapes. - FunctionDef specialized; - TF_RETURN_IF_ERROR(MakeSpecializedFunctionDef(item, flib, &specialized)); + FunctionDef specialized_func; + TF_RETURN_IF_ERROR(MakeFunctionDef(item, flib, &specialized_func)); // Find a name for specialized function. const string specialized_func_name = UniqueSpecializedFunctionName(func, func_node, flib); - specialized.mutable_signature()->set_name(specialized_func_name); - auto* specialized_attr = specialized.mutable_attr(); + specialized_func.mutable_signature()->set_name(specialized_func_name); + auto* specialized_attr = specialized_func.mutable_attr(); (*specialized_attr)[kGrapplerSpecializedFuncAttr].set_b(true); // Add specialized function to the library. TF_RETURN_IF_ERROR( - ctx->mutable_function_library().AddFunctionDef(specialized)); + ctx->mutable_function_library().AddFunctionDef(specialized_func)); // Add a function call node for the specialized function. NodeDef* specialized_func_node = optimized_graph->add_node(); @@ -226,6 +309,8 @@ Status HookInlinedFunctionOutputs( Status InlineFunction(const NodeDef& func_node, const FunctionDef& func, const FunctionOptimizerContext& ctx, GraphDef* optimized_graph) { + VLOG(2) << "Inline function instantiation: " << SummarizeNodeDef(func_node); + const std::unordered_map func_attr( func_node.attr().begin(), func_node.attr().end()); @@ -359,6 +444,8 @@ class SymbolicGradientEnv { Status InlineSymbolicGradient(const NodeDef& node, SymbolicGradientEnv* env, GraphDef* inlined_graph) { + VLOG(2) << "Inline symbolic gradient: " << SummarizeNodeDef(node); + GraphDef graph_def; // Create a node to anchor the gradient inputs @@ -454,13 +541,16 @@ Status InlineSymbolicGradient(const NodeDef& node, SymbolicGradientEnv* env, Status FunctionOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item, GraphDef* optimized_graph) { + VLOG(2) << "Optimize function library: id=" << item.id; + // Nothing to do here. if (item.graph.library().function_size() == 0) { + VLOG(3) << "Skip Grappler item with empty function library"; *optimized_graph = item.graph; return Status::OK(); } - FunctionOptimizerContext ctx(item, opt_level_); + FunctionOptimizerContext ctx(opt_level_, item); SymbolicGradientEnv env(item.graph.versions().producer(), item.graph.library()); @@ -506,9 +596,11 @@ Status FunctionOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item, *optimized_graph->add_node() = node; } - // TODO(bsteiner): trim the library to remove unused function definitions *optimized_graph->mutable_versions() = item.graph.versions(); - *optimized_graph->mutable_library() = ctx.function_library().ToProto(); + *optimized_graph->mutable_library() = + options_.enable_trim_function_library + ? TrimFunctionLibrary(ctx.function_library(), *optimized_graph) + : ctx.function_library().ToProto(); return Status::OK(); } diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.h b/tensorflow/core/grappler/optimizers/function_optimizer.h index c555fadf83..e307b4e533 100644 --- a/tensorflow/core/grappler/optimizers/function_optimizer.h +++ b/tensorflow/core/grappler/optimizers/function_optimizer.h @@ -26,8 +26,9 @@ namespace grappler { // operations to make the overall graph more efficient. class FunctionOptimizer : public GraphOptimizer { public: - FunctionOptimizer(RewriterConfig::Toggle opt_level) : opt_level_(opt_level) {} - ~FunctionOptimizer() override {} + explicit FunctionOptimizer(RewriterConfig::Toggle opt_level) + : opt_level_(opt_level) {} + ~FunctionOptimizer() override = default; string name() const override { return "function_optimizer"; }; @@ -44,6 +45,7 @@ class FunctionOptimizer : public GraphOptimizer { bool enable_function_inlining = true; bool enable_function_specialization = true; bool enable_symbolic_gradient_inlining = true; + bool enable_trim_function_library = true; }; RewriterConfig::Toggle opt_level_; diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc index fb006d4868..6147e8a27c 100644 --- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc +++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc @@ -31,20 +31,8 @@ constexpr char kDevice[] = "/device:CPU:0"; class FunctionOptimizerTest : public GrapplerTest { protected: - void DisableAll(FunctionOptimizer* optimizer) { - optimizer->options_.enable_function_inlining = false; + void DisableFunctionSpecialization(FunctionOptimizer* optimizer) { optimizer->options_.enable_function_specialization = false; - optimizer->options_.enable_symbolic_gradient_inlining = false; - } - - void EnableOnlyFunctionInlining(FunctionOptimizer* optimizer) { - DisableAll(optimizer); - optimizer->options_.enable_function_inlining = true; - } - - void EnableOnlyFunctionSpecialization(FunctionOptimizer* optimizer) { - DisableAll(optimizer); - optimizer->options_.enable_function_specialization = true; } }; @@ -352,7 +340,7 @@ TEST_F(FunctionOptimizerTest, InlineFunction_FunctionWithoutInput) { using test::function::NDef; FunctionOptimizer optimizer(RewriterConfig::DEFAULT); - EnableOnlyFunctionInlining(&optimizer); + DisableFunctionSpecialization(&optimizer); // do not specialize noinline func const Tensor kTwo = test::AsScalar(2); FunctionDef func = FunctionDefHelper::Define( @@ -626,14 +614,13 @@ TEST_F(FunctionOptimizerTest, SpecializeFunction_XTimesTwo) { using test::function::NDef; FunctionOptimizer optimizer(RewriterConfig::DEFAULT); - EnableOnlyFunctionSpecialization(&optimizer); - // Mark XTimesTwo as noinline + // Mark XTimesTwo as noinline. FunctionDef x_times_two = test::function::XTimesTwo(); (*x_times_two.mutable_attr())["_noinline"].set_b(true); std::vector function_library = {x_times_two}; - // Build a graph to compute y = XTimesTwo(x) + // Build a graph to compute y = XTimesTwo(x). GrapplerItem item; item.graph = test::function::GDef( {NDef("x", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice), @@ -644,12 +631,13 @@ TEST_F(FunctionOptimizerTest, SpecializeFunction_XTimesTwo) { GraphDef output; TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &output)); - // Make sure that specialized function was added to the library - EXPECT_EQ(2, output.library().function_size()); + // Make sure that specialized function was added to the library and original + // function was removed. + EXPECT_EQ(1, output.library().function_size()); EXPECT_EQ("XTimesTwo_specialized_for_y", - output.library().function(1).signature().name()); + output.library().function(0).signature().name()); - // And 'y' node is calling specialized function + // And 'y' node is calling specialized function. int count = 0; for (const NodeDef& node : output.node()) { if (node.name() == "y" && count++) { @@ -658,7 +646,7 @@ TEST_F(FunctionOptimizerTest, SpecializeFunction_XTimesTwo) { } EXPECT_EQ(1, count); - // And that graph evaluation yields the same result + // And that graph evaluation yields the same result. Tensor pi = test::AsScalar(3.14f); item.fetch = {"z"}; item.feed.emplace_back("x", pi); diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc index 558b8a77e8..22799311bc 100644 --- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc @@ -14,6 +14,7 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/grappler/optimizers/meta_optimizer.h" +#include "tensorflow/core/common_runtime/function.h" #include "tensorflow/core/framework/function.pb.h" #include "tensorflow/core/framework/versions.pb.h" #include "tensorflow/core/grappler/optimizers/arithmetic_optimizer.h" @@ -29,6 +30,7 @@ limitations under the License. #include "tensorflow/core/grappler/optimizers/memory_optimizer.h" #include "tensorflow/core/grappler/optimizers/model_pruner.h" #include "tensorflow/core/grappler/utils/colocation.h" +#include "tensorflow/core/grappler/utils/functions.h" #include "tensorflow/core/grappler/utils/topological_sort.h" #include "tensorflow/core/lib/core/status.h" @@ -36,6 +38,9 @@ namespace tensorflow { namespace grappler { namespace { + +constexpr int kDefaultNumberOfIterations = 1; + int64 NumEdges(const GraphDef& graph) { int64 num_edges = 0; for (const auto& node : graph.node()) { @@ -50,144 +55,138 @@ string PrintSizesBeforeAfter(const GraphDef& before, const GraphDef& after) { NumEdges(after), " edges (", NumEdges(after) - NumEdges(before), ")"); } + +int NumIterations(const RewriterConfig& cfg) { + return cfg.meta_optimizer_iterations() == RewriterConfig::DEFAULT_NUM_ITERS + ? kDefaultNumberOfIterations + : cfg.meta_optimizer_iterations(); +} + +// Check if optimizer is allowed to run only once. +int IsRunOnceOptimizer(const string& name) { return name == "layout"; } + } // namespace -std::unique_ptr MetaOptimizer::NewOptimizer( - const string& optimizer) { - std::unique_ptr graph_optimizer; - if (optimizer == "pruning") { - graph_optimizer.reset(new ModelPruner()); - } - if (optimizer == "function") { - graph_optimizer.reset(new FunctionOptimizer(cfg_.function_optimization())); +std::unique_ptr MetaOptimizer::MakeNewOptimizer( + const string& optimizer) const { +#define MK_OPT(NAME, VALUE) \ + if (optimizer == NAME) return std::unique_ptr(VALUE) + + MK_OPT("pruning", new ModelPruner()); + MK_OPT("function", new FunctionOptimizer(cfg_.function_optimization())); + MK_OPT("constfold", new ConstantFolding(cpu_device_)); + MK_OPT("layout", new LayoutOptimizer()); + MK_OPT("memory", new MemoryOptimizer(RewriterConfig::MANUAL)); + MK_OPT("arithmetic", new ArithmeticOptimizer(cfg_.arithmetic_optimization())); + MK_OPT("autoparallel", new AutoParallel(cfg_.auto_parallel().num_replicas())); + MK_OPT("loop", new LoopOptimizer(cfg_.loop_optimization())); + MK_OPT("dependency", new DependencyOptimizer(cfg_.dependency_optimization())); + MK_OPT("debug_stripper", new DebugStripper()); + + return std::unique_ptr(); +#undef MK_OPT +} + +Status MetaOptimizer::InitializeOptimizers( + std::vector>* optimizers) const { + if (!cfg_.disable_model_pruning()) { + optimizers->emplace_back(new ModelPruner()); } - if (optimizer == "constfold") { - graph_optimizer.reset(new ConstantFolding(cpu_device_)); + if (cfg_.function_optimization() != RewriterConfig::OFF) { + optimizers->emplace_back( + new FunctionOptimizer(cfg_.function_optimization())); } - if (optimizer == "layout") { - graph_optimizer.reset(new LayoutOptimizer()); + if (cfg_.debug_stripper() == RewriterConfig::ON) { + optimizers->emplace_back(new DebugStripper()); } - if (optimizer == "memory") { - graph_optimizer.reset(new MemoryOptimizer(RewriterConfig::MANUAL)); + if (cfg_.constant_folding() != RewriterConfig::OFF) { + optimizers->emplace_back( + new ConstantFolding(cfg_.constant_folding(), cpu_device_)); } - if (optimizer == "arithmetic") { - graph_optimizer.reset( + if (cfg_.arithmetic_optimization() != RewriterConfig::OFF) { + optimizers->emplace_back( new ArithmeticOptimizer(cfg_.arithmetic_optimization())); } - if (optimizer == "autoparallel") { - graph_optimizer.reset( - new AutoParallel(cfg_.auto_parallel().num_replicas())); - } - if (optimizer == "loop") { - graph_optimizer.reset(new LoopOptimizer(cfg_.loop_optimization())); + if (cfg_.loop_optimization() != RewriterConfig::OFF) { + optimizers->emplace_back(new LoopOptimizer(cfg_.loop_optimization())); } - if (optimizer == "dependency") { - graph_optimizer.reset( + if (cfg_.dependency_optimization() != RewriterConfig::OFF) { + optimizers->emplace_back( new DependencyOptimizer(cfg_.dependency_optimization())); } - if (optimizer == "debug_stripper") { - graph_optimizer.reset(new DebugStripper()); + if (cfg_.layout_optimizer() != RewriterConfig::OFF) { + optimizers->emplace_back(new LayoutOptimizer()); + } + if (cfg_.memory_optimization() != RewriterConfig::NO_MEM_OPT) { + if (cfg_.memory_optimizer_target_node_name_scope().empty()) { + optimizers->emplace_back( + // Use the default target node name prefix "gradients/" + new MemoryOptimizer(cfg_.memory_optimization())); + } else { + optimizers->emplace_back( + new MemoryOptimizer(cfg_.memory_optimization(), + cfg_.memory_optimizer_target_node_name_scope())); + } + } + if (cfg_.auto_parallel().enable()) { + optimizers->emplace_back( + new AutoParallel(cfg_.auto_parallel().num_replicas())); } - return graph_optimizer; + return Status::OK(); } -Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item, - GraphDef* optimized_graph) { - std::vector> optimizers; - if (cfg_.optimizers().empty()) { - if (!cfg_.disable_model_pruning()) { - optimizers.push_back(std::unique_ptr(new ModelPruner())); - } - if (cfg_.function_optimization() != RewriterConfig::OFF) { - optimizers.push_back(std::unique_ptr( - new FunctionOptimizer(cfg_.function_optimization()))); - } - if (cfg_.debug_stripper() == RewriterConfig::ON) { - optimizers.push_back( - std::unique_ptr(new DebugStripper())); - } - if (cfg_.constant_folding() != RewriterConfig::OFF) { - optimizers.push_back(std::unique_ptr( - new ConstantFolding(cfg_.constant_folding(), cpu_device_))); - } - if (cfg_.arithmetic_optimization() != RewriterConfig::OFF) { - optimizers.push_back(std::unique_ptr( - new ArithmeticOptimizer(cfg_.arithmetic_optimization()))); +Status MetaOptimizer::InitializeOptimizersByName( + std::vector>* optimizers) const { + for (const string& optimizer_name : cfg_.optimizers()) { + auto optimizer = MakeNewOptimizer(optimizer_name); + if (optimizer) { + VLOG(2) << "Registered default graph optimizer: " << optimizer_name; + optimizers->push_back(std::move(optimizer)); + continue; } - if (cfg_.loop_optimization() != RewriterConfig::OFF) { - optimizers.push_back(std::unique_ptr( - new LoopOptimizer(cfg_.loop_optimization()))); - } - if (cfg_.dependency_optimization() != RewriterConfig::OFF) { - optimizers.push_back(std::unique_ptr( - new DependencyOptimizer(cfg_.dependency_optimization()))); - } - if (cfg_.layout_optimizer() != RewriterConfig::OFF) { - optimizers.push_back( - std::unique_ptr(new LayoutOptimizer())); - } - if (cfg_.memory_optimization() != RewriterConfig::NO_MEM_OPT) { - if (cfg_.memory_optimizer_target_node_name_scope().empty()) { - optimizers.push_back(std::unique_ptr( - // Use the default target node name prefix "gradients/" - new MemoryOptimizer(cfg_.memory_optimization()))); - } else { - optimizers.push_back( - std::unique_ptr(new MemoryOptimizer( - cfg_.memory_optimization(), - cfg_.memory_optimizer_target_node_name_scope()))); - } - } - if (cfg_.auto_parallel().enable()) { - optimizers.push_back(std::unique_ptr( - new AutoParallel(cfg_.auto_parallel().num_replicas()))); - } - } else { - const std::set available_optimizers = { - "pruning", "function", "constfold", "layout", - "memory", "autoparallel", "arithmetic", "loop", - "dependency", "debug_stripper"}; - std::vector custom_optimizer_names; - for (const auto& optimizer_name : cfg_.optimizers()) { - if (available_optimizers.find(optimizer_name) != - available_optimizers.end()) { - optimizers.push_back(NewOptimizer(optimizer_name)); - } else { - custom_optimizer_names.push_back(optimizer_name); - } - } - // Now run the custom optimizers. - for (const auto& optimizer_name : custom_optimizer_names) { - std::unique_ptr opt = - CustomGraphOptimizerRegistry::CreateByNameOrNull(optimizer_name); - if (opt == nullptr) continue; - TF_RETURN_IF_ERROR(opt->Init()); - optimizers.push_back(std::move(opt)); + + auto custom_optimizer = + CustomGraphOptimizerRegistry::CreateByNameOrNull(optimizer_name); + + if (custom_optimizer) { + VLOG(2) << "Registered custom graph optimizer: " << optimizer_name; + TF_RETURN_IF_ERROR(custom_optimizer->Init()); + optimizers->push_back(std::move(custom_optimizer)); + } else { + VLOG(2) << "Can't register an optimizer by name: " << optimizer_name; } } + return Status::OK(); +} + +Status MetaOptimizer::OptimizeGraph(Cluster* cluster, const GrapplerItem& item, + GraphDef* optimized_graph) { + VLOG(2) << "Optimize GrapplerItem: item.id=" << item.id; + + std::vector> optimizers; + bool register_by_name = !cfg_.optimizers().empty(); + TF_RETURN_IF_ERROR(register_by_name ? InitializeOptimizersByName(&optimizers) + : InitializeOptimizers(&optimizers)); if (optimizers.empty()) { *optimized_graph = item.graph; return Status::OK(); } - // Some optimizers should be run only once. - const std::set run_once_optimizers = {"layout"}; - bool already_optimized = false; - const int num_iterations = - cfg_.meta_optimizer_iterations() == RewriterConfig::DEFAULT_NUM_ITERS - ? 1 - : cfg_.meta_optimizer_iterations(); + // Invariant: optimized_graph contains the most recently optimized version of + // the graph. GrapplerItem optimized_item = item; optimized_graph->Swap(&optimized_item.graph); - for (int iteration = 0; iteration < num_iterations; ++iteration) { - VLOG(1) << "Starting optimization iteration " << iteration + 1; + + GraphOptimizationResult optimization_result(item.id); + + for (int iteration = 0; iteration < NumIterations(cfg_); ++iteration) { + VLOG(4) << "Starting optimization iteration " << iteration + 1; + for (const auto& optimizer : optimizers) { - // Invariant: optimized_graph contains the most recently optimized - // version of the graph. - if (iteration > 0 && run_once_optimizers.count(optimizer->name())) { - continue; - } + // Some optimizers can run only once. + if (iteration > 0 && IsRunOnceOptimizer(optimizer->name())) continue; + uint64 start_us = Env::Default()->NowMicros(); // This swaps the current optimized_graph into optimized item and // resets optimized_graph to an empty graph. @@ -195,45 +194,114 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item, *optimized_graph = GraphDef(); Status status = optimizer->Optimize(cluster, optimized_item, optimized_graph); - uint64 end_us = Env::Default()->NowMicros(); - float duration_ms = (end_us - start_us) / 1000.0f; + string result; if (!status.ok()) { - VLOG(1) << "Not able to apply optimizer " << optimizer->name() << ": " - << status.ToString(); optimized_graph->Swap(&optimized_item.graph); result = status.ToString(); } else { - already_optimized = true; + optimization_result.is_optimized = true; + float duration_ms = (end_us - start_us) / 1000.0f; result = strings::StrCat( - optimizer->name(), ": ", PrintSizesBeforeAfter(optimized_item.graph, *optimized_graph), ", time = ", duration_ms, "ms."); } - result_.emplace_back(optimizer->name(), result); - VLOG(1) << result; + VLOG(4) << optimizer->name() << ": " << result; + + OptimizerResult optimizer_result{optimizer->name(), result}; + optimization_result.results.push_back(optimizer_result); } } - if (already_optimized) { + // Record graph optimization result. + optimization_results_.push_back(optimization_result); + + if (optimization_result.is_optimized) { TF_RETURN_IF_ERROR(TopologicalSort(optimized_graph)); ReassignColocation(optimized_graph); - // Make sure that the optimizers preserved the graph version and library. - DCHECK_GE(optimized_graph->library().function_size(), - item.graph.library().function_size()); - DCHECK_GE(optimized_graph->library().gradient_size(), - item.graph.library().gradient_size()); + // Make sure that the optimizers preserved the graph version. DCHECK_EQ(optimized_graph->versions().producer(), item.graph.versions().producer()); } + + return Status::OK(); +} + +Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item, + GraphDef* optimized_graph) { + optimization_results_.clear(); + + // 1. Optimize main graph + TF_RETURN_IF_ERROR(OptimizeGraph(cluster, item, optimized_graph)); + + // 2. Optimize function library + FunctionLibraryDefinition flib(OpRegistry::Global(), + optimized_graph->library()); + + // Optimize each function only once. + std::unordered_set optimized_funcs; + bool optimize_function_library = true; + + while (optimize_function_library) { + optimize_function_library = false; + + for (const FunctionDef& func : optimized_graph->library().function()) { + const string& func_name = func.signature().name(); + + // Skip already optimized functions. + if (optimized_funcs.find(func_name) != optimized_funcs.end()) continue; + + // Skip parametrized functions (function type or body is defined only at + // function call time by caller node attributes). + if (IsParametrized(func)) continue; + + VLOG(3) << "Optimize function: function=" << func_name; + + // Function optimization might specialize nested function calls, so we + // have to reset the flag and do at least one more pass over the library. + optimize_function_library = true; + optimized_funcs.insert(func_name); + + // Make a GrapplerItem from a FunctionDef. + GrapplerFunctionItem func_item; + TF_RETURN_IF_ERROR(MakeGrapplerFunctionItem(func, flib, &func_item)); + + // Optimize function body graph. + GraphDef optimized_func_graph; + TF_RETURN_IF_ERROR( + OptimizeGraph(cluster, func_item, &optimized_func_graph)); + + // Function body optimization might have created new specialized + // functions, add them to the library. + TF_RETURN_IF_ERROR(flib.AddLibrary(optimized_func_graph.library())); + + // Convert optimized graph back to FunctionDef. + FunctionDef optimized_func; + func_item.SwapFunctionBody(std::move(optimized_func_graph)); + TF_RETURN_IF_ERROR(MakeFunctionDef(func_item, flib, &optimized_func)); + + // Replace optimized function with a new FunctionDef. + TF_RETURN_IF_ERROR(flib.RemoveFunction(func_name)); + TF_RETURN_IF_ERROR(flib.AddFunctionDef(optimized_func)); + } + + // If optimized at least one function, update the graph library. + if (optimize_function_library) { + *optimized_graph->mutable_library() = flib.ToProto(); + } + } + return Status::OK(); } void MetaOptimizer::PrintResult() { - for (const auto& result : result_) { - LOG(INFO) << "Return status of optimizer " << result.first << ": " - << result.second; + for (const GraphOptimizationResult& graph_result : optimization_results_) { + LOG(INFO) << "Optimization results for grappler item: " << graph_result.id; + for (const OptimizerResult& result : graph_result.results) { + LOG(INFO) << "Return status of optimizer " << result.optimizer_name + << ": " << result.result; + } } } diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.h b/tensorflow/core/grappler/optimizers/meta_optimizer.h index 382cfe51d4..7cf9a40c2d 100644 --- a/tensorflow/core/grappler/optimizers/meta_optimizer.h +++ b/tensorflow/core/grappler/optimizers/meta_optimizer.h @@ -30,7 +30,7 @@ class MetaOptimizer : public GraphOptimizer { public: MetaOptimizer(DeviceBase* cpu_device, const RewriterConfig& cfg) : cpu_device_(cpu_device), cfg_(cfg) {} - ~MetaOptimizer() override {} + ~MetaOptimizer() override = default; string name() const override { return "meta_optimizer"; }; @@ -43,10 +43,37 @@ class MetaOptimizer : public GraphOptimizer { const GraphDef& optimized_graph, double result) override; private: - std::unique_ptr NewOptimizer(const string& optimizer); + std::unique_ptr MakeNewOptimizer( + const string& optimizer) const; + + // Initialize active optimizers from RewriterConfig toggles. + Status InitializeOptimizers( + std::vector>* optimizers) const; + // Initialize active optimizers from RewriterConfig optimizer names. + Status InitializeOptimizersByName( + std::vector>* optimizers) const; + + // Run optimization pass over a single GrapplerItem. Meta optimizer might run + // multiple such passes: 1) for the main graph 2) for the function library + Status OptimizeGraph(Cluster* cluster, const GrapplerItem& item, + GraphDef* optimized_graph); + DeviceBase* const cpu_device_; // may be NULL RewriterConfig cfg_; - std::vector> result_; + + struct OptimizerResult { + string optimizer_name; + string result; + }; + + struct GraphOptimizationResult { + explicit GraphOptimizationResult(const string& id) : id(id) {} + string id; + bool is_optimized = false; + std::vector results; + }; + + std::vector optimization_results_; }; bool MetaOptimizerEnabled(const RewriterConfig& cfg); diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc b/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc index d9a386b9be..8793ad9633 100644 --- a/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc +++ b/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc @@ -16,11 +16,14 @@ limitations under the License. #include "tensorflow/core/grappler/optimizers/meta_optimizer.h" #include "tensorflow/cc/ops/standard_ops.h" +#include "tensorflow/core/framework/function_testlib.h" +#include "tensorflow/core/framework/tensor_testutil.h" #include "tensorflow/core/grappler/grappler_item.h" #include "tensorflow/core/grappler/inputs/trivial_test_graph_input_yielder.h" #include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h" #include "tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.h" #include "tensorflow/core/grappler/utils.h" +#include "tensorflow/core/grappler/utils/grappler_test.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/platform/test.h" @@ -28,6 +31,8 @@ namespace tensorflow { namespace grappler { namespace { +constexpr char kDevice[] = "/device:CPU:0"; + class TestOptimizer : public CustomGraphOptimizer { public: static void SetOptimized(const bool flag_value) { optimized_ = flag_value; } @@ -56,7 +61,9 @@ bool TestOptimizer::optimized_; REGISTER_GRAPH_OPTIMIZER(TestOptimizer); -TEST(MetaOptimizerTest, RunsCustomOptimizer) { +class MetaOptimizerTest : public GrapplerTest {}; + +TEST_F(MetaOptimizerTest, RunsCustomOptimizer) { TrivialTestGraphInputYielder fake_input(4, 1, 10, false, {"CPU:0"}); GrapplerItem item; CHECK(fake_input.NextItem(&item)); @@ -72,7 +79,7 @@ TEST(MetaOptimizerTest, RunsCustomOptimizer) { EXPECT_TRUE(TestOptimizer::IsOptimized()); } -TEST(MetaOptimizerTest, RunOptimizersTwice) { +TEST_F(MetaOptimizerTest, RunOptimizersTwice) { TrivialTestGraphInputYielder fake_input(4, 1, 10, false, {"CPU:0"}); GrapplerItem item; CHECK(fake_input.NextItem(&item)); @@ -86,6 +93,167 @@ TEST(MetaOptimizerTest, RunOptimizersTwice) { TF_EXPECT_OK(status); } +TEST_F(MetaOptimizerTest, OptimizeFunctionLibrary) { + using test::function::NDef; + + // Enable ony function optimization. + RewriterConfig rewriter_config; + rewriter_config.set_meta_optimizer_iterations(RewriterConfig::TWO); + rewriter_config.set_function_optimization(RewriterConfig::ON); + rewriter_config.add_optimizers("function"); + + MetaOptimizer optimizer(nullptr, rewriter_config); + + // Define function library: + // + // MyMul(x, y) = x * y + // *MySquare(x) = MyMul(x, x) + // *MyQuadratic(x) = MySquare(MySquare(x)) + // + // * - marked as noinline + + FunctionDef mul_func = FunctionDefHelper::Create( + "MyMul", {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"}, + {{{"mul"}, "Mul", {"x", "y"}, {{"T", "$T"}}}}, + /* Mapping between function returns and function node outputs. */ + {{"z", "mul:z:0"}}); + + FunctionDef square_func = FunctionDefHelper::Create( + "MySquare", {"x:T"}, {"z:T"}, {"T: {float, double}"}, + {{{"my_mul"}, "MyMul", {"x", "x"}, {{"T", "$T"}}}}, + /* Mapping between function returns and function node outputs. */ + {{"z", "my_mul:z:0"}}); + (*square_func.mutable_attr())["_noinline"].set_b(true); + + FunctionDef quadratic_func = FunctionDefHelper::Create( + "MyQuadratic", {"x:T"}, {"z:T"}, {"T: {float, double}"}, + {{{"square"}, "MySquare", {"x"}, {{"T", "$T"}}}, + {{"quadratic"}, "MySquare", {"square:z"}, {{"T", "$T"}}}}, + /* Mapping between function returns and function node outputs. */ + {{"z", "quadratic:z:0"}}); + (*quadratic_func.mutable_attr())["_noinline"].set_b(true); + + // Tensorflow graph: + // + // a = tf.Placeholder(tf.float); + // b = tf.Placeholder(tf.int32); + // + // square = MySquare(a); // a^2 + // quadratic = MyQuadratic(b); // b^4 + GrapplerItem item; + item.graph = test::function::GDef( + {NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice), + NDef("b", "Placeholder", {}, {{"dtype", DT_INT32}}, kDevice), + // Calls into function library + NDef("square", "MySquare", {"a"}, {{"T", DT_FLOAT}}, kDevice), + NDef("quadratic", "MyQuadratic", {"b"}, {{"T", DT_INT32}}, kDevice), + // Forward outputs + NDef("out_s", "Identity", {"square:0"}, {{"T", DT_FLOAT}}, kDevice), + NDef("out_q", "Identity", {"quadratic:0"}, {{"T", DT_INT32}}, kDevice)}, + // FunctionLib + {mul_func, square_func, quadratic_func}); + + GraphDef output; + TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &output)); + + FunctionLibraryDefinition optimized_flib(OpRegistry::Global(), + output.library()); + + // Specialized and optimized functions should be added to the graph. + EXPECT_EQ(6, optimized_flib.num_functions()); + + // MyQuadratic should be specialized once: + // 0. 'quadratic' node in the main graph + const string optimized_0 = "MyQuadratic_specialized_for_quadratic"; + + // MySquare should be specialized and optimized for 3 instantiations: + // 1. 'square' node in the main graph + // 2. 'square' node in the MyQuadratic specialization + // 3. 'quadratic' node in the MyQuadratic specialization + + const string optimized_1 = "MySquare_specialized_for_square"; + const string optimized_2 = "MySquare_specialized_for_square_1"; + const string optimized_3 = "MySquare_specialized_for_quadratic"; + + const FunctionDef* optimized_func_0 = optimized_flib.Find(optimized_0); + const FunctionDef* optimized_func_1 = optimized_flib.Find(optimized_1); + const FunctionDef* optimized_func_2 = optimized_flib.Find(optimized_2); + const FunctionDef* optimized_func_3 = optimized_flib.Find(optimized_3); + + ASSERT_NE(optimized_func_0, nullptr); + ASSERT_NE(optimized_func_1, nullptr); + ASSERT_NE(optimized_func_2, nullptr); + ASSERT_NE(optimized_func_3, nullptr); + + // Graph should call optimized function. + int count = 0; + for (const NodeDef& node : output.node()) { + if (node.name() == "square" && count++) { + EXPECT_EQ("MySquare_specialized_for_square", node.op()); + } else if (node.name() == "quadratic" && count++) { + EXPECT_EQ("MyQuadratic_specialized_for_quadratic", node.op()); + } + } + EXPECT_EQ(2, count); + + // Specialized MySquare should call specialized functions. + count = 0; + for (const NodeDef& node : optimized_func_0->node_def()) { + if (node.name() == "square" && count++) { + EXPECT_EQ(optimized_2, node.op()); + } else if (node.name() == "quadratic" && count++) { + EXPECT_EQ(optimized_3, node.op()); + } + } + EXPECT_EQ(2, count); + + const std::vector optimized_funcs = { + optimized_func_1, optimized_func_1, optimized_func_3}; + + // MyMul should be inlined into all optimized versions of MySquare. + for (const FunctionDef* optimized_func : optimized_funcs) { + count = 0; + for (const NodeDef& node : optimized_func->node_def()) { + if (node.name() == "my_mul/inlined_inputs" && count++) { + EXPECT_EQ("IdentityN", node.op()); + EXPECT_EQ(2, node.input_size()); + EXPECT_EQ("x:0", node.input(0)); + EXPECT_EQ("x:0", node.input(1)); + } else if (node.name() == "my_mul/x" && count++) { + EXPECT_EQ("Identity", node.op()); + EXPECT_EQ(1, node.input_size()); + EXPECT_EQ("my_mul/inlined_inputs:output:0", node.input(0)); + } else if (node.name() == "my_mul/y" && count++) { + EXPECT_EQ("Identity", node.op()); + EXPECT_EQ(1, node.input_size()); + EXPECT_EQ("my_mul/inlined_inputs:output:1", node.input(0)); + } else if (node.name() == "my_mul/mul" && count++) { + EXPECT_EQ("Mul", node.op()); + EXPECT_EQ(2, node.input_size()); + EXPECT_EQ("my_mul/x:output:0", node.input(0)); + EXPECT_EQ("my_mul/y:output:0", node.input(1)); + } else if (node.name() == "my_mul" && count++) { + EXPECT_EQ("IdentityN", node.op()); + EXPECT_EQ(1, node.input_size()); + EXPECT_EQ("my_mul/mul:z:0", node.input(0)); + } + EXPECT_TRUE(node.device().empty()); + } + EXPECT_EQ(5, count); + } + + item.fetch = {"out_s", "out_q"}; + item.feed.emplace_back("a", test::AsScalar(2.0f)); + item.feed.emplace_back("b", test::AsScalar(4)); + auto tensors_expected = EvaluateFetchNodes(item); + + GrapplerItem optimized(item, std::move(output)); + auto tensors = EvaluateFetchNodes(optimized); + + test::ExpectTensorEqual(tensors_expected[0], tensors[0]); + test::ExpectTensorEqual(tensors_expected[1], tensors[1]); +} + } // namespace } // namespace grappler } // namespace tensorflow diff --git a/tensorflow/core/grappler/utils/functions.cc b/tensorflow/core/grappler/utils/functions.cc index 638fe1999a..790809bc67 100644 --- a/tensorflow/core/grappler/utils/functions.cc +++ b/tensorflow/core/grappler/utils/functions.cc @@ -545,6 +545,12 @@ Status MakeGrapplerFunctionItem(const FunctionDef& func, return Status::OK(); } +Status MakeGrapplerFunctionItem(const FunctionDef& func, + const FunctionLibraryDefinition& flib, + GrapplerFunctionItem* item) { + return MakeGrapplerFunctionItem(func, AttrValueMap(), flib, item); +} + // Register GrapplerFunctionItem input arg expansion and function body outputs // in the GrapplerFunctionConnectivity. Status RegisterGrapplerFunctionConnectivity( @@ -560,9 +566,9 @@ Status RegisterGrapplerFunctionConnectivity( return Status::OK(); } -Status MakeSpecializedFunctionDef(const GrapplerFunctionItem& item, - const FunctionLibraryDefinition& flib, - FunctionDef* func) { +Status MakeFunctionDef(const GrapplerFunctionItem& item, + const FunctionLibraryDefinition& flib, + FunctionDef* func) { func->mutable_signature()->set_name(item.id); func->mutable_signature()->set_is_stateful(item.is_stateful()); diff --git a/tensorflow/core/grappler/utils/functions.h b/tensorflow/core/grappler/utils/functions.h index ab369bcad7..5e8b6c6960 100644 --- a/tensorflow/core/grappler/utils/functions.h +++ b/tensorflow/core/grappler/utils/functions.h @@ -38,7 +38,8 @@ using AttrValueMap = std::unordered_map; // function body in place of function inputs and a resolved input data type. struct InputArgExpansion { // TODO(ezhulenev): Add support for functions with tensor sequence inputs of - // different data types + // different data types. + // TODO(ezhulenev): Support type parametrized inputs? string input_name; // name of the function input argument DataType data_type; // input data type bool is_ref; // if true, inputs are required to be refs @@ -53,7 +54,8 @@ struct InputArgExpansion { // tensors of a function body nodes and a resolved output data type struct OutputArgExpansion { // TODO(ezhulenev): Add support for functions with tensor sequence outputs of - // different data types + // different data types. + // TODO(ezhulenev): Support type parametrized outputs? string output_name; // name of the function output argument DataType data_type; // output data type bool is_ref; // if true, outputs are refs @@ -186,13 +188,6 @@ bool HasParametrizedBody(const FunctionDef& func); // Check if function has parametrized type or body. bool IsParametrized(const FunctionDef& func); -// Make a GrapplerFunctionItem from the function definition and attributes. -// Return error if the given function def cannot be converted. -Status MakeGrapplerFunctionItem( - const FunctionDef& func, - const std::unordered_map& func_instantiation_attr, - const FunctionLibraryDefinition& flib, GrapplerFunctionItem* item); - // Register GrapplerFunctionItem input arg expansion and function body outputs // in the GrapplerFunctionConnectivity. Use function library definition to // lookup function body nodes output names and ranges. @@ -200,11 +195,28 @@ Status RegisterGrapplerFunctionConnectivity( const GrapplerFunctionItem& item, const FunctionLibraryDefinition& flib, GrapplerFunctionConnectivity* connectivity); -// Make a specialized FunctionDef from the GrapplerFunctionItem. Use function -// library definition to lookup function body nodes output names and ranges. -Status MakeSpecializedFunctionDef(const GrapplerFunctionItem& item, - const FunctionLibraryDefinition& flib, - FunctionDef* func); +// Make a GrapplerFunctionItem from the function definition and function +// instantiation attributes (caller node attributes). Returns error if the given +// function def cannot be converted (e.g. not all attributes are defined). +Status MakeGrapplerFunctionItem( + const FunctionDef& func, + const std::unordered_map& func_instantiation_attr, + const FunctionLibraryDefinition& flib, GrapplerFunctionItem* item); + +// Make a GrapplerFunction item from the function definition. Function must be +// fully defined (no type or body parametrization). +// TODO(ezhulenev): Support parametrized functions without fully defined +// instantiation attributes? Do we ever want to optimize parametrized function +// without specializing it to it's instantiation attributes (at least types)? +Status MakeGrapplerFunctionItem(const FunctionDef& func, + const FunctionLibraryDefinition& flib, + GrapplerFunctionItem* item); + +// Make a FunctionDef from the GrapplerFunctionItem. Use function library +// definition to lookup function body nodes output names and ranges. +Status MakeFunctionDef(const GrapplerFunctionItem& item, + const FunctionLibraryDefinition& flib, + FunctionDef* func); } // end namespace grappler } // end namespace tensorflow diff --git a/tensorflow/core/grappler/utils/functions_test.cc b/tensorflow/core/grappler/utils/functions_test.cc index 54d235a8a4..6dfd49b943 100644 --- a/tensorflow/core/grappler/utils/functions_test.cc +++ b/tensorflow/core/grappler/utils/functions_test.cc @@ -524,7 +524,7 @@ TEST_F(FunctionsTest, FromFunctionDefWithoutInput) { EXPECT_EQ("two", cast.input(0)); } -TEST_F(FunctionsTest, MakeSpecializedFunctionDef) { +TEST_F(FunctionsTest, MakeFunctionDef) { const Tensor kTwo = test::AsScalar(2); FunctionDef func = FunctionDefHelper::Define( // Name @@ -550,7 +550,7 @@ TEST_F(FunctionsTest, MakeSpecializedFunctionDef) { TF_EXPECT_OK(MakeGrapplerFunctionItem(func, func_attr, flib, &item)); FunctionDef specialized; - TF_EXPECT_OK(MakeSpecializedFunctionDef(item, flib, &specialized)); + TF_EXPECT_OK(MakeFunctionDef(item, flib, &specialized)); // Input and output types are resolved based on instantiation attributes. EXPECT_EQ("x", specialized.signature().input_arg(0).name()); @@ -573,7 +573,7 @@ TEST_F(FunctionsTest, MakeSpecializedFunctionDef) { EXPECT_EQ(2, count); } -TEST_F(FunctionsTest, SwapFunctionBodyAndMakeSpecializedFunctionDef) { +TEST_F(FunctionsTest, SwapFunctionBodyAndMakeFunctionDef) { using test::function::NDef; FunctionDef mul_func = FunctionDefHelper::Create( @@ -606,7 +606,7 @@ TEST_F(FunctionsTest, SwapFunctionBodyAndMakeSpecializedFunctionDef) { // Replace function body with identity function item.SwapFunctionBody(std::move(id_func_body)); FunctionDef specialized; - TF_EXPECT_OK(MakeSpecializedFunctionDef(item, flib, &specialized)); + TF_EXPECT_OK(MakeFunctionDef(item, flib, &specialized)); // Check that graph body was updated. int count = 0; -- GitLab From 39a2787272f948a043a1ca103159307cfb0f7248 Mon Sep 17 00:00:00 2001 From: ImSheridan Date: Fri, 20 Apr 2018 09:20:38 +0800 Subject: [PATCH 710/791] Fix incorrect math equation renderings broken by backtick (#18386) * Fix incorrect `` typo format * Remove breaking ``` for math equations * fix one more typo * fix more math equation broken ` typos in py --- .../bayesflow/python/ops/monte_carlo_impl.py | 22 ++++++--------- .../factorization/python/ops/kmeans.py | 4 +-- .../python/contrib.bayesflow.monte_carlo.md | 28 ++++++++----------- tensorflow/python/ops/nn_ops.py | 2 +- 4 files changed, 23 insertions(+), 33 deletions(-) diff --git a/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py b/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py index 48ff083532..032b859d46 100644 --- a/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py +++ b/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py @@ -44,15 +44,13 @@ def expectation_importance_sampler(f, n=None, seed=None, name='expectation_importance_sampler'): - r"""Monte Carlo estimate of `\\(E_p[f(Z)] = E_q[f(Z) p(Z) / q(Z)]\\)`. + r"""Monte Carlo estimate of \\(E_p[f(Z)] = E_q[f(Z) p(Z) / q(Z)]\\). - With `\\(p(z) := exp^{log_p(z)}\\)`, this `Op` returns + With \\(p(z) := exp^{log_p(z)}\\), this `Op` returns - ``` \\(n^{-1} sum_{i=1}^n [ f(z_i) p(z_i) / q(z_i) ], z_i ~ q,\\) \\(\approx E_q[ f(Z) p(Z) / q(Z) ]\\) \\(= E_p[f(Z)]\\) - ``` This integral is done in log-space with max-subtraction to better handle the often extreme values that `f(z) p(z) / q(z)` can take on. @@ -121,14 +119,12 @@ def expectation_importance_sampler_logspace( name='expectation_importance_sampler_logspace'): r"""Importance sampling with a positive function, in log-space. - With `\\(p(z) := exp^{log_p(z)}\\)`, and `\\(f(z) = exp{log_f(z)}\\)`, + With \\(p(z) := exp^{log_p(z)}\\), and \\(f(z) = exp{log_f(z)}\\), this `Op` returns - ``` \\(Log[ n^{-1} sum_{i=1}^n [ f(z_i) p(z_i) / q(z_i) ] ], z_i ~ q,\\) \\(\approx Log[ E_q[ f(Z) p(Z) / q(Z) ] ]\\) \\(= Log[E_p[f(Z)]]\\) - ``` This integral is done in log-space with max-subtraction to better handle the often extreme values that `f(z) p(z) / q(z)` can take on. @@ -196,13 +192,11 @@ def _logspace_mean(log_values): def expectation(f, samples, log_prob=None, use_reparametrization=True, axis=0, keep_dims=False, name=None): - """Computes the Monte-Carlo approximation of `\\(E_p[f(X)]\\)`. + """Computes the Monte-Carlo approximation of \\(E_p[f(X)]\\). This function computes the Monte-Carlo approximation of an expectation, i.e., - ```none \\(E_p[f(X)] \approx= m^{-1} sum_i^m f(x_j), x_j\ ~iid\ p(X)\\) - ``` where: @@ -216,8 +210,8 @@ def expectation(f, samples, log_prob=None, use_reparametrization=True, parameterless distribution (e.g., `Normal(Y; m, s) <=> Y = sX + m, X ~ Normal(0,1)`), we can swap gradient and expectation, i.e., - `grad[ Avg{ \\(s_i : i=1...n\\) } ] = Avg{ grad[\\(s_i\\)] : i=1...n }` where - `S_n = Avg{\\(s_i\\)}` and `\\(s_i = f(x_i), x_i ~ p\\)`. + grad[ Avg{ \\(s_i : i=1...n\\) } ] = Avg{ grad[\\(s_i\\)] : i=1...n } where + S_n = Avg{\\(s_i\\)}` and `\\(s_i = f(x_i), x_i ~ p\\). However, if p is not reparameterized, TensorFlow's gradient will be incorrect since the chain-rule stops at samples of non-reparameterized distributions. @@ -296,7 +290,7 @@ def expectation(f, samples, log_prob=None, use_reparametrization=True, Args: f: Python callable which can return `f(samples)`. samples: `Tensor` of samples used to form the Monte-Carlo approximation of - `\\(E_p[f(X)]\\)`. A batch of samples should be indexed by `axis` + \\(E_p[f(X)]\\). A batch of samples should be indexed by `axis` dimensions. log_prob: Python callable which can return `log_prob(samples)`. Must correspond to the natural-logarithm of the pdf/pmf of each sample. Only @@ -317,7 +311,7 @@ def expectation(f, samples, log_prob=None, use_reparametrization=True, Returns: approx_expectation: `Tensor` corresponding to the Monte-Carlo approximation - of `\\(E_p[f(X)]\\)`. + of \\(E_p[f(X)]\\). Raises: ValueError: if `f` is not a Python `callable`. diff --git a/tensorflow/contrib/factorization/python/ops/kmeans.py b/tensorflow/contrib/factorization/python/ops/kmeans.py index bfe338c9f9..9ffdd3ba5e 100644 --- a/tensorflow/contrib/factorization/python/ops/kmeans.py +++ b/tensorflow/contrib/factorization/python/ops/kmeans.py @@ -374,11 +374,11 @@ class KMeansClustering(estimator.Estimator): than `num_clusters`, a TensorFlow runtime error occurs. distance_metric: The distance metric used for clustering. One of: * `KMeansClustering.SQUARED_EUCLIDEAN_DISTANCE`: Euclidean distance - between vectors `u` and `v` is defined as `\\(||u - v||_2\\)` + between vectors `u` and `v` is defined as \\(||u - v||_2\\) which is the square root of the sum of the absolute squares of the elements' difference. * `KMeansClustering.COSINE_DISTANCE`: Cosine distance between vectors - `u` and `v` is defined as `\\(1 - (u . v) / (||u||_2 ||v||_2)\\)`. + `u` and `v` is defined as \\(1 - (u . v) / (||u||_2 ||v||_2)\\). random_seed: Python integer. Seed for PRNG used to initialize centers. use_mini_batch: A boolean specifying whether to use the mini-batch k-means algorithm. See explanation above. diff --git a/tensorflow/docs_src/api_guides/python/contrib.bayesflow.monte_carlo.md b/tensorflow/docs_src/api_guides/python/contrib.bayesflow.monte_carlo.md index f3db5857ae..74fe4a323a 100644 --- a/tensorflow/docs_src/api_guides/python/contrib.bayesflow.monte_carlo.md +++ b/tensorflow/docs_src/api_guides/python/contrib.bayesflow.monte_carlo.md @@ -6,43 +6,39 @@ Monte Carlo integration and helpers. ## Background Monte Carlo integration refers to the practice of estimating an expectation with -a sample mean. For example, given random variable `Z in \\(R^k\\)` with density `p`, +a sample mean. For example, given random variable Z in \\(R^k\\) with density `p`, the expectation of function `f` can be approximated like: -``` $$E_p[f(Z)] = \int f(z) p(z) dz$$ $$ ~ S_n := n^{-1} \sum_{i=1}^n f(z_i), z_i\ iid\ samples\ from\ p.$$ -``` -If `\\(E_p[|f(Z)|] < infinity\\)`, then `\\(S_n\\) --> \\(E_p[f(Z)]\\)` by the strong law of large -numbers. If `\\(E_p[f(Z)^2] < infinity\\)`, then `\\(S_n\\)` is asymptotically normal with -variance `\\(Var[f(Z)] / n\\)`. +If \\(E_p[|f(Z)|] < infinity\\), then \\(S_n\\) --> \\(E_p[f(Z)]\\) by the strong law of large +numbers. If \\(E_p[f(Z)^2] < infinity\\), then \\(S_n\\) is asymptotically normal with +variance \\(Var[f(Z)] / n\\). Practitioners of Bayesian statistics often find themselves wanting to estimate -`\\(E_p[f(Z)]\\)` when the distribution `p` is known only up to a constant. For +\\(E_p[f(Z)]\\) when the distribution `p` is known only up to a constant. For example, the joint distribution `p(z, x)` may be known, but the evidence -`\\(p(x) = \int p(z, x) dz\\)` may be intractable. In that case, a parameterized -distribution family `\\(q_\lambda(z)\\)` may be chosen, and the optimal `\\(\lambda\\)` is the -one minimizing the KL divergence between `\\(q_\lambda(z)\\)` and -`\\(p(z | x)\\)`. We only know `p(z, x)`, but that is sufficient to find `\\(\lambda\\)`. +\\(p(x) = \int p(z, x) dz\\) may be intractable. In that case, a parameterized +distribution family \\(q_\lambda(z)\\) may be chosen, and the optimal \\(\lambda\\) is the +one minimizing the KL divergence between \\(q_\lambda(z)\\) and +\\(p(z | x)\\). We only know `p(z, x)`, but that is sufficient to find \\(\lambda\\). ## Log-space evaluation and subtracting the maximum Care must be taken when the random variable lives in a high dimensional space. -For example, the naive importance sample estimate `\\(E_q[f(Z) p(Z) / q(Z)]\\)` -involves the ratio of two terms `\\(p(Z) / q(Z)\\)`, each of which must have tails -dropping off faster than `\\(O(|z|^{-(k + 1)})\\)` in order to have finite integral. +For example, the naive importance sample estimate \\(E_q[f(Z) p(Z) / q(Z)]\\) +involves the ratio of two terms \\(p(Z) / q(Z)\\), each of which must have tails +dropping off faster than \\(O(|z|^{-(k + 1)})\\) in order to have finite integral. This ratio would often be zero or infinity up to numerical precision. For that reason, we write -``` $$Log E_q[ f(Z) p(Z) / q(Z) ]$$ $$ = Log E_q[ \exp\{Log[f(Z)] + Log[p(Z)] - Log[q(Z)] - C\} ] + C,$$ where $$C := Max[ Log[f(Z)] + Log[p(Z)] - Log[q(Z)] ].$$ -``` The maximum value of the exponentiated term will be 0.0, and the expectation can be evaluated in a stable manner. diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py index a8d0293d13..cd07550d2e 100644 --- a/tensorflow/python/ops/nn_ops.py +++ b/tensorflow/python/ops/nn_ops.py @@ -1155,7 +1155,7 @@ def atrous_conv2d(value, filters, rate, padding, name=None): Returns: A `Tensor` with the same type as `value`. - Output shape with `'VALID`` padding is: + Output shape with `'VALID'` padding is: [batch, height - 2 * (filter_width - 1), width - 2 * (filter_height - 1), out_channels]. -- GitLab From a734919fd8fd6d74edf1e7c3abec3ee11fec83fd Mon Sep 17 00:00:00 2001 From: Jiajia Li Date: Fri, 20 Apr 2018 09:22:26 +0800 Subject: [PATCH 711/791] Fix the error looking for libhdfs.so, Mac OS using libhdfs.dylib (#18486) --- tensorflow/core/platform/hadoop/hadoop_file_system.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/core/platform/hadoop/hadoop_file_system.cc b/tensorflow/core/platform/hadoop/hadoop_file_system.cc index 9a71fbe2b7..a8cb40502c 100644 --- a/tensorflow/core/platform/hadoop/hadoop_file_system.cc +++ b/tensorflow/core/platform/hadoop/hadoop_file_system.cc @@ -109,6 +109,8 @@ class LibHDFS { // in the libhdfs documentation. #if defined(PLATFORM_WINDOWS) const char* kLibHdfsDso = "hdfs.dll"; +#elif defined(MACOS) || defined(TARGET_OS_MAC) + const char* kLibHdfsDso = "libhdfs.dylib"; #else const char* kLibHdfsDso = "libhdfs.so"; #endif -- GitLab From 256aad5324d163c028da0dc0318c3e00cf2fc3ab Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Apr 2018 18:29:00 -0700 Subject: [PATCH 712/791] [XLA] Fix a bug in the name_uniquer. The problem happens because the name_uniquer stripped away the numeric suffix if it <=0. The solution is, if there was a numeric suffix, the result should also have a numeric suffix. PiperOrigin-RevId: 193606838 --- tensorflow/compiler/xla/service/name_uniquer.cc | 11 ++++++----- tensorflow/compiler/xla/service/name_uniquer_test.cc | 11 +++++++++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/tensorflow/compiler/xla/service/name_uniquer.cc b/tensorflow/compiler/xla/service/name_uniquer.cc index 7d8c05fffa..f74bcb0b79 100644 --- a/tensorflow/compiler/xla/service/name_uniquer.cc +++ b/tensorflow/compiler/xla/service/name_uniquer.cc @@ -53,17 +53,18 @@ NameUniquer::NameUniquer(const string& separator) { } string NameUniquer::GetUniqueName(tensorflow::StringPiece prefix) { - string root = prefix.empty() ? "name" : prefix.ToString(); - root = GetSanitizedName(root); + string root = GetSanitizedName(prefix.empty() ? "name" : prefix.ToString()); // Strip away numeric suffix (if any). Only recognize separator if it is in // the middle of the name. + bool has_numeric_suffix = false; + int64 numeric_suffix = 0; size_t separator_index = root.rfind(separator_); if (separator_index != string::npos && (separator_index > 0) && (separator_index < root.size() - 1)) { string after_suffix = root.substr(separator_index + 1); - int64 numeric_suffix; if (tensorflow::strings::safe_strto64(after_suffix, &numeric_suffix)) { + has_numeric_suffix = true; // Remove numeric suffix from root. root = root.substr(0, separator_index); // Update count to at least the numeric suffix value to avoid future @@ -71,11 +72,11 @@ string NameUniquer::GetUniqueName(tensorflow::StringPiece prefix) { generated_names_[root] = std::max(generated_names_[root], numeric_suffix); } } - int64* count = &(generated_names_[root]); if (*count == 0) { *count = 1; - return root; + return has_numeric_suffix ? tensorflow::strings::StrCat(root, separator_, 0) + : root; } else { tensorflow::strings::StrAppend(&root, separator_, *count); // Increment lookup under old 'root' name. diff --git a/tensorflow/compiler/xla/service/name_uniquer_test.cc b/tensorflow/compiler/xla/service/name_uniquer_test.cc index 4258cf1687..2ec255558c 100644 --- a/tensorflow/compiler/xla/service/name_uniquer_test.cc +++ b/tensorflow/compiler/xla/service/name_uniquer_test.cc @@ -57,11 +57,18 @@ TEST_F(NameUniquerTest, NumericSuffixes) { EXPECT_EQ("foo.55", uniquer.GetUniqueName("foo")); EXPECT_EQ("foo.55.1", uniquer.GetUniqueName("foo.55.1")); EXPECT_EQ("foo.55.2", uniquer.GetUniqueName("foo.55.1")); - EXPECT_EQ("bar", uniquer.GetUniqueName("bar.-1000")); + EXPECT_EQ("bar.0", uniquer.GetUniqueName("bar.-1000")); EXPECT_EQ("bar.1", uniquer.GetUniqueName("bar.-2000")); EXPECT_EQ("bar.2", uniquer.GetUniqueName("bar.1")); } +TEST_F(NameUniquerTest, PrefixHasSuffix) { + NameUniquer uniquer("."); + + EXPECT_EQ("foo.11.0", uniquer.GetUniqueName("foo.11.0")); + EXPECT_EQ("foo.11", uniquer.GetUniqueName("foo.11")); +} + TEST_F(NameUniquerTest, Sanitize) { NameUniquer uniquer("_"); @@ -73,7 +80,7 @@ TEST_F(NameUniquerTest, Sanitize) { EXPECT_EQ("foo_55", uniquer.GetUniqueName("foo")); // Invalid characters will be replaced with '_'. - EXPECT_EQ("bar", uniquer.GetUniqueName("bar<-1000")); + EXPECT_EQ("bar_0", uniquer.GetUniqueName("bar<-1000")); EXPECT_EQ("bar_1", uniquer.GetUniqueName("bar<-2000")); EXPECT_EQ("bar_2", uniquer.GetUniqueName("bar_1")); -- GitLab From 052c3863cf8b901303a1a32e82b6525dc6ea6dbd Mon Sep 17 00:00:00 2001 From: Anna R Date: Thu, 19 Apr 2018 18:45:47 -0700 Subject: [PATCH 713/791] Internal change. PiperOrigin-RevId: 193608140 --- tensorflow/compiler/xla/python/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/compiler/xla/python/BUILD b/tensorflow/compiler/xla/python/BUILD index 0b9333b406..ecb87bd889 100644 --- a/tensorflow/compiler/xla/python/BUILD +++ b/tensorflow/compiler/xla/python/BUILD @@ -8,7 +8,6 @@ py_library( name = "xla_client", srcs = ["xla_client.py"], srcs_version = "PY2AND3", - tags = ["no_oss"], visibility = ["//visibility:public"], deps = [ ":pywrap_xla", @@ -21,6 +20,7 @@ py_test( srcs = ["xla_client_test.py"], main = "xla_client_test.py", srcs_version = "PY2AND3", + tags = ["no_oss"], deps = [ ":xla_client", "//tensorflow/python:platform_test", -- GitLab From 6e2df5e471295cd32f9887d76e6ddbf1b4e2a11a Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Thu, 19 Apr 2018 19:03:03 -0700 Subject: [PATCH 714/791] Automated g4 rollback of changelist 193593761 PiperOrigin-RevId: 193609407 --- tensorflow/compiler/xla/service/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index d5d09bd8a3..9009cbf845 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -699,7 +699,6 @@ cc_library( "//tensorflow/compiler/xla/service/cpu:cpu_compiler", "//tensorflow/compiler/xla/service/cpu:cpu_transfer_manager", "//tensorflow/core:stream_executor_no_cuda", - "//tensorflow/stream_executor:stream_executor_impl", ], ) -- GitLab From b001827146ff95c9e0ce5668c85d8cc2daf6b78d Mon Sep 17 00:00:00 2001 From: Igor Saprykin Date: Thu, 19 Apr 2018 19:11:37 -0700 Subject: [PATCH 715/791] Support variable parameter structure in TPU distribution strategy. TPUStrategy is added to a few more tests. There appears to be an issue with the batch norm test in minimize_loss_test where the moving averages stay at 0. I'm trying to resolve that separately as the next CL. PiperOrigin-RevId: 193610264 --- tensorflow/contrib/distribute/python/BUILD | 18 +++-- .../distribute/python/minimize_loss_test.py | 19 ++++- .../distribute/python/single_loss_example.py | 7 +- .../contrib/distribute/python/tpu_strategy.py | 70 +++++++++++-------- .../contrib/distribute/python/values.py | 34 +++++++-- 5 files changed, 104 insertions(+), 44 deletions(-) diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD index 837a1f1348..c2834d8226 100644 --- a/tensorflow/contrib/distribute/python/BUILD +++ b/tensorflow/contrib/distribute/python/BUILD @@ -231,15 +231,14 @@ py_library( srcs = ["tpu_strategy.py"], visibility = ["//tensorflow:internal"], deps = [ - "//tensorflow/contrib/distribute/python:one_device_strategy", - "//tensorflow/contrib/eager/python:datasets", - "//tensorflow/contrib/optimizer_v2:training", + ":one_device_strategy", + ":values", "//tensorflow/contrib/tpu", - "//tensorflow/python:array_ops", + "//tensorflow/contrib/tpu:tpu_py", + "//tensorflow/python:constant_op", + "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python/eager:context", - "@six_archive//:six", + "//tensorflow/python:util", ], ) @@ -249,9 +248,13 @@ py_library( srcs = ["minimize_loss_test.py"], deps = [ ":combinations", + ":mirrored_strategy", ":single_loss_example", + "//tensorflow/contrib/tpu:tpu_lib", "//tensorflow/python:control_flow_ops", + "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", + "//tensorflow/python:variable_scope", "//tensorflow/python:variables", "//tensorflow/python/data/ops:dataset_ops", "//tensorflow/python/eager:context", @@ -324,6 +327,7 @@ py_library( srcs = ["single_loss_example.py"], deps = [ ":step_fn", + "//tensorflow/contrib/data/python/ops:batching", "//tensorflow/python:array_ops", "//tensorflow/python:constant_op", "//tensorflow/python:layers", diff --git a/tensorflow/contrib/distribute/python/minimize_loss_test.py b/tensorflow/contrib/distribute/python/minimize_loss_test.py index 43b2e91cbf..e134fe34e1 100644 --- a/tensorflow/contrib/distribute/python/minimize_loss_test.py +++ b/tensorflow/contrib/distribute/python/minimize_loss_test.py @@ -96,8 +96,17 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase): combinations.times( combinations.distributions_and_v1_optimizers() + combinations.distributions_and_v2_optimizers(), - combinations.combine(mode=["graph", "eager"]))) - def testOptimizerInsideModelFn(self, distribution, optimizer_fn): + combinations.combine(mode=["graph", "eager"], is_tpu=[False])) + + combinations.combine( + distribution=[combinations.tpu_strategy], + optimizer_fn=[ + combinations.adam_optimizer_v1_fn, + combinations.gradient_descent_optimizer_v1_fn + ], + mode=["graph"], + is_tpu=[True])) + + def testOptimizerInsideModelFn(self, distribution, optimizer_fn, is_tpu): created_variables = [] trainable_variables = [] @@ -128,11 +137,17 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase): if not context.executing_eagerly(): with self.test_session() as sess: + if is_tpu: + sess.run(tpu.initialize_system()) run_step = sess.make_callable(run_step()) self.evaluate(variables_lib.global_variables_initializer()) run_step() + if is_tpu: + with self.test_session() as sess: + sess.run(tpu.shutdown_system()) + def get_expected_variables(optimizer_fn, num_parameter_devices): variables_map = { "GradientDescent": ["dense/kernel", "dense/bias"], diff --git a/tensorflow/contrib/distribute/python/single_loss_example.py b/tensorflow/contrib/distribute/python/single_loss_example.py index abd13c6cc6..0db0b59fca 100644 --- a/tensorflow/contrib/distribute/python/single_loss_example.py +++ b/tensorflow/contrib/distribute/python/single_loss_example.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.contrib.data.python.ops import batching from tensorflow.contrib.distribute.python import step_fn from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import constant_op @@ -54,7 +55,11 @@ def minimize_loss_example(optimizer_fn, """Example of non-distribution-aware legacy code.""" def dataset_fn(): - return dataset_ops.Dataset.from_tensors([[1.]]).repeat().batch(2) + dataset = dataset_ops.Dataset.from_tensors([[1.]]).repeat() + # TODO(isaprykin): map_and_batch with drop_remainder causes shapes to be + # fully defined for TPU. Remove this when XLA supports dynamic shapes. + return dataset.apply( + batching.map_and_batch(lambda x: x, batch_size=2, drop_remainder=True)) # An Optimizer instance is created either outside or inside model_fn. outer_optimizer = None diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py index ceb52ceca7..a7e4fe80f3 100644 --- a/tensorflow/contrib/distribute/python/tpu_strategy.py +++ b/tensorflow/contrib/distribute/python/tpu_strategy.py @@ -21,15 +21,16 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import itertools + from tensorflow.contrib import tpu from tensorflow.contrib.distribute.python import one_device_strategy from tensorflow.contrib.distribute.python import values from tensorflow.contrib.tpu.python.ops import tpu_ops from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops +from tensorflow.python.util import nest # TODO(isaprykin): Consider whether inheriting is really appropriate. @@ -37,48 +38,53 @@ class TPUStrategy(one_device_strategy.OneDeviceStrategy): """Experimental TPU distribution strategy implementation.""" def __init__(self, - global_batch_size=2, num_cores_per_host=2, iterations_per_step=2): - # TODO(isaprykin): Generalize the defaults. + # TODO(isaprykin): Generalize the defaults. They are currently tailored for + # the unit test. super(TPUStrategy, self).__init__('/cpu:0') # TODO(isaprykin): Auto-detect number of cores and hosts. self._num_cores_per_host = num_cores_per_host - self._global_batch_size = global_batch_size # TODO(isaprykin): This might have to be per-call. self._iterations_per_step = iterations_per_step def distribute_dataset(self, dataset_fn): return values.PerIterationDataset( - self._call_dataset_fn(dataset_fn), self._iterations_per_step) + self._call_dataset_fn(dataset_fn), self._iterations_per_step, + self._num_cores_per_host) def _call_for_each_tower(self, fn, *args, **kwargs): kwargs.pop('run_concurrently', None) - # TODO(isaprykin): Support variable arguments similar to PerDevice+regroup. - inputs = args[0] + inputs = {'args': args, 'kwargs': kwargs} + flat_inputs = nest.flatten(inputs) + + feed_mask = [isinstance(f, values.PerIteration) for f in flat_inputs] - sharded_shape = [None] # Python 2 nonlocal. + feeds = lambda: itertools.compress(flat_inputs, feed_mask) + shapes = [f.get_shape() for f in feeds()] + if any([not s.is_fully_defined() for s in shapes]): + raise ValueError( + 'TPU currently requires fully defined shapes. Either use ' + 'set_shape() on the input tensors or use ' + 'dataset.apply(map_and_batch(..., drop_remainder=True)).') + types = [f.get_dtype() for f in feeds()] def infeed_input(i): """Get input, split it and then enqueue.""" - batches = array_ops.gather(inputs, i) + iteration_inputs = [f.get(i) for f in feeds()] - # TODO(isaprykin): Handle partial batch. - global_shape = [self._global_batch_size] + list(batches.get_shape())[1:] - sharded_shape[0] = ([self._global_batch_size / self._num_cores_per_host] + - list(global_shape)[1:]) + infeed_inputs = [[inputs_per_core[core_id] + for inputs_per_core in iteration_inputs] + for core_id in range(self._num_cores_per_host)] - batches.set_shape(global_shape) - batches = array_ops.split(batches, self._num_cores_per_host) + infeed_ops = [] + for core_id, infeed_input in enumerate(infeed_inputs): + infeed_ops.append( + tpu_ops.infeed_enqueue_tuple( + inputs=infeed_input, shapes=shapes, device_ordinal=core_id)) - infeeds = [ - tpu_ops.infeed_enqueue_tuple( - inputs=[batches[j]], shapes=[sharded_shape[0]], device_ordinal=j) - for j in range(self._num_cores_per_host) - ] - - with ops.control_dependencies(infeeds): + with ops.control_dependencies(infeed_ops): return i + 1 with ops.device('/task:0/device:CPU:0'): @@ -87,13 +93,21 @@ class TPUStrategy(one_device_strategy.OneDeviceStrategy): infeed_input, [constant_op.constant(0)], parallel_iterations=1) - assert sharded_shape[0] - def dequeueing_fn(*args, **kwargs): + """Dequeue input arguments and supply them to `fn`.""" del args, kwargs - x, = tpu.infeed_dequeue_tuple( - dtypes=[dtypes.float32], shapes=[sharded_shape[0]]) - return fn(x) + dequeued = tpu.infeed_dequeue_tuple(dtypes=types, shapes=shapes) + dequeued = iter(dequeued) + + fn_inputs = [] + for inp, is_feed in zip(flat_inputs, feed_mask): + if is_feed: + fn_inputs.append(next(dequeued)) + else: + fn_inputs.append(inp) + + fn_inputs = nest.pack_sequence_as(inputs, fn_inputs) + return fn(*fn_inputs['args'], **fn_inputs['kwargs']) def iterate_on_tpu(): return tpu.repeat(self._iterations_per_step, dequeueing_fn, []) diff --git a/tensorflow/contrib/distribute/python/values.py b/tensorflow/contrib/distribute/python/values.py index 62016c3a78..8cb5276579 100644 --- a/tensorflow/contrib/distribute/python/values.py +++ b/tensorflow/contrib/distribute/python/values.py @@ -570,18 +570,36 @@ class PerDeviceDataset(object): dataset_iterator, self._devices, self._prefetch_on_device) +class PerIteration(object): + """Holds input for multiple iterations at once.""" + + def __init__(self, index): + self._index = index + + def get(self, iteration): + return array_ops.gather(self._index, iteration) + + def get_shape(self): + return self._index[-1][-1].get_shape() + + def get_dtype(self): + return self._index[-1][-1].dtype + + class MultiIterator(object): """Iterator that returns results of multiple get_next()s.""" - def __init__(self, dataset_iterator, iterations): + def __init__(self, dataset_iterator, iterations, batches_per_iteration): self._dataset_iterator = dataset_iterator self._iterations = iterations + self._batches_per_iteration = batches_per_iteration def get_next(self, name=None): - return [ + return PerIteration([[ self._dataset_iterator.get_next(name=name) - for _ in range(self._iterations) + for _ in range(self._batches_per_iteration) ] + for _ in range(self._iterations)]) @property def initializer(self): @@ -589,18 +607,22 @@ class MultiIterator(object): class PerIterationDataset(object): + """A dataset that returns MultiIterators.""" - def __init__(self, dataset, iterations): + def __init__(self, dataset, iterations, batches_per_iteration): self._dataset = dataset self._iterations = iterations + self._batches_per_iteration = batches_per_iteration def make_one_shot_iterator(self): iterator = self._dataset.make_one_shot_iterator() - return MultiIterator(iterator, self._iterations) + return MultiIterator(iterator, self._iterations, + self._batches_per_iteration) def make_initializable_iterator(self): iterator = self._dataset.make_initializable_iterator() - return MultiIterator(iterator, self._iterations) + return MultiIterator(iterator, self._iterations, + self._batches_per_iteration) class MapOutput(object): -- GitLab From 8723770b4cbcac0a528354d8508a5ef83716d1fa Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Thu, 19 Apr 2018 19:27:34 -0700 Subject: [PATCH 716/791] [XLA] Remove default argument on virtual function DeviceMemoryAllocator::Allocate(). Default args on virtual functions are disallowed by the Google style guide, for good reason. They have the extremely surprising behavior that the defaults you get when calling a function on a pointer depend not on the underlying type of the object, but on whatever is the semantic type of the pointer! PiperOrigin-RevId: 193611213 --- .../xla/service/device_memory_allocator.h | 30 ++++++++++++++----- .../xla/tests/local_client_test_base.cc | 3 +- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/tensorflow/compiler/xla/service/device_memory_allocator.h b/tensorflow/compiler/xla/service/device_memory_allocator.h index 240acf8973..da45c4d45a 100644 --- a/tensorflow/compiler/xla/service/device_memory_allocator.h +++ b/tensorflow/compiler/xla/service/device_memory_allocator.h @@ -38,13 +38,25 @@ class DeviceMemoryAllocator { virtual ~DeviceMemoryAllocator() {} // 'retry_on_failure': If false, and the first attempt to allocate the memory - // fails, the allocation should return immediately without retrying. - // An example use case is optional scratch spaces where a failure - // has only performance impact. + // fails, the allocation should return immediately without retrying. An + // example use case is optional scratch spaces where a failure has only + // performance impact. + // // Allocate() should return a null pointer for a size-0 allocation. // Deallocate() must be a no-op for null pointers. - virtual StatusOr Allocate( - int device_ordinal, uint64 size, bool retry_on_failure = true) = 0; + virtual StatusOr Allocate(int device_ordinal, + uint64 size, + bool retry_on_failure) = 0; + + // Two-arg version of Allocate(), which sets retry-on-failure to true. + // + // (We don't simply use a default argument on the virtual Allocate function + // because default args on virtual functions are disallowed by the Google + // style guide.) + StatusOr Allocate(int device_ordinal, uint64 size) { + return Allocate(device_ordinal, size, /*retry_on_failure=*/true); + } + virtual tensorflow::Status Deallocate(int device_ordinal, se::DeviceMemoryBase* mem) = 0; @@ -67,8 +79,12 @@ class StreamExecutorMemoryAllocator : public DeviceMemoryAllocator { const se::Platform* platform, tensorflow::gtl::ArraySlice stream_executors); - StatusOr Allocate( - int device_ordinal, uint64 size, bool retry_on_failure = true) override; + StatusOr Allocate(int device_ordinal, uint64 size, + bool retry_on_failure) override; + + // Pull in two-arg overload that sets retry_on_failure to true. + using DeviceMemoryAllocator::Allocate; + tensorflow::Status Deallocate(int device_ordinal, se::DeviceMemoryBase* mem) override; diff --git a/tensorflow/compiler/xla/tests/local_client_test_base.cc b/tensorflow/compiler/xla/tests/local_client_test_base.cc index c60ba2422f..bb5aabb214 100644 --- a/tensorflow/compiler/xla/tests/local_client_test_base.cc +++ b/tensorflow/compiler/xla/tests/local_client_test_base.cc @@ -44,7 +44,8 @@ StatusOr TestAllocator::Allocate(int device_ordinal, allocation_count_++; device_allocation_count_[device_ordinal]++; } - return StreamExecutorMemoryAllocator::Allocate(device_ordinal, size); + return StreamExecutorMemoryAllocator::Allocate(device_ordinal, size, + retry_on_failure); } tensorflow::Status TestAllocator::Deallocate(int device_ordinal, -- GitLab From 2a956c9b8f9950405b481ccc0e05636873ecc9ae Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 20 Apr 2018 02:40:37 +0000 Subject: [PATCH 717/791] Support string tensors for tf.count_nonzero This fix tries to address the issue raised in 18712 where `tf.count_nonzero` does not support string tensors. The implementation of `tf.count_nonzero` relies on `tf.not_equal` which actually support string tensors. The reason the string tensor does not work is because `tf.count_nonzero` created a numpy type `zero` which uses `input_tensor.dtype.as_numpy_dtype()`. The numpy type `zero` is then passed to `tf.not_equal (which converts numpy `zero` into a tensor zero). However, `input_tensor.dtype.as_numpy_dtype()` will converts tf.string to numpy.object thus the exception. But that is not necessary as `zero` could be created with `tf.zeros` directly without back and forth conversion to numpy. This fix fixes the issue. This fix fixes 18712. Signed-off-by: Yong Tang --- tensorflow/python/ops/math_ops.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py index 781b1c557f..8c9ad66b0e 100644 --- a/tensorflow/python/ops/math_ops.py +++ b/tensorflow/python/ops/math_ops.py @@ -1487,7 +1487,8 @@ def count_nonzero(input_tensor, with ops.name_scope(name, "count_nonzero", [input_tensor]): input_tensor = ops.convert_to_tensor(input_tensor, name="input_tensor") - zero = input_tensor.dtype.as_numpy_dtype() + # A scalar of 'zero' is enough as `not_equal` will broadcast. + zero = array_ops.zeros([], dtype=input_tensor.dtype) return cast( reduce_sum( # int64 reduction happens on GPU -- GitLab From 37999ce500f27d587100f0bf45e87957936f5ada Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 20 Apr 2018 02:48:15 +0000 Subject: [PATCH 718/791] Add test case for tf.string support with tf.count_nonzero Signed-off-by: Yong Tang --- tensorflow/python/kernel_tests/reduction_ops_test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tensorflow/python/kernel_tests/reduction_ops_test.py b/tensorflow/python/kernel_tests/reduction_ops_test.py index 589ea54973..0be89e1ff4 100644 --- a/tensorflow/python/kernel_tests/reduction_ops_test.py +++ b/tensorflow/python/kernel_tests/reduction_ops_test.py @@ -958,6 +958,12 @@ class CountNonzeroReductionTest(test.TestCase): y = math_ops.count_nonzero(x, [0]) self.assertAllEqual(y.eval(), np.zeros(9938)) + def testStringReduce(self): + # Test case for GitHub issue 18712 + with self.test_session() as sess: + v = math_ops.count_nonzero(constant_op.constant(["test"])) + self.assertAllClose(sess.run(v), 1) + if __name__ == "__main__": test.main() -- GitLab From 7358025743951b42fe0f99fb85b4418769de5357 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 20 Apr 2018 02:51:54 +0000 Subject: [PATCH 719/791] Add test cases with axis and keepdims for tf.count_nonzero and string Signed-off-by: Yong Tang --- .../python/kernel_tests/reduction_ops_test.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/kernel_tests/reduction_ops_test.py b/tensorflow/python/kernel_tests/reduction_ops_test.py index 0be89e1ff4..943b80b787 100644 --- a/tensorflow/python/kernel_tests/reduction_ops_test.py +++ b/tensorflow/python/kernel_tests/reduction_ops_test.py @@ -889,9 +889,9 @@ class AnyReductionTest(test.TestCase): class CountNonzeroReductionTest(test.TestCase): - def _compare(self, x, reduction_axes, keepdims, use_gpu=False, + def _compare(self, x, reduction_axes, keepdims, use_gpu=False, zero=0, feed_dict=None): - np_ans = (x != 0).astype(np.int32) + np_ans = (x != zero).astype(np.int32) if reduction_axes is None: np_ans = np.sum(np_ans, keepdims=keepdims) else: @@ -964,6 +964,15 @@ class CountNonzeroReductionTest(test.TestCase): v = math_ops.count_nonzero(constant_op.constant(["test"])) self.assertAllClose(sess.run(v), 1) + def testStringReduce1D(self): + # Create a 1D array of strings + x = np.asarray(["", "", "a", "", "", "b"]) + self._compare(x, None, keepdims=False, zero=np.str("")) + self._compare(x, [], keepdims=False, zero=np.str("")) + self._compare(x, [0], keepdims=False, zero=np.str("")) + self._compare(x, None, keepdims=True, zero=np.str("")) + self._compare(x, [], keepdims=True, zero=np.str("")) + self._compare(x, [0], keepdims=True, zero=np.str("")) if __name__ == "__main__": test.main() -- GitLab From 01ab85f0fdce13f98b705c54901284a165ed7bd8 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 20 Apr 2018 02:53:57 +0000 Subject: [PATCH 720/791] Add n-D test cases for better coverage Signed-off-by: Yong Tang --- .../python/kernel_tests/reduction_ops_test.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tensorflow/python/kernel_tests/reduction_ops_test.py b/tensorflow/python/kernel_tests/reduction_ops_test.py index 943b80b787..ea78b58d88 100644 --- a/tensorflow/python/kernel_tests/reduction_ops_test.py +++ b/tensorflow/python/kernel_tests/reduction_ops_test.py @@ -974,5 +974,21 @@ class CountNonzeroReductionTest(test.TestCase): self._compare(x, [], keepdims=True, zero=np.str("")) self._compare(x, [0], keepdims=True, zero=np.str("")) + def testStringReduce2D(self): + # Create a 2D array of strings + x = np.asarray([["", "", "a", "", "", "b"], + ["", "c", "", "d", "", ""], + ["e", "", "f", "", "", ""]]) + self._compare(x, None, keepdims=False, zero=np.str("")) + self._compare(x, [], keepdims=False, zero=np.str("")) + self._compare(x, [0], keepdims=False, zero=np.str("")) + self._compare(x, [1], keepdims=False, zero=np.str("")) + self._compare(x, [0, 1], keepdims=False, zero=np.str("")) + self._compare(x, None, keepdims=True, zero=np.str("")) + self._compare(x, [], keepdims=True, zero=np.str("")) + self._compare(x, [0], keepdims=True, zero=np.str("")) + self._compare(x, [0, 1], keepdims=True, zero=np.str("")) + + if __name__ == "__main__": test.main() -- GitLab From 38dcc57681612c2321169367c8756bb218472dd7 Mon Sep 17 00:00:00 2001 From: Yifei Feng Date: Thu, 19 Apr 2018 19:56:09 -0700 Subject: [PATCH 721/791] Revert part of tensorflow/core/grappler/optimizers/meta_optimizer.cc from #18479. --- .../grappler/optimizers/meta_optimizer.cc | 22 +------------------ 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc index bca779c3b3..22799311bc 100644 --- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc @@ -168,26 +168,6 @@ Status MetaOptimizer::OptimizeGraph(Cluster* cluster, const GrapplerItem& item, TF_RETURN_IF_ERROR(register_by_name ? InitializeOptimizersByName(&optimizers) : InitializeOptimizers(&optimizers)); - // Append custom configurable optimizers. - std::vector - custom_configurable_optimizers; - for (const auto& optimizer : cfg_.custom_optimizers()) { - if (available_optimizers.find(optimizer.name()) != - available_optimizers.end()) { - optimizers.push_back(NewOptimizer(optimizer.name())); - } else { - custom_configurable_optimizers.push_back(optimizer); - } - } - // Now initialize and configure the custom optimizers. - for (const auto& optimizer : custom_configurable_optimizers) { - std::unique_ptr opt = - CustomGraphOptimizerRegistry::CreateByNameOrNull(optimizer.name()); - if (opt == nullptr) continue; - TF_RETURN_IF_ERROR(opt->Init(&optimizer)); - optimizers.push_back(std::move(opt)); - } - if (optimizers.empty()) { *optimized_graph = item.graph; return Status::OK(); @@ -341,7 +321,7 @@ bool MetaOptimizerEnabled(const RewriterConfig& cfg) { cfg.auto_parallel().enable() || cfg.memory_optimization() != RewriterConfig::NO_MEM_OPT || cfg.debug_stripper() == RewriterConfig::ON || - !cfg.optimizers().empty() || !cfg.custom_optimizers().empty(); + !cfg.optimizers().empty(); } Status RunMetaOptimizer(const GrapplerItem& item, const RewriterConfig& cfg, -- GitLab From 4ef9de422d452683ac661d3a6313aeb2972b836d Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Thu, 19 Apr 2018 20:00:21 -0700 Subject: [PATCH 722/791] Always include the local worker in the list of filtered targets. It is currently legal to specify a device filter that doesn't include the local worker. In that case, the MasterSession includes all local devices regardless of the filter. This change extends this behavior to the list of filtered workers, which will be crucial for backwards compatibility when we enable CreateWorkerSession for all MasterSessions, because we need to call CreateWorkerSession on all potential workers. PiperOrigin-RevId: 193613313 --- tensorflow/core/distributed_runtime/master.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/distributed_runtime/master.cc b/tensorflow/core/distributed_runtime/master.cc index 288656e7f8..e60386fd34 100644 --- a/tensorflow/core/distributed_runtime/master.cc +++ b/tensorflow/core/distributed_runtime/master.cc @@ -167,13 +167,16 @@ class DeviceFinder { } // Enumerates all known workers' target. A target name is a // prefix of a device name. E.g., /job:mnist/replica:0/task:10. + CHECK_GT(env_->local_devices.size(), 0) << "No local devices provided."; + const string& local_device_name = env_->local_devices[0]->name(); std::vector workers; worker_cache->ListWorkers(&workers); if (filters_.empty()) { std::swap(workers, targets_); } else { for (const string& name : workers) { - if (MatchFilters(name)) { + if (MatchFilters(name) || + DeviceNameUtils::IsSameAddressSpace(name, local_device_name)) { targets_.push_back(name); } } -- GitLab From ddd763de08c5095d9a0dbb8acceb82135c0aa485 Mon Sep 17 00:00:00 2001 From: imsheridan Date: Fri, 20 Apr 2018 11:08:34 +0800 Subject: [PATCH 723/791] Fix unwanted typo caused protobuf load failure --- tensorflow/core/api_def/base_api/api_def_ResourceApplyAdam.pbtxt | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/core/api_def/base_api/api_def_ResourceApplyAdam.pbtxt b/tensorflow/core/api_def/base_api/api_def_ResourceApplyAdam.pbtxt index 743247bb60..ad0aeac004 100644 --- a/tensorflow/core/api_def/base_api/api_def_ResourceApplyAdam.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_ResourceApplyAdam.pbtxt @@ -80,4 +80,5 @@ $$lr_t := \text{learning_rate} * \sqrt{(1 - beta_2^t) / (1 - beta_1^t)}$$ $$m_t := beta_1 * m_{t-1} + (1 - beta_1) * g$$ $$v_t := beta_2 * v_{t-1} + (1 - beta_2) * g * g$$ $$variable := variable - lr_t * m_t / (\sqrt{v_t} + \epsilon)$$ +END } -- GitLab From 7f3baa210a45cd0b41e21b63c2be6dd54230ea0b Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 20 Apr 2018 02:55:31 +0000 Subject: [PATCH 724/791] Update doc string for tf.count_nonzero to add string type Signed-off-by: Yong Tang --- tensorflow/python/ops/math_ops.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py index 8c9ad66b0e..31ce83905b 100644 --- a/tensorflow/python/ops/math_ops.py +++ b/tensorflow/python/ops/math_ops.py @@ -1467,7 +1467,8 @@ def count_nonzero(input_tensor, ``` Args: - input_tensor: The tensor to reduce. Should be of numeric type, or `bool`. + input_tensor: The tensor to reduce. Should be of numeric type, `string`, + or `bool`. axis: The dimensions to reduce. If `None` (the default), reduces all dimensions. Must be in the range `[-rank(input_tensor), rank(input_tensor))`. -- GitLab From 2273c4e56334caf31de01c6b6f8f4edd48432972 Mon Sep 17 00:00:00 2001 From: Yifei Feng Date: Thu, 19 Apr 2018 21:33:41 -0700 Subject: [PATCH 725/791] Skip tests with no_oss tag in XLA builds. PiperOrigin-RevId: 193619344 --- tensorflow/tools/ci_build/xla/linux/gpu/run_py3.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/ci_build/xla/linux/gpu/run_py3.sh b/tensorflow/tools/ci_build/xla/linux/gpu/run_py3.sh index a94a627dfb..a410c10b61 100755 --- a/tensorflow/tools/ci_build/xla/linux/gpu/run_py3.sh +++ b/tensorflow/tools/ci_build/xla/linux/gpu/run_py3.sh @@ -35,7 +35,7 @@ echo "build --distinct_host_configuration=false" >> .tf_configure.bazelrc bazel clean # Run bazel test command. Double test timeouts to avoid flakes. -bazel test --config=cuda --test_tag_filters=-no_gpu,-benchmark-test -k \ +bazel test --config=cuda --test_tag_filters=-no_gpu,-benchmark-test,-no_oss -k \ --jobs=${N_JOBS} --test_timeout 300,450,1200,3600 \ --build_tests_only --test_output=errors --local_test_jobs=8 \ --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \ -- GitLab From 06bb3364795e443206910c98cee132d719cf41e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Fri, 20 Apr 2018 13:33:05 +0800 Subject: [PATCH 726/791] TST: byte string for python3 --- .../python/kernel_tests/scatter_nd_ops_test.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py index dfe9600dbb..b7477a768a 100644 --- a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py +++ b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py @@ -365,31 +365,35 @@ class ScatterNdTest(test.TestCase): return array_ops.scatter_nd(indices, updates, shape) def testString(self): - indices = constant_op.constant([[4], [3], [1], [7]], dtype=dtypes.int32) + indices = constant_op.constant([[4], [3], [1], [7]], + dtype=dtypes.int32) updates = constant_op.constant(["four", "three", "one", "seven"], dtype=dtypes.string) - expected = np.array(["", "one", "", "three", "four", "", "", "seven"]) + expected = np.array([b"", b"one", b"", b"three", b"four", + b"", b"", b"seven"]) scatter = self.scatter_nd(indices, updates, shape=(8,)) with self.test_session() as sess: result = sess.run(scatter) self.assertAllEqual(expected, result) # Same indice is updated twice by same value. - indices = constant_op.constant([[4], [3], [3], [7]], dtype=dtypes.int32) + indices = constant_op.constant([[4], [3], [3], [7]], + dtype=dtypes.int32) updates = constant_op.constant(["a", "b", "b", "c"], dtype=dtypes.string) - expected = np.array(["", "", "", "bb", "a", "", "", "c"]) + expected = np.array([b"", b"", b"", b"bb", b"a", b"", b"", b"c"]) scatter = self.scatter_nd(indices, updates, shape=(8,)) with self.test_session() as sess: result = sess.run(scatter) self.assertAllEqual(expected, result) # Same indice is updated twice by different value. - indices = constant_op.constant([[4], [3], [3], [7]], dtype=dtypes.int32) + indices = constant_op.constant([[4], [3], [3], [7]], + dtype=dtypes.int32) updates = constant_op.constant(["a", "b", "c", "d"], dtype=dtypes.string) - expected = [np.array(["", "", "", "bc", "a", "", "", "d"]), - np.array(["", "", "", "cb", "a", "", "", "d"])] + expected = [np.array([b"", b"", b"", b"bc", b"a", b"", b"", b"d"]), + np.array([b"", b"", b"", b"cb", b"a", b"", b"", b"d"])] scatter = self.scatter_nd(indices, updates, shape=(8,)) with self.test_session() as sess: result = sess.run(scatter) -- GitLab From 70b8d21edcc84818835c9e2940a5df288c309d45 Mon Sep 17 00:00:00 2001 From: Roy Frostig Date: Thu, 19 Apr 2018 23:01:07 -0700 Subject: [PATCH 727/791] [XLA] Rework the local XLA client's Shape class with separate array and tuple shape constructors. PiperOrigin-RevId: 193624591 --- .../compiler/xla/python/numpy_bridge.cc | 20 +-- tensorflow/compiler/xla/python/xla_client.py | 137 ++++++++++++------ .../compiler/xla/python/xla_client_test.py | 10 +- 3 files changed, 103 insertions(+), 64 deletions(-) diff --git a/tensorflow/compiler/xla/python/numpy_bridge.cc b/tensorflow/compiler/xla/python/numpy_bridge.cc index eec48479c9..dc6f5fe5fc 100644 --- a/tensorflow/compiler/xla/python/numpy_bridge.cc +++ b/tensorflow/compiler/xla/python/numpy_bridge.cc @@ -181,16 +181,6 @@ StatusOr XlaShapeFromPyShape(PyObject* o) { PyObjectCppRepr(o).c_str()); }; - auto get_attr = [o, &error](const string& field) -> StatusOr { - PyObject* result = - PyObject_GetAttrString(o, const_cast(field.c_str())); - if (result == nullptr) { - return error(tensorflow::strings::StrCat( - "Failed to get attribute of Shape object:", field)); - } - return result; - }; - auto call_method = [o, &error](const string& method) -> StatusOr { PyObject* result = PyObject_CallMethod(o, const_cast(method.c_str()), nullptr); @@ -202,12 +192,16 @@ StatusOr XlaShapeFromPyShape(PyObject* o) { }; PyObject* np_type; - TF_ASSIGN_OR_RETURN(np_type, get_attr("np_dtype")); + TF_ASSIGN_OR_RETURN(np_type, call_method("numpy_dtype")); if (np_type->ob_type != &PyArrayDescr_Type) { - return error("Shape attribute np_dtype is not an integer numpy dtype"); + return error( + "Return value of shape method numpy_dtype " + "is not an integer numpy dtype"); } if (!NumpyTypeIsValid(NumpyTypenum(np_type))) { - return error("Shape attribute np_dtype is not a valid integer numpy dtype"); + return error( + "Return value of shape method numpy_dtype " + "is not a valid integer numpy dtype"); } const PrimitiveType element_type = NumpyTypeToPrimitiveType(NumpyTypenum(np_type)); diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py index 9c81f6439d..f6809b6b87 100644 --- a/tensorflow/compiler/xla/python/xla_client.py +++ b/tensorflow/compiler/xla/python/xla_client.py @@ -166,14 +166,14 @@ class LocalBuffer(object): self._delete = c_api.DeleteLocalShapedBuffer @staticmethod - def from_py(npval, layout_fn=None): - npval = require_numpy_array_layout(npval) + def from_pyval(pyval, layout_fn=None): + pyval = require_numpy_array_layout(pyval) if layout_fn: - shape = Shape.from_numpy(npval) + shape = Shape.from_pyval(pyval) shape = shape.map_leaves(layout_fn) else: shape = None - return LocalBuffer(c_api.LocalShapedBuffer.FromLiteral(npval, shape)) + return LocalBuffer(c_api.LocalShapedBuffer.FromLiteral(pyval, shape)) def to_py(self): return self.c_local_shaped_buffer.ToLiteral() @@ -191,53 +191,104 @@ class LocalBuffer(object): class Shape(object): - """XLA shape. + """Represents an XLA shape. - Represents an XLA shape by a corresponding Python/Numpy type and a - list of dimensions, which are themselves Shapes in case this one - represents an XLA tuple. + A shape is either an array shape, having rank-many integer + dimensions and an element type (represented by a Numpy dtype), or it + is a tuple shape, having a shape for every tuple component: + + type shape = + TupleShape of shape list + | ArrayShape of { dimensions: int list; element_type: dtype } + + Callers are expected to instantiate this class only via the static + constructors: tuple_shape, array_shape, and from_pyval. """ - def __init__(self, np_dtype, dimensions, minor_to_major=None): + @staticmethod + def tuple_shape(tuple_shapes): + """Construct a tuple shape.""" + if (not isinstance(tuple_shapes, (tuple, list)) or + not all(isinstance(t, Shape) for t in tuple_shapes)): + raise TypeError('tuple_shapes must be a tuple of Shapes') + return Shape(tuple_shapes, tuple) + + @staticmethod + def array_shape(element_type, dimensions, minor_to_major=None): + """Construct an array shape.""" + if (not isinstance(dimensions, tuple) or + not all(isinstance(i, int) for i in dimensions)): + dimensions = tuple(int(i) for i in dimensions) + return Shape(dimensions, np.dtype(element_type), + minor_to_major=minor_to_major) + + @staticmethod + def from_pyval(pyval): + def convert(pyval): + if isinstance(pyval, tuple): + return Shape.tuple_shape(tuple(convert(elt) for elt in pyval)) + else: + pyval = require_numpy_array_layout(pyval) + return Shape.array_shape(pyval.dtype, np.shape(pyval)) + return convert(pyval) + + def __init__(self, dimensions, dtype, minor_to_major=None): assert isinstance(dimensions, tuple) - self.np_dtype = np_dtype self._dimensions = dimensions + self._dtype = dtype + self._is_tuple = dtype == tuple self._minor_to_major = minor_to_major self._check_minor_to_major() def __eq__(self, other): # pylint: disable=protected-access - return (self.np_dtype == other.np_dtype and + return (self._dtype == other._dtype and self._dimensions == other._dimensions and self._minor_to_major == other._minor_to_major) def __repr__(self): - return ('xla_client.Shape(np_dtype={!r}, dimensions={!r}, ' - 'minor_to_major={!r})').format(self.np_dtype, self._dimensions, - self._minor_to_major) - - def element_type(self): - return DTYPE_TO_XLA_ELEMENT_TYPE[str(self.np_dtype)] + return ('xla_client.Shape(_dtype={!r}, _dimensions={!r}, ' + '_is_tuple={!r}), _minor_to_major={!r}').format( + self._dtype, self._dimensions, self._is_tuple, + self._minor_to_major) def is_tuple(self): - return self.element_type() == xla_data_pb2.TUPLE + return self._is_tuple - def dimensions(self): - if self.is_tuple(): - raise ValueError('Tuple shape has no dimensions') - return self._dimensions - - def minor_to_major(self): - return self._minor_to_major + def is_array(self): + return not self._is_tuple def tuple_shapes(self): if not self.is_tuple(): - raise ValueError('Shape is not a tuple shape') + raise ValueError('not a tuple shape') + return self._dimensions + + def numpy_dtype(self): + """Like element_type(), but returns dtype('O') in case of a tuple shape.""" + if self.is_tuple(): + return np.dtype(np.object) + else: + return self.element_type() + + def xla_element_type(self): + return DTYPE_TO_XLA_ELEMENT_TYPE[str(self.numpy_dtype())] + + def element_type(self): + if not self.is_array(): + raise ValueError('not an array shape') + return self._dtype + + def dimensions(self): + if not self.is_array(): + raise ValueError('not an array shape') return self._dimensions def rank(self): return len(self.dimensions()) + def minor_to_major(self): + return self._minor_to_major + def map_leaves(self, f): """Map f over each leaf-level array subshape. @@ -250,7 +301,7 @@ class Shape(object): """ if self.is_tuple(): children = tuple(child.map_leaves(f) for child in self.tuple_shapes()) - return Shape(np.dtype('O'), children) + return Shape.tuple_shape(children) else: mapped = f(self) return self if mapped is None else mapped @@ -264,30 +315,24 @@ class Shape(object): assert sorted(mtm) == range(len(mtm)), self def update_minor_to_major(self, minor_to_major): + if not self.is_array(): + raise ValueError('not an array shape') if not isinstance(minor_to_major, tuple): raise TypeError('minor_to_major must be a tuple') - updated = Shape(self.np_dtype, tuple(self.dimensions()), minor_to_major) + updated = Shape.array_shape( + self.element_type(), self.dimensions(), minor_to_major) updated._check_minor_to_major() # pylint: disable=protected-access return updated - @staticmethod - def from_numpy(npval): - - def convert(npval): - if isinstance(npval, tuple): - return Shape(np.dtype('O'), tuple(convert(elt) for elt in npval)) - else: - return Shape(npval.dtype, np.shape(npval)) - - return convert(require_numpy_array_layout(npval)) - def _wrap_shape(shape_info): dtype, dims = shape_info element_type = DTYPE_TO_XLA_ELEMENT_TYPE[str(dtype)] if element_type == xla_data_pb2.TUPLE: - dims = tuple(_wrap_shape(subshape_info) for subshape_info in dims) - return Shape(dtype, dims) + shapes = tuple(_wrap_shape(subshape_info) for subshape_info in dims) + return Shape.tuple_shape(shapes) + else: + return Shape.array_shape(dtype, dims) def _wrap_data_handle(handle): @@ -420,7 +465,7 @@ class LocalComputation(object): compile_options=None, layout_fn=None): return self.Compile( - argument_shapes=[Shape.from_numpy(arg) for arg in arguments], + argument_shapes=[Shape.from_pyval(arg) for arg in arguments], compile_options=compile_options, layout_fn=layout_fn) @@ -428,7 +473,7 @@ class LocalComputation(object): """Execute with Python values as arguments and return value.""" if not self.is_compiled: raise ValueError('Cannot execute an uncompiled local XLA computation.') - argument_shapes = [Shape.from_numpy(arg) for arg in arguments] + argument_shapes = [Shape.from_pyval(arg) for arg in arguments] if layout_fn: argument_shapes = [ shape.map_leaves(layout_fn) for shape in argument_shapes @@ -607,7 +652,7 @@ class ComputationBuilder(object): A ComputationDataHandle message. """ return self.ParameterWithShape( - Shape.from_numpy(value), name=name, parameter_num=parameter_num) + Shape.from_pyval(value), name=name, parameter_num=parameter_num) def Broadcast(self, operand, sizes): """Enqueues a broadcast operation onto the computation. @@ -968,7 +1013,7 @@ class ComputationBuilder(object): Returns: a ComputationDataHandle to the generated array of F32 values. """ - shape = Shape(self.GetShape(mu).np_dtype, dims) + shape = Shape.array_shape(self.GetShape(mu).element_type(), dims) return _wrap_data_handle( self._client.RngNormal( _unwrap_data_handle(mu), _unwrap_data_handle(sigma), shape)) @@ -988,7 +1033,7 @@ class ComputationBuilder(object): Returns: a ComputationDataHandle to the generated array of values with the same numeric type (F32, S32, or U32) as the arguments a and b. """ - shape = Shape(self.GetShape(a).np_dtype, dims) + shape = Shape.array_shape(self.GetShape(a).element_type(), dims) return _wrap_data_handle( self._client.RngUniform( _unwrap_data_handle(a), _unwrap_data_handle(b), shape)) diff --git a/tensorflow/compiler/xla/python/xla_client_test.py b/tensorflow/compiler/xla/python/xla_client_test.py index d97264ea64..6fe7b242e4 100644 --- a/tensorflow/compiler/xla/python/xla_client_test.py +++ b/tensorflow/compiler/xla/python/xla_client_test.py @@ -319,7 +319,7 @@ class LocalBufferTest(LocalComputationTest): def _Execute(self, c, arguments): compiled_c = c.Build().CompileWithExampleArguments(arguments) - arg_buffers = [xla_client.LocalBuffer.from_py(arg) for arg in arguments] + arg_buffers = [xla_client.LocalBuffer.from_pyval(arg) for arg in arguments] result_buffer = compiled_c.ExecuteWithLocalBuffers(arg_buffers) return result_buffer.to_py() @@ -350,7 +350,7 @@ class LocalBufferTest(LocalComputationTest): c.Add(c.ParameterFromNumpy(NumpyArrayF32(0.)), c.ConstantF32Scalar(3.14)) arg = NumpyArrayF32(1.11) compiled_c = c.Build().CompileWithExampleArguments([arg]) - arg_buffer = xla_client.LocalBuffer.from_py(arg) + arg_buffer = xla_client.LocalBuffer.from_pyval(arg) arg_buffer.delete() with self.assertRaises(ValueError): compiled_c.ExecuteWithLocalBuffers([arg_buffer]) @@ -1288,7 +1288,7 @@ class EmbeddedComputationsTest(LocalComputationTest): def testInfeedS32Values(self): to_infeed = NumpyArrayS32([1, 2, 3, 4]) c = self._NewComputation() - c.Infeed(xla_client.Shape.from_numpy(to_infeed[0])) + c.Infeed(xla_client.Shape.from_pyval(to_infeed[0])) compiled_c = c.Build().CompileWithExampleArguments() for item in to_infeed: xla_client.transfer_to_infeed(item) @@ -1300,7 +1300,7 @@ class EmbeddedComputationsTest(LocalComputationTest): def testInfeedThenOutfeedS32(self): to_round_trip = NumpyArrayS32([1, 2, 3, 4]) c = self._NewComputation() - x = c.Infeed(xla_client.Shape.from_numpy(to_round_trip[0])) + x = c.Infeed(xla_client.Shape.from_pyval(to_round_trip[0])) c.Outfeed(x) compiled_c = c.Build().CompileWithExampleArguments() @@ -1310,7 +1310,7 @@ class EmbeddedComputationsTest(LocalComputationTest): execution.start() xla_client.transfer_to_infeed(want) got = xla_client.transfer_from_outfeed( - xla_client.Shape.from_numpy(to_round_trip[0])) + xla_client.Shape.from_pyval(to_round_trip[0])) execution.join() self.assertEqual(want, got) -- GitLab From f7e8fbb28a0fa4e979a94d7b458706abf48f7deb Mon Sep 17 00:00:00 2001 From: Asim Shankar Date: Thu, 19 Apr 2018 23:08:53 -0700 Subject: [PATCH 728/791] Automated g4 rollback of changelist 193602050 PiperOrigin-RevId: 193625346 --- tensorflow/core/lib/io/record_reader.cc | 147 ++++---------- tensorflow/core/lib/io/record_reader.h | 16 +- tensorflow/core/lib/io/recordio_test.cc | 212 ++++++++++++++------- tensorflow/core/lib/io/zlib_inputstream.cc | 16 +- tensorflow/core/lib/io/zlib_inputstream.h | 19 +- 5 files changed, 220 insertions(+), 190 deletions(-) diff --git a/tensorflow/core/lib/io/record_reader.cc b/tensorflow/core/lib/io/record_reader.cc index 6de850bb20..c24628be57 100644 --- a/tensorflow/core/lib/io/record_reader.cc +++ b/tensorflow/core/lib/io/record_reader.cc @@ -56,110 +56,55 @@ RecordReaderOptions RecordReaderOptions::CreateRecordReaderOptions( RecordReader::RecordReader(RandomAccessFile* file, const RecordReaderOptions& options) - : src_(file), options_(options) { + : options_(options), + input_stream_(new RandomAccessInputStream(file)), + last_read_failed_(false) { if (options.buffer_size > 0) { - input_stream_.reset(new BufferedInputStream(file, options.buffer_size)); - } else { - input_stream_.reset(new RandomAccessInputStream(file)); + input_stream_.reset(new BufferedInputStream(input_stream_.release(), + options.buffer_size, true)); } if (options.compression_type == RecordReaderOptions::ZLIB_COMPRESSION) { // We don't have zlib available on all embedded platforms, so fail. #if defined(IS_SLIM_BUILD) LOG(FATAL) << "Zlib compression is unsupported on mobile platforms."; #else // IS_SLIM_BUILD - zlib_input_stream_.reset(new ZlibInputStream( - input_stream_.get(), options.zlib_options.input_buffer_size, - options.zlib_options.output_buffer_size, options.zlib_options)); + input_stream_.reset(new ZlibInputStream( + input_stream_.release(), options.zlib_options.input_buffer_size, + options.zlib_options.output_buffer_size, options.zlib_options, true)); #endif // IS_SLIM_BUILD } else if (options.compression_type == RecordReaderOptions::NONE) { // Nothing to do. } else { - LOG(FATAL) << "Unspecified compression type :" << options.compression_type; + LOG(FATAL) << "Unrecognized compression type :" << options.compression_type; } } // Read n+4 bytes from file, verify that checksum of first n bytes is // stored in the last 4 bytes and store the first n bytes in *result. -// May use *storage as backing store. -Status RecordReader::ReadChecksummed(uint64 offset, size_t n, - StringPiece* result, string* storage) { +// +// offset corresponds to the user-provided value to ReadRecord() +// and is used only in error messages. +Status RecordReader::ReadChecksummed(uint64 offset, size_t n, string* result) { if (n >= SIZE_MAX - sizeof(uint32)) { return errors::DataLoss("record size too large"); } const size_t expected = n + sizeof(uint32); - storage->resize(expected); - -#if !defined(IS_SLIM_BUILD) - if (zlib_input_stream_) { - // If we have a zlib compressed buffer, we assume that the - // file is being read sequentially, and we use the underlying - // implementation to read the data. - // - // No checks are done to validate that the file is being read - // sequentially. At some point the zlib input buffer may support - // seeking, possibly inefficiently. - TF_RETURN_IF_ERROR(zlib_input_stream_->ReadNBytes(expected, storage)); - - if (storage->size() != expected) { - if (storage->empty()) { - return errors::OutOfRange("eof"); - } else { - return errors::DataLoss("truncated record at ", offset); - } - } + TF_RETURN_IF_ERROR(input_stream_->ReadNBytes(expected, result)); - uint32 masked_crc = core::DecodeFixed32(storage->data() + n); - if (crc32c::Unmask(masked_crc) != crc32c::Value(storage->data(), n)) { - return errors::DataLoss("corrupted record at ", offset); - } - *result = StringPiece(storage->data(), n); - } else { -#endif // IS_SLIM_BUILD - if (options_.buffer_size > 0) { - // If we have a buffer, we assume that the file is being read - // sequentially, and we use the underlying implementation to read the - // data. - // - // No checks are done to validate that the file is being read - // sequentially. - TF_RETURN_IF_ERROR(input_stream_->ReadNBytes(expected, storage)); - - if (storage->size() != expected) { - if (storage->empty()) { - return errors::OutOfRange("eof"); - } else { - return errors::DataLoss("truncated record at ", offset); - } - } - - const uint32 masked_crc = core::DecodeFixed32(storage->data() + n); - if (crc32c::Unmask(masked_crc) != crc32c::Value(storage->data(), n)) { - return errors::DataLoss("corrupted record at ", offset); - } - *result = StringPiece(storage->data(), n); + if (result->size() != expected) { + if (result->empty()) { + return errors::OutOfRange("eof"); } else { - // This version supports reading from arbitrary offsets - // since we are accessing the random access file directly. - StringPiece data; - TF_RETURN_IF_ERROR(src_->Read(offset, expected, &data, &(*storage)[0])); - if (data.size() != expected) { - if (data.empty()) { - return errors::OutOfRange("eof"); - } else { - return errors::DataLoss("truncated record at ", offset); - } - } - const uint32 masked_crc = core::DecodeFixed32(data.data() + n); - if (crc32c::Unmask(masked_crc) != crc32c::Value(data.data(), n)) { - return errors::DataLoss("corrupted record at ", offset); - } - *result = StringPiece(data.data(), n); + return errors::DataLoss("truncated record at ", offset); } -#if !defined(IS_SLIM_BUILD) } -#endif // IS_SLIM_BUILD + const uint32 masked_crc = core::DecodeFixed32(result->data() + n); + if (crc32c::Unmask(masked_crc) != crc32c::Value(result->data(), n)) { + return errors::DataLoss("corrupted record at ", offset); + } + result->resize(n); return Status::OK(); } @@ -167,50 +112,42 @@ Status RecordReader::ReadRecord(uint64* offset, string* record) { static const size_t kHeaderSize = sizeof(uint64) + sizeof(uint32); static const size_t kFooterSize = sizeof(uint32); + // Position the input stream. + int64 curr_pos = input_stream_->Tell(); + int64 desired_pos = static_cast(*offset); + if (curr_pos > desired_pos || curr_pos < 0 /* EOF */ || + (curr_pos == desired_pos && last_read_failed_)) { + last_read_failed_ = false; + TF_RETURN_IF_ERROR(input_stream_->Reset()); + TF_RETURN_IF_ERROR(input_stream_->SkipNBytes(desired_pos)); + } else if (curr_pos < desired_pos) { + TF_RETURN_IF_ERROR(input_stream_->SkipNBytes(desired_pos - curr_pos)); + } + DCHECK_EQ(desired_pos, input_stream_->Tell()); + // Read header data. - StringPiece lbuf; - Status s = ReadChecksummed(*offset, sizeof(uint64), &lbuf, record); + Status s = ReadChecksummed(*offset, sizeof(uint64), record); if (!s.ok()) { + last_read_failed_ = true; return s; } - const uint64 length = core::DecodeFixed64(lbuf.data()); + const uint64 length = core::DecodeFixed64(record->data()); // Read data - StringPiece data; - s = ReadChecksummed(*offset + kHeaderSize, length, &data, record); + s = ReadChecksummed(*offset + kHeaderSize, length, record); if (!s.ok()) { + last_read_failed_ = true; if (errors::IsOutOfRange(s)) { s = errors::DataLoss("truncated record at ", *offset); } return s; } - if (record->data() != data.data()) { - // RandomAccessFile placed the data in some other location. - memmove(&(*record)[0], data.data(), data.size()); - } - - record->resize(data.size()); - *offset += kHeaderSize + length + kFooterSize; + DCHECK_EQ(*offset, input_stream_->Tell()); return Status::OK(); } -Status RecordReader::SkipNBytes(uint64 offset) { -#if !defined(IS_SLIM_BUILD) - if (zlib_input_stream_) { - TF_RETURN_IF_ERROR(zlib_input_stream_->SkipNBytes(offset)); - } else { -#endif - if (options_.buffer_size > 0) { - TF_RETURN_IF_ERROR(input_stream_->SkipNBytes(offset)); - } -#if !defined(IS_SLIM_BUILD) - } -#endif - return Status::OK(); -} // namespace io - SequentialRecordReader::SequentialRecordReader( RandomAccessFile* file, const RecordReaderOptions& options) : underlying_(file, options), offset_(0) {} diff --git a/tensorflow/core/lib/io/record_reader.h b/tensorflow/core/lib/io/record_reader.h index 26278e0328..f6d587dfa0 100644 --- a/tensorflow/core/lib/io/record_reader.h +++ b/tensorflow/core/lib/io/record_reader.h @@ -69,25 +69,14 @@ class RecordReader { // Read the record at "*offset" into *record and update *offset to // point to the offset of the next record. Returns OK on success, // OUT_OF_RANGE for end of file, or something else for an error. - // - // Note: if buffering is used (with or without compression), access must be - // sequential. Status ReadRecord(uint64* offset, string* record); - // Skip the records till "offset". Returns OK on success, - // OUT_OF_RANGE for end of file, or something else for an error. - Status SkipNBytes(uint64 offset); - private: - Status ReadChecksummed(uint64 offset, size_t n, StringPiece* result, - string* storage); + Status ReadChecksummed(uint64 offset, size_t n, string* result); - RandomAccessFile* src_; RecordReaderOptions options_; std::unique_ptr input_stream_; -#if !defined(IS_SLIM_BUILD) - std::unique_ptr zlib_input_stream_; -#endif // IS_SLIM_BUILD + bool last_read_failed_; TF_DISALLOW_COPY_AND_ASSIGN(RecordReader); }; @@ -121,7 +110,6 @@ class SequentialRecordReader { return errors::InvalidArgument( "Trying to seek offset: ", offset, " which is less than the current offset: ", offset_); - TF_RETURN_IF_ERROR(underlying_.SkipNBytes(offset - offset_)); offset_ = offset; return Status::OK(); } diff --git a/tensorflow/core/lib/io/recordio_test.cc b/tensorflow/core/lib/io/recordio_test.cc index 63235761d9..da514bd21c 100644 --- a/tensorflow/core/lib/io/recordio_test.cc +++ b/tensorflow/core/lib/io/recordio_test.cc @@ -26,10 +26,11 @@ limitations under the License. namespace tensorflow { namespace io { +namespace { // Construct a string of the specified length made out of the supplied // partial string. -static string BigString(const string& partial_string, size_t n) { +string BigString(const string& partial_string, size_t n) { string result; while (result.size() < n) { result.append(partial_string); @@ -39,62 +40,66 @@ static string BigString(const string& partial_string, size_t n) { } // Construct a string from a number -static string NumberString(int n) { +string NumberString(int n) { char buf[50]; snprintf(buf, sizeof(buf), "%d.", n); return string(buf); } // Return a skewed potentially long string -static string RandomSkewedString(int i, random::SimplePhilox* rnd) { +string RandomSkewedString(int i, random::SimplePhilox* rnd) { return BigString(NumberString(i), rnd->Skewed(17)); } -class RecordioTest : public ::testing::Test { +class StringDest : public WritableFile { + public: + explicit StringDest(string* contents) : contents_(contents) {} + + Status Close() override { return Status::OK(); } + Status Flush() override { return Status::OK(); } + Status Sync() override { return Status::OK(); } + Status Append(const StringPiece& slice) override { + contents_->append(slice.data(), slice.size()); + return Status::OK(); + } + private: - class StringDest : public WritableFile { - public: - string contents_; - - Status Close() override { return Status::OK(); } - Status Flush() override { return Status::OK(); } - Status Sync() override { return Status::OK(); } - Status Append(const StringPiece& slice) override { - contents_.append(slice.data(), slice.size()); - return Status::OK(); + string* contents_; +}; + +class StringSource : public RandomAccessFile { + public: + explicit StringSource(string* contents) + : contents_(contents), force_error_(false) {} + + Status Read(uint64 offset, size_t n, StringPiece* result, + char* scratch) const override { + if (force_error_) { + force_error_ = false; + return errors::DataLoss("read error"); } - }; - - class StringSource : public RandomAccessFile { - public: - StringPiece contents_; - mutable bool force_error_; - mutable bool returned_partial_; - StringSource() : force_error_(false), returned_partial_(false) {} - - Status Read(uint64 offset, size_t n, StringPiece* result, - char* scratch) const override { - EXPECT_FALSE(returned_partial_) << "must not Read() after eof/error"; - - if (force_error_) { - force_error_ = false; - returned_partial_ = true; - return errors::DataLoss("read error"); - } - - if (offset >= contents_.size()) { - return errors::OutOfRange("end of file"); - } - - if (contents_.size() < offset + n) { - n = contents_.size() - offset; - returned_partial_ = true; - } - *result = StringPiece(contents_.data() + offset, n); - return Status::OK(); + + if (offset >= contents_->size()) { + return errors::OutOfRange("end of file"); + } + + if (contents_->size() < offset + n) { + n = contents_->size() - offset; } - }; + *result = StringPiece(contents_->data() + offset, n); + return Status::OK(); + } + + void force_error() { force_error_ = true; } + + private: + string* contents_; + mutable bool force_error_; +}; +class RecordioTest : public ::testing::Test { + private: + string contents_; StringDest dest_; StringSource source_; bool reading_; @@ -104,7 +109,9 @@ class RecordioTest : public ::testing::Test { public: RecordioTest() - : reading_(false), + : dest_(&contents_), + source_(&contents_), + reading_(false), readpos_(0), writer_(new RecordWriter(&dest_)), reader_(new RecordReader(&source_)) {} @@ -119,12 +126,11 @@ class RecordioTest : public ::testing::Test { TF_ASSERT_OK(writer_->WriteRecord(StringPiece(msg))); } - size_t WrittenBytes() const { return dest_.contents_.size(); } + size_t WrittenBytes() const { return contents_.size(); } string Read() { if (!reading_) { reading_ = true; - source_.contents_ = StringPiece(dest_.contents_); } string record; Status s = reader_->ReadRecord(&readpos_, &record); @@ -137,26 +143,20 @@ class RecordioTest : public ::testing::Test { } } - void IncrementByte(int offset, int delta) { - dest_.contents_[offset] += delta; - } + void IncrementByte(int offset, int delta) { contents_[offset] += delta; } - void SetByte(int offset, char new_byte) { - dest_.contents_[offset] = new_byte; - } + void SetByte(int offset, char new_byte) { contents_[offset] = new_byte; } - void ShrinkSize(int bytes) { - dest_.contents_.resize(dest_.contents_.size() - bytes); - } + void ShrinkSize(int bytes) { contents_.resize(contents_.size() - bytes); } void FixChecksum(int header_offset, int len) { // Compute crc of type/len/data - uint32_t crc = crc32c::Value(&dest_.contents_[header_offset + 6], 1 + len); + uint32_t crc = crc32c::Value(&contents_[header_offset + 6], 1 + len); crc = crc32c::Mask(crc); - core::EncodeFixed32(&dest_.contents_[header_offset], crc); + core::EncodeFixed32(&contents_[header_offset], crc); } - void ForceError() { source_.force_error_ = true; } + void ForceError() { source_.force_error(); } void StartReadingAt(uint64_t initial_offset) { readpos_ = initial_offset; } @@ -165,7 +165,6 @@ class RecordioTest : public ::testing::Test { Write("bar"); Write(BigString("x", 10000)); reading_ = true; - source_.contents_ = StringPiece(dest_.contents_); uint64 offset = WrittenBytes() + offset_past_end; string record; Status s = reader_->ReadRecord(&offset, &record); @@ -217,16 +216,100 @@ TEST_F(RecordioTest, RandomRead) { ASSERT_EQ("EOF", Read()); } +void TestNonSequentialReads(const RecordWriterOptions& writer_options, + const RecordReaderOptions& reader_options) { + string contents; + StringDest dst(&contents); + RecordWriter writer(&dst, writer_options); + for (int i = 0; i < 10; ++i) { + TF_ASSERT_OK(writer.WriteRecord(NumberString(i))) << i; + } + TF_ASSERT_OK(writer.Close()); + + StringSource file(&contents); + RecordReader reader(&file, reader_options); + + string record; + // First read sequentially to fill in the offsets table. + uint64 offsets[10] = {0}; + uint64 offset = 0; + for (int i = 0; i < 10; ++i) { + offsets[i] = offset; + TF_ASSERT_OK(reader.ReadRecord(&offset, &record)) << i; + } + + // Read randomly: First go back to record #3 then forward to #8. + offset = offsets[3]; + TF_ASSERT_OK(reader.ReadRecord(&offset, &record)); + EXPECT_EQ("3.", record); + EXPECT_EQ(offsets[4], offset); + + offset = offsets[8]; + TF_ASSERT_OK(reader.ReadRecord(&offset, &record)); + EXPECT_EQ("8.", record); + EXPECT_EQ(offsets[9], offset); +} + +TEST_F(RecordioTest, NonSequentialReads) { + TestNonSequentialReads(RecordWriterOptions(), RecordReaderOptions()); +} + +TEST_F(RecordioTest, NonSequentialReadsWithReadBuffer) { + RecordReaderOptions options; + options.buffer_size = 1 << 10; + TestNonSequentialReads(RecordWriterOptions(), options); +} + +TEST_F(RecordioTest, NonSequentialReadsWithCompression) { + TestNonSequentialReads( + RecordWriterOptions::CreateRecordWriterOptions("ZLIB"), + RecordReaderOptions::CreateRecordReaderOptions("ZLIB")); +} + // Tests of all the error paths in log_reader.cc follow: -static void AssertHasSubstr(StringPiece s, StringPiece expected) { +void AssertHasSubstr(StringPiece s, StringPiece expected) { EXPECT_TRUE(str_util::StrContains(s, expected)) << s << " does not contain " << expected; } +void TestReadError(const RecordWriterOptions& writer_options, + const RecordReaderOptions& reader_options) { + const string wrote = BigString("well hello there!", 100); + string contents; + StringDest dst(&contents); + TF_ASSERT_OK(RecordWriter(&dst, writer_options).WriteRecord(wrote)); + + StringSource file(&contents); + RecordReader reader(&file, reader_options); + + uint64 offset = 0; + string read; + file.force_error(); + Status status = reader.ReadRecord(&offset, &read); + ASSERT_TRUE(errors::IsDataLoss(status)); + ASSERT_EQ(0, offset); + + // A failed Read() shouldn't update the offset, and thus a retry shouldn't + // lose the record. + status = reader.ReadRecord(&offset, &read); + ASSERT_TRUE(status.ok()) << status; + EXPECT_GT(offset, 0); + EXPECT_EQ(wrote, read); +} + TEST_F(RecordioTest, ReadError) { - Write("foo"); - ForceError(); - AssertHasSubstr(Read(), "Data loss"); + TestReadError(RecordWriterOptions(), RecordReaderOptions()); +} + +TEST_F(RecordioTest, ReadErrorWithBuffering) { + RecordReaderOptions options; + options.buffer_size = 1 << 20; + TestReadError(RecordWriterOptions(), options); +} + +TEST_F(RecordioTest, ReadErrorWithCompression) { + TestReadError(RecordWriterOptions::CreateRecordWriterOptions("ZLIB"), + RecordReaderOptions::CreateRecordReaderOptions("ZLIB")); } TEST_F(RecordioTest, CorruptLength) { @@ -257,5 +340,6 @@ TEST_F(RecordioTest, ReadEnd) { CheckOffsetPastEndReturnsNoRecords(0); } TEST_F(RecordioTest, ReadPastEnd) { CheckOffsetPastEndReturnsNoRecords(5); } +} // namespace } // namespace io } // namespace tensorflow diff --git a/tensorflow/core/lib/io/zlib_inputstream.cc b/tensorflow/core/lib/io/zlib_inputstream.cc index 984fbc2810..47de36bf6c 100644 --- a/tensorflow/core/lib/io/zlib_inputstream.cc +++ b/tensorflow/core/lib/io/zlib_inputstream.cc @@ -25,8 +25,9 @@ ZlibInputStream::ZlibInputStream( InputStreamInterface* input_stream, size_t input_buffer_bytes, // size of z_stream.next_in buffer size_t output_buffer_bytes, // size of z_stream.next_out buffer - const ZlibCompressionOptions& zlib_options) - : input_stream_(input_stream), + const ZlibCompressionOptions& zlib_options, bool owns_input_stream) + : owns_input_stream_(owns_input_stream), + input_stream_(input_stream), input_buffer_capacity_(input_buffer_bytes), output_buffer_capacity_(output_buffer_bytes), z_stream_input_(new Bytef[input_buffer_capacity_]), @@ -37,14 +38,25 @@ ZlibInputStream::ZlibInputStream( InitZlibBuffer(); } +ZlibInputStream::ZlibInputStream(InputStreamInterface* input_stream, + size_t input_buffer_bytes, + size_t output_buffer_bytes, + const ZlibCompressionOptions& zlib_options) + : ZlibInputStream(input_stream, input_buffer_bytes, output_buffer_bytes, + zlib_options, false) {} + ZlibInputStream::~ZlibInputStream() { if (z_stream_) { inflateEnd(z_stream_.get()); } + if (owns_input_stream_) { + delete input_stream_; + } } Status ZlibInputStream::Reset() { TF_RETURN_IF_ERROR(input_stream_->Reset()); + inflateEnd(z_stream_.get()); InitZlibBuffer(); bytes_read_ = 0; return Status::OK(); diff --git a/tensorflow/core/lib/io/zlib_inputstream.h b/tensorflow/core/lib/io/zlib_inputstream.h index 9c7e14441c..37339163ee 100644 --- a/tensorflow/core/lib/io/zlib_inputstream.h +++ b/tensorflow/core/lib/io/zlib_inputstream.h @@ -40,7 +40,15 @@ class ZlibInputStream : public InputStreamInterface { // Create a ZlibInputStream for `input_stream` with a buffer of size // `input_buffer_bytes` bytes for reading contents from `input_stream` and // another buffer with size `output_buffer_bytes` for caching decompressed - // contents. Does *not* take ownership of "input_stream". + // contents. + // + // Takes ownership of `input_stream` iff `owns_input_stream` is true. + ZlibInputStream(InputStreamInterface* input_stream, size_t input_buffer_bytes, + size_t output_buffer_bytes, + const ZlibCompressionOptions& zlib_options, + bool owns_input_stream); + + // Equivalent to the previous constructor with owns_input_stream=false. ZlibInputStream(InputStreamInterface* input_stream, size_t input_buffer_bytes, size_t output_buffer_bytes, const ZlibCompressionOptions& zlib_options); @@ -65,10 +73,11 @@ class ZlibInputStream : public InputStreamInterface { private: void InitZlibBuffer(); - InputStreamInterface* input_stream_; // Not owned - size_t input_buffer_capacity_; // Size of z_stream_input_ - size_t output_buffer_capacity_; // Size of z_stream_output_ - char* next_unread_byte_; // Next unread byte in z_stream_output_ + const bool owns_input_stream_; + InputStreamInterface* input_stream_; + size_t input_buffer_capacity_; // Size of z_stream_input_ + size_t output_buffer_capacity_; // Size of z_stream_output_ + char* next_unread_byte_; // Next unread byte in z_stream_output_ // Buffer for storing contents read from compressed stream. // TODO(srbs): Consider using circular buffers. That would greatly simplify -- GitLab From d2fd0bbac6368a6b41e73d18c93b24442f5653f1 Mon Sep 17 00:00:00 2001 From: Dimitris Vardoulakis Date: Thu, 19 Apr 2018 23:35:04 -0700 Subject: [PATCH 729/791] [TF:XLA] Factor out the handling of while instructions to make HloVerifier::Run shorter. PiperOrigin-RevId: 193626864 --- .../compiler/xla/service/hlo_verifier.cc | 83 +++++++++++-------- .../compiler/xla/service/hlo_verifier.h | 8 +- 2 files changed, 55 insertions(+), 36 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc index 8c875698eb..80ed6d6832 100644 --- a/tensorflow/compiler/xla/service/hlo_verifier.cc +++ b/tensorflow/compiler/xla/service/hlo_verifier.cc @@ -731,6 +731,55 @@ Status HloVerifier::CheckFusionInstruction(HloInstruction* fusion) const { return tensorflow::Status::OK(); } +Status HloVerifier::CheckWhileInstruction(HloInstruction* instruction) { + auto* while_cond = instruction->while_condition(); + auto* while_body = instruction->while_body(); + if (while_cond->num_parameters() != 1) { + return FailedPrecondition( + "While condition must have exactly 1 parameter; had %lld : %s", + while_cond->num_parameters(), while_cond->ToString().c_str()); + } + if (while_body->num_parameters() != 1) { + return FailedPrecondition( + "While body must have exactly 1 parameter; had %lld : %s", + while_body->num_parameters(), while_body->ToString().c_str()); + } + if (instruction->operand_count() != 1) { + return FailedPrecondition( + "While loop must have exactly one operand; had %lld : %s", + instruction->operand_count(), instruction->ToString().c_str()); + } + auto* init = instruction->operand(0); + auto* cond_param = while_cond->parameter_instruction(0); + if (!ShapeUtil::Compatible(init->shape(), cond_param->shape())) { + return FailedPrecondition( + "While condition's parameter must have the same shape as the " + "loop's 'init'. init: %s, param: %s", + init->ToString().c_str(), cond_param->ToString().c_str()); + } + auto* cond_root = while_cond->root_instruction(); + if (!ShapeUtil::Compatible(cond_root->shape(), + ShapeUtil::MakeShape(PRED, {}))) { + return FailedPrecondition("While condition should have shape PRED: %s", + cond_root->ToString().c_str()); + } + auto* body_param = while_body->parameter_instruction(0); + if (!ShapeUtil::Compatible(init->shape(), body_param->shape())) { + return FailedPrecondition( + "While body's parameter must have the same shape as the loop's" + " 'init'. init: %s, param: %s", + init->ToString().c_str(), body_param->ToString().c_str()); + } + auto* body_root = while_body->root_instruction(); + if (!ShapeUtil::Compatible(init->shape(), body_root->shape())) { + return FailedPrecondition( + "While body should have same shape as the loop's 'init'." + "init: %s, body: %s", + init->ToString().c_str(), body_root->ToString().c_str()); + } + return tensorflow::Status::OK(); +} + StatusOr HloVerifier::Run(HloModule* module) { TF_RETURN_IF_ERROR(VerifyHloStructure(module)); @@ -771,39 +820,7 @@ StatusOr HloVerifier::Run(HloModule* module) { << instruction->dimensions().size() << " != " << ShapeUtil::Rank(instruction->operand(0)->shape()); } else if (instruction->opcode() == HloOpcode::kWhile) { - auto* while_cond = instruction->while_condition(); - auto* while_body = instruction->while_body(); - TF_RET_CHECK(while_cond->num_parameters() == 1) - << "While condition must have exactly 1 parameter; had " - << while_cond->num_parameters() << ": " << while_cond->ToString(); - TF_RET_CHECK(while_body->num_parameters() == 1) - << "While body must have exactly 1 parameter; had " - << while_body->num_parameters() << ": " << while_body->ToString(); - TF_RET_CHECK(instruction->operand_count() == 1) - << "While loop must have exactly one operand; had " - << instruction->operand_count() << ": " << instruction->ToString(); - - auto* init = instruction->operand(0); - auto* cond_param = while_cond->parameter_instruction(0); - TF_RET_CHECK(ShapeUtil::Compatible(init->shape(), cond_param->shape())) - << "While condition's parameter must have the same shape as the " - "loop's 'init'. init: " - << init->ToString() << ", param: " << cond_param->ToString(); - auto* cond_root = while_cond->root_instruction(); - TF_RET_CHECK(ShapeUtil::Compatible(cond_root->shape(), - ShapeUtil::MakeShape(PRED, {}))) - << "While condition should have shape PRED: " - << cond_root->ToString(); - - auto* body_param = while_body->parameter_instruction(0); - TF_RET_CHECK(ShapeUtil::Compatible(init->shape(), body_param->shape())) - << "While body's parameter must have the same shape as the loop's " - "'init'. init: " - << init->ToString() << ", param: " << body_param->ToString(); - auto* body_root = while_body->root_instruction(); - TF_RET_CHECK(ShapeUtil::Compatible(init->shape(), body_root->shape())) - << "While body should have same shape as the loop's 'init'. init: " - << init->ToString() << ", body: " << body_root->ToString(); + TF_RETURN_IF_ERROR(CheckWhileInstruction(instruction)); } auto previous = instructions.find(instruction->name()); diff --git a/tensorflow/compiler/xla/service/hlo_verifier.h b/tensorflow/compiler/xla/service/hlo_verifier.h index 1dd7ec3c51..1ec55a9bdc 100644 --- a/tensorflow/compiler/xla/service/hlo_verifier.h +++ b/tensorflow/compiler/xla/service/hlo_verifier.h @@ -102,7 +102,7 @@ class ShapeVerifier : public DfsHloVisitor { Status CheckTernaryShape(const HloInstruction* instruction); Status CheckVariadicShape(const HloInstruction* instruction); - // Checks if the given two instructions shares the same channel id. + // Checks if the given two instructions share the same channel id. Status CheckSameChannel(const HloInstruction* instr1, const HloInstruction* instr2); @@ -144,9 +144,11 @@ class HloVerifier : public HloPassInterface { // CHECKs various invariants of a fusion instruction. Status CheckFusionInstruction(HloInstruction* fusion) const; + Status CheckWhileInstruction(HloInstruction* instruction); + // Creates a ShapeVerifier that checks that shapes match inferred - // expectations. This is a factory function because ShapeVerifier, Note that - // ShapeVerifier, being a DfsHloVisitor, is stateful. We want a clean object + // expectations. This is a factory function because ShapeVerifier, + // being a DfsHloVisitor, is stateful. We want a clean object // for each run of the verifier. ShapeVerifierFactory shape_verifier_factory_; }; -- GitLab From 9e0037513040fd09ee01442bd062936b41bee40c Mon Sep 17 00:00:00 2001 From: SukHwan Kim <30820468+jerry4897@users.noreply.github.com> Date: Fri, 20 Apr 2018 18:24:52 +0900 Subject: [PATCH 730/791] Update c_api_test.cc Typo --- tensorflow/c/c_api_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/c/c_api_test.cc b/tensorflow/c/c_api_test.cc index ca80db23ed..9b86425aa5 100644 --- a/tensorflow/c/c_api_test.cc +++ b/tensorflow/c/c_api_test.cc @@ -1700,7 +1700,7 @@ TEST_F(CApiGradientsTest, OpWithNoGradientRegistered_NoGradInputs) { TestGradientsError(false); } -// REGISTER_OP for CApiTestAttributesTest test cases. +// REGISTER_OP for CApiAttributesTest test cases. // Registers two ops, each with a single attribute called 'v'. // The attribute in one op will have a type 'type', the other // will have list(type). -- GitLab From 1ad32703d4e728d8fba835aaf24418f19cf85dbe Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Fri, 20 Apr 2018 03:29:31 -0700 Subject: [PATCH 731/791] [TF:XLA] Implement ClipByValue. PiperOrigin-RevId: 193646890 --- tensorflow/compiler/tests/ternary_ops_test.py | 18 ++++++ tensorflow/compiler/tf2xla/kernels/BUILD | 1 + .../tf2xla/kernels/clip_by_value_op.cc | 61 +++++++++++++++++++ 3 files changed, 80 insertions(+) create mode 100644 tensorflow/compiler/tf2xla/kernels/clip_by_value_op.cc diff --git a/tensorflow/compiler/tests/ternary_ops_test.py b/tensorflow/compiler/tests/ternary_ops_test.py index ba5f829936..75a2cf07c5 100644 --- a/tensorflow/compiler/tests/ternary_ops_test.py +++ b/tensorflow/compiler/tests/ternary_ops_test.py @@ -23,6 +23,7 @@ import numpy as np from tensorflow.compiler.tests.xla_test import XLATestCase from tensorflow.python.framework import dtypes from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops import math_ops from tensorflow.python.platform import googletest @@ -119,6 +120,23 @@ class TernaryOpsTest(XLATestCase): np.array([2, 1], dtype=np.int32), expected=np.array([[2], [5]], dtype=dtype)) + def testClipByValue(self): + # TODO(b/78258593): enable integer types here too. + for dtype in self.float_types: + test_cases = [ + (np.array([2, 4, 5], dtype=dtype), dtype(7)), # + (dtype(1), np.array([2, 4, 5], dtype=dtype)), # + (np.array([-2, 7, 7], dtype=dtype), np.array([-2, 9, 8], dtype=dtype)) + ] + x = np.array([-2, 10, 6], dtype=dtype) + for lower, upper in test_cases: + self._testTernary( + gen_math_ops._clip_by_value, + x, + lower, + upper, + expected=np.minimum(np.maximum(x, lower), upper)) + if __name__ == "__main__": googletest.main() diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD index 579b669699..00fd08b1a0 100644 --- a/tensorflow/compiler/tf2xla/kernels/BUILD +++ b/tensorflow/compiler/tf2xla/kernels/BUILD @@ -21,6 +21,7 @@ tf_kernel_library( "cast_op.cc", "categorical_op.cc", "cholesky_op.cc", + "clip_by_value_op.cc", "concat_op.cc", "const_op.cc", "conv_ops.cc", diff --git a/tensorflow/compiler/tf2xla/kernels/clip_by_value_op.cc b/tensorflow/compiler/tf2xla/kernels/clip_by_value_op.cc new file mode 100644 index 0000000000..fdf75be7b1 --- /dev/null +++ b/tensorflow/compiler/tf2xla/kernels/clip_by_value_op.cc @@ -0,0 +1,61 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/tf2xla/xla_op_kernel.h" +#include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "tensorflow/core/framework/tensor_shape.h" + +namespace tensorflow { +namespace { + +class ClipByValueOp : public XlaOpKernel { + public: + explicit ClipByValueOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {} + + void Compile(XlaOpKernelContext* ctx) override { + const TensorShape shape = ctx->InputShape(0); + const TensorShape min_shape = ctx->InputShape(1); + const TensorShape max_shape = ctx->InputShape(2); + + xla::ComputationBuilder* builder = ctx->builder(); + auto input = ctx->Input(0); + auto min = ctx->Input(1); + auto max = ctx->Input(2); + + auto shape_error = [&]() -> tensorflow::Status { + return errors::InvalidArgument( + "clip_value_min and clip_value_max must be either of " + "the same shape as input, or a scalar. ", + "Input shape: ", shape.DebugString(), + " clip_value_min shape: ", min_shape.DebugString(), + " clip_value_max shape: ", max_shape.DebugString()); + }; + + if (shape != min_shape) { + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(min_shape), shape_error()); + min = builder->Broadcast(min, shape.dim_sizes()); + } + if (shape != max_shape) { + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(max_shape), shape_error()); + max = builder->Broadcast(max, shape.dim_sizes()); + } + ctx->SetOutput(0, builder->Clamp(min, input, max)); + } +}; + +REGISTER_XLA_OP(Name("ClipByValue"), ClipByValueOp); + +} // namespace +} // namespace tensorflow -- GitLab From 0c03255aa5f4b37de97e0685ffa15888fc16e4b3 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Apr 2018 06:36:56 -0700 Subject: [PATCH 732/791] internal change PiperOrigin-RevId: 193659701 --- .../lite/toco/graph_transformations/propagate_fixed_sizes.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc index b34aca1f09..ba244cf5ef 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc +++ b/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc @@ -1516,10 +1516,7 @@ void ProcessArgMaxOperator(Model* model, ArgMaxOperator* op) { return; } - // The current ArgMax implementation only supports 4-dimensional inputs with - // the last dimension as the axis to perform ArgMax for. const std::vector& input_dims = input_array.shape().dims(); - CHECK_EQ(input_dims.size(), 4); std::vector output_dims; output_dims.reserve(input_dims.size() - 1); -- GitLab From c212d5542bb666b613a8567338983288a3ab15f4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Apr 2018 08:08:01 -0700 Subject: [PATCH 733/791] Eliminate the guard around Winograd non-fused convolutions with cudnn7. PiperOrigin-RevId: 193669636 --- .../fused_conv2d_bias_activation_op.cc | 3 +- .../core/kernels/conv_grad_filter_ops.cc | 3 +- .../core/kernels/conv_grad_input_ops.cc | 3 +- tensorflow/core/kernels/conv_grad_ops_3d.cc | 8 +++-- tensorflow/core/kernels/conv_ops.cc | 3 +- tensorflow/core/kernels/conv_ops_3d.cc | 4 ++- tensorflow/core/kernels/conv_ops_gpu.h | 35 +++++++++++++------ tensorflow/core/kernels/conv_ops_test.cc | 26 +++++++++----- 8 files changed, 59 insertions(+), 26 deletions(-) diff --git a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc index 0e06575d96..1e8f011b5d 100644 --- a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc +++ b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc @@ -543,7 +543,8 @@ void LaunchFusedConv2DBiasActivationOp:: fused_conv_parameters, &algorithm_config)) { std::vector algorithms; CHECK(stream->parent()->GetConvolveAlgorithms( - fused_conv_parameters.ShouldIncludeWinogradNonfusedAlgo(), + fused_conv_parameters.ShouldIncludeWinogradNonfusedAlgo( + stream->parent()), &algorithms)); dnn::ProfileResult best_result; dnn::ProfileResult best_result_no_scratch; diff --git a/tensorflow/core/kernels/conv_grad_filter_ops.cc b/tensorflow/core/kernels/conv_grad_filter_ops.cc index 66ee474ca3..f3b91494b9 100644 --- a/tensorflow/core/kernels/conv_grad_filter_ops.cc +++ b/tensorflow/core/kernels/conv_grad_filter_ops.cc @@ -912,7 +912,8 @@ void LaunchConv2DBackpropFilterOp::operator()( conv_parameters, &algorithm_config)) { std::vector algorithms; CHECK(stream->parent()->GetConvolveBackwardFilterAlgorithms( - conv_parameters.ShouldIncludeWinogradNonfusedAlgo(), &algorithms)); + conv_parameters.ShouldIncludeWinogradNonfusedAlgo(stream->parent()), + &algorithms)); ProfileResult best_result; ProfileResult best_result_no_scratch; for (auto profile_algorithm : algorithms) { diff --git a/tensorflow/core/kernels/conv_grad_input_ops.cc b/tensorflow/core/kernels/conv_grad_input_ops.cc index 71ea0d5d72..66d15c6e78 100644 --- a/tensorflow/core/kernels/conv_grad_input_ops.cc +++ b/tensorflow/core/kernels/conv_grad_input_ops.cc @@ -961,7 +961,8 @@ void LaunchConv2DBackpropInputOp::operator()( conv_parameters, &algorithm_config)) { std::vector algorithms; CHECK(stream->parent()->GetConvolveBackwardDataAlgorithms( - conv_parameters.ShouldIncludeWinogradNonfusedAlgo(), &algorithms)); + conv_parameters.ShouldIncludeWinogradNonfusedAlgo(stream->parent()), + &algorithms)); ProfileResult best_result; ProfileResult best_result_no_scratch; for (auto profile_algorithm : algorithms) { diff --git a/tensorflow/core/kernels/conv_grad_ops_3d.cc b/tensorflow/core/kernels/conv_grad_ops_3d.cc index 3650ab53b2..1234997bc5 100644 --- a/tensorflow/core/kernels/conv_grad_ops_3d.cc +++ b/tensorflow/core/kernels/conv_grad_ops_3d.cc @@ -662,7 +662,9 @@ class Conv3DBackpropInputOp : public OpKernel { conv_parameters, &algorithm_config)) { std::vector algorithms; CHECK(stream->parent()->GetConvolveBackwardDataAlgorithms( - conv_parameters.ShouldIncludeWinogradNonfusedAlgo(), &algorithms)); + conv_parameters.ShouldIncludeWinogradNonfusedAlgo( + stream->parent()), + &algorithms)); ProfileResult best_result; ProfileResult best_result_no_scratch; for (auto profile_algorithm : algorithms) { @@ -1029,7 +1031,9 @@ class Conv3DBackpropFilterOp : public OpKernel { conv_parameters, &algorithm_config)) { std::vector algorithms; CHECK(stream->parent()->GetConvolveBackwardFilterAlgorithms( - conv_parameters.ShouldIncludeWinogradNonfusedAlgo(), &algorithms)); + conv_parameters.ShouldIncludeWinogradNonfusedAlgo( + stream->parent()), + &algorithms)); ProfileResult best_result; ProfileResult best_result_no_scratch; for (auto profile_algorithm : algorithms) { diff --git a/tensorflow/core/kernels/conv_ops.cc b/tensorflow/core/kernels/conv_ops.cc index 88843e4da7..f0888c655f 100644 --- a/tensorflow/core/kernels/conv_ops.cc +++ b/tensorflow/core/kernels/conv_ops.cc @@ -710,7 +710,8 @@ void LaunchConv2DOp::operator()( !AutoTuneConv::GetInstance()->Find(conv_parameters, &algorithm_config)) { std::vector algorithms; CHECK(stream->parent()->GetConvolveAlgorithms( - conv_parameters.ShouldIncludeWinogradNonfusedAlgo(), &algorithms)); + conv_parameters.ShouldIncludeWinogradNonfusedAlgo(stream->parent()), + &algorithms)); ProfileResult best_result; ProfileResult best_result_no_scratch; for (auto profile_algorithm : algorithms) { diff --git a/tensorflow/core/kernels/conv_ops_3d.cc b/tensorflow/core/kernels/conv_ops_3d.cc index 21c84b2a0e..0b7c1524e6 100644 --- a/tensorflow/core/kernels/conv_ops_3d.cc +++ b/tensorflow/core/kernels/conv_ops_3d.cc @@ -396,7 +396,9 @@ struct LaunchConvOp { conv_parameters, &algorithm_config)) { std::vector algorithms; CHECK(stream->parent()->GetConvolveAlgorithms( - conv_parameters.ShouldIncludeWinogradNonfusedAlgo(), &algorithms)); + conv_parameters.ShouldIncludeWinogradNonfusedAlgo( + stream->parent()), + &algorithms)); ProfileResult best_result; ProfileResult best_result_no_scratch; for (auto profile_algorithm : algorithms) { diff --git a/tensorflow/core/kernels/conv_ops_gpu.h b/tensorflow/core/kernels/conv_ops_gpu.h index f0085be3a5..7f9cfec981 100644 --- a/tensorflow/core/kernels/conv_ops_gpu.h +++ b/tensorflow/core/kernels/conv_ops_gpu.h @@ -137,20 +137,18 @@ class ConvParameters { // clang-format on } - // TODO(yangzihao): The purpose of this function is to disable winograd - // nonfused conv algorithm for certain input parameters so as to avoid a bug - // in cuDNNv5 and cuDNNv6. Remove this once switch to cuDNNv7. + // The purpose of this function is to disable winograd nonfused conv algorithm + // for certain input parameters so as to avoid a bug in cuDNNv5 and cuDNNv6. template - bool ShouldIncludeWinogradNonfusedAlgo() const { - int64 total_size = 16 * std::ceil(batch_ / 16.0) * - std::max(in_depths_, out_depths_) * in_[0] * in_[1] * - sizeof(T); - int64 threshold = 1LL << 31; - if (total_size >= threshold) { - return false; - } else { + bool ShouldIncludeWinogradNonfusedAlgo( + perftools::gputools::StreamExecutor* stream_exec) const { + // Skip this check for cuDNN 7 and newer. + perftools::gputools::port::StatusOr> version = + stream_exec->AsDnn()->GetVersion(); + if (version.ok() && std::get<0>(version.ValueOrDie()) >= 7) { return true; } + return ShouldIncludeWinogradNonfusedAlgoPreCudnn7(); } protected: @@ -166,6 +164,21 @@ class ConvParameters { uint64 hash_code_; private: + friend struct ConvParametersPeer; // For testing purposes. + + template + bool ShouldIncludeWinogradNonfusedAlgoPreCudnn7() const { + int64 total_size = 16 * std::ceil(batch_ / 16.0) * + std::max(in_depths_, out_depths_) * in_[0] * in_[1] * + sizeof(T); + int64 threshold = 1LL << 31; + if (total_size >= threshold) { + return false; + } else { + return true; + } + } + int64 batch_; int64 in_depths_; int64 out_depths_; diff --git a/tensorflow/core/kernels/conv_ops_test.cc b/tensorflow/core/kernels/conv_ops_test.cc index e2e166c02f..8afe6a2cbd 100644 --- a/tensorflow/core/kernels/conv_ops_test.cc +++ b/tensorflow/core/kernels/conv_ops_test.cc @@ -22,20 +22,28 @@ limitations under the License. #include "tensorflow/core/framework/node_def_builder.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/kernels/conv_ops_gpu.h" #include "tensorflow/core/kernels/ops_testutil.h" #include "tensorflow/core/kernels/ops_util.h" #include "tensorflow/core/platform/test.h" #include "tensorflow/core/platform/test_benchmark.h" #include "tensorflow/core/public/session.h" -#include "tensorflow/core/kernels/conv_ops_gpu.h" - namespace tensorflow { #if GOOGLE_CUDA +struct ConvParametersPeer { + template + bool ShouldIncludeWinogradNonfusedAlgoPreCudnn7() { + return params.ShouldIncludeWinogradNonfusedAlgoPreCudnn7(); + } + + ConvParameters params; +}; + TEST(ConvParameters, WinogradNonfusedAlgoSize) { - ConvParameters conv_params_small = { + ConvParametersPeer conv_params_small = {{ 1, // batch 32, // in_depths {{300, // in_rows @@ -51,10 +59,11 @@ TEST(ConvParameters, WinogradNonfusedAlgoSize) { 0}}, // padding_cols DT_FLOAT, // tensor datatype 0, // device_id - }; - EXPECT_TRUE(conv_params_small.ShouldIncludeWinogradNonfusedAlgo()); + }}; + EXPECT_TRUE( + conv_params_small.ShouldIncludeWinogradNonfusedAlgoPreCudnn7()); - ConvParameters conv_params_large = { + ConvParametersPeer conv_params_large = {{ 1, // batch 128, // in_depths {{300, // in_rows @@ -70,8 +79,9 @@ TEST(ConvParameters, WinogradNonfusedAlgoSize) { 0}}, // padding_cols DT_FLOAT, // tensor datatype 0, // device_id - }; - EXPECT_FALSE(conv_params_large.ShouldIncludeWinogradNonfusedAlgo()); + }}; + EXPECT_FALSE( + conv_params_large.ShouldIncludeWinogradNonfusedAlgoPreCudnn7()); } #endif // GOOGLE_CUDA -- GitLab From 814ab7e37dcbfa7f4749a1fd9d687d6be0207cb8 Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Fri, 20 Apr 2018 09:20:36 -0700 Subject: [PATCH 734/791] [TF:XLA] Bump open source llvm revision to r330313 PiperOrigin-RevId: 193678317 --- tensorflow/workspace.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index d7bd2a2be0..aeaf8d7a24 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -451,11 +451,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""): tf_http_archive( name = "llvm", urls = [ - "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/3210e64b499a31193051208f2f8922dadfc4bb6f.tar.gz", - "https://github.com/llvm-mirror/llvm/archive/3210e64b499a31193051208f2f8922dadfc4bb6f.tar.gz", + "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/c1e9b6f826c86c87a7e7173f1baf7e7df9f43e32.tar.gz", + "https://github.com/llvm-mirror/llvm/archive/c1e9b6f826c86c87a7e7173f1baf7e7df9f43e32.tar.gz", ], - sha256 = "017d7db029cc175634d75416c326770139c76590575ed44a3794c11ab160c955", - strip_prefix = "llvm-3210e64b499a31193051208f2f8922dadfc4bb6f", + sha256 = "92b7c01074f694a77b4d664951d1ec071e30ef19c61e673158e95fbb6e447b54", + strip_prefix = "llvm-c1e9b6f826c86c87a7e7173f1baf7e7df9f43e32", build_file = clean_dep("//third_party/llvm:llvm.BUILD"), ) -- GitLab From d0e3e998376f5e7d59678e5d42f3497e52ca7622 Mon Sep 17 00:00:00 2001 From: Saurabh Saxena Date: Fri, 20 Apr 2018 09:23:52 -0700 Subject: [PATCH 735/791] Fix msan error in MapAndBatchDataset. While checkpointing tensors in BatchResult.output save only the initialized slice. If the final batch is short, the entire batch tensor may not be initialized. PiperOrigin-RevId: 193678679 --- .../kernels/data/map_and_batch_dataset_op.cc | 44 +++++++++++++++---- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc index b8105552a0..605ef3c0b7 100644 --- a/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc +++ b/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc @@ -331,7 +331,7 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel { } CHECK_EQ(batch_results_.size(), batch_results_size); for (size_t i = 0; i < batch_results_size; ++i) { - TF_RETURN_IF_ERROR(ReadBatchResultLocked(reader, i)); + TF_RETURN_IF_ERROR(ReadBatchResultLocked(ctx, reader, i)); } return Status::OK(); } @@ -573,7 +573,9 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel { // finish. This may delay saving a checkpoint by a bit but keeps the // code clean and also saves us from checkpointing the state of the // `BlockingCounter`. - batch_results_[index].counter->Wait(); + int64 num_elements = 0; + WaitForBatch(index, &num_elements).IgnoreError(); + const BatchResult& result = batch_results_[index]; string prefix = strings::StrCat("batch_results_", index); { @@ -587,14 +589,24 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel { full_name(strings::StrCat(prefix, "_output_size")), result.output.size())); for (size_t i = 0; i < result.output.size(); i++) { - TF_RETURN_IF_ERROR(writer->WriteTensor( - full_name(strings::StrCat(prefix, "_output_", i)), - result.output[i])); + // If the batch is not full, we only store the first + // `num_elements` values. The rest of the batch tensor is + // *uninitialized* and accessing that will raise msan errors. + if (num_elements < dataset()->batch_size_) { + TF_RETURN_IF_ERROR(writer->WriteTensor( + full_name(strings::StrCat(prefix, "_output_", i)), + result.output[i].Slice(0, num_elements))); + } else { + TF_RETURN_IF_ERROR(writer->WriteTensor( + full_name(strings::StrCat(prefix, "_output_", i)), + result.output[i])); + } } return Status::OK(); } - Status ReadBatchResultLocked(IteratorStateReader* reader, size_t index) + Status ReadBatchResultLocked(IteratorContext* ctx, + IteratorStateReader* reader, size_t index) EXCLUSIVE_LOCKS_REQUIRED(mu_) { BatchResult* result = &batch_results_[index]; string prefix = strings::StrCat("batch_results_", index); @@ -618,10 +630,24 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel { } result->output.reserve(output_size); for (size_t i = 0; i < output_size; i++) { - result->output.emplace_back(); + Tensor t; TF_RETURN_IF_ERROR(reader->ReadTensor( - full_name(strings::StrCat(prefix, "_output_", i)), - &result->output.back())); + full_name(strings::StrCat(prefix, "_output_", i)), &t)); + // If the batch was not full, we may have stored only the relevant + // slice. Since tensors in `BatchResult.output` are expected to + // have the leading dimension of size batch_size, we build a larger + // tensor and copy the slice read from the checkpoint into it. + if (t.dim_size(0) < dataset()->batch_size_) { + TensorShape component_shape(t.shape()); + component_shape.set_dim(0, dataset()->batch_size_); + AllocatorAttributes attr; + attr.set_gpu_compatible(true); + Tensor new_t(ctx->allocator(attr), t.dtype(), component_shape); + TF_RETURN_IF_ERROR(CopyPartialBatch(&new_t, t, t.dim_size(0))); + result->output.emplace_back(std::move(new_t)); + } else { + result->output.emplace_back(std::move(t)); + } } return Status::OK(); } -- GitLab From cd462f39e58674a43d1f8c156f23235722b2281e Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Fri, 20 Apr 2018 09:31:08 -0700 Subject: [PATCH 736/791] Don't delete inbound_nodes and outbound_nodes, these no longer exist. PiperOrigin-RevId: 193679512 --- tensorflow/tools/docs/generate.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tensorflow/tools/docs/generate.py b/tensorflow/tools/docs/generate.py index c750539a76..fc93085e3e 100644 --- a/tensorflow/tools/docs/generate.py +++ b/tensorflow/tools/docs/generate.py @@ -43,10 +43,6 @@ if __name__ == '__main__': flags = doc_generator.parse_known_args() - # Suppress documentation of some symbols that users should never use. - del tf.layers.Layer.inbound_nodes - del tf.layers.Layer.outbound_nodes - # tf_debug is not imported with tf, it's a separate module altogether doc_generator.set_py_modules([('tf', tf), ('tfdbg', tf_debug)]) -- GitLab From fb23c0e166179ccf372203982d8fe79de441e360 Mon Sep 17 00:00:00 2001 From: James Keeling Date: Fri, 20 Apr 2018 09:54:50 -0700 Subject: [PATCH 737/791] Correct error in "Adding An Op" docs. The macro `REGISTER_KERNEL_BUILDER` always declared a functor specialized on floats, instead of the type actually passed into the macro. PiperOrigin-RevId: 193682519 --- tensorflow/docs_src/extend/adding_an_op.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/docs_src/extend/adding_an_op.md b/tensorflow/docs_src/extend/adding_an_op.md index 84da2165b5..c3795492ce 100644 --- a/tensorflow/docs_src/extend/adding_an_op.md +++ b/tensorflow/docs_src/extend/adding_an_op.md @@ -267,7 +267,7 @@ REGISTER_CPU(int32); #ifdef GOOGLE_CUDA #define REGISTER_GPU(T) \ /* Declare explicit instantiations in kernel_example.cu.cc. */ \ - extern template ExampleFunctor; \ + extern template ExampleFunctor; \ REGISTER_KERNEL_BUILDER( \ Name("Example").Device(DEVICE_GPU).TypeConstraint("T"), \ ExampleOp); -- GitLab From a749a6b95932d6f7438a01a2f5fd661343ad536f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Apr 2018 10:16:03 -0700 Subject: [PATCH 738/791] Change the TF record reader to use 16MB buffering by default in order to improve performance. PiperOrigin-RevId: 193685521 --- tensorflow/python/lib/io/py_record_reader.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/lib/io/py_record_reader.cc b/tensorflow/python/lib/io/py_record_reader.cc index 5fcb51b3b2..9500fc6a7c 100644 --- a/tensorflow/python/lib/io/py_record_reader.cc +++ b/tensorflow/python/lib/io/py_record_reader.cc @@ -43,9 +43,10 @@ PyRecordReader* PyRecordReader::New(const string& filename, uint64 start_offset, reader->offset_ = start_offset; reader->file_ = file.release(); + static const uint64 kReaderBufferSize = 16 * 1024 * 1024; RecordReaderOptions options = RecordReaderOptions::CreateRecordReaderOptions(compression_type_string); - + options.buffer_size = kReaderBufferSize; reader->reader_ = new RecordReader(reader->file_, options); return reader; } -- GitLab From 729192823935156ae29d7f0d5f64c0bcd6034c7a Mon Sep 17 00:00:00 2001 From: Jacques Pienaar Date: Fri, 20 Apr 2018 10:32:24 -0700 Subject: [PATCH 739/791] Adding Shape inference functions to outfeed enqueue ops. PiperOrigin-RevId: 193688099 --- tensorflow/contrib/tpu/ops/outfeed_ops.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/contrib/tpu/ops/outfeed_ops.cc b/tensorflow/contrib/tpu/ops/outfeed_ops.cc index 5900c61a38..b05c76ca64 100644 --- a/tensorflow/contrib/tpu/ops/outfeed_ops.cc +++ b/tensorflow/contrib/tpu/ops/outfeed_ops.cc @@ -26,6 +26,7 @@ REGISTER_OP("OutfeedEnqueue") .Input("input: dtype") .Attr("dtype: type") .SetIsStateful() + .SetShapeFn(shape_inference::NoOutputs) .Doc(R"doc( An op which emits a single Tensor value from an XLA computation. @@ -36,6 +37,7 @@ REGISTER_OP("OutfeedEnqueueTuple") .Input("inputs: dtypes") .Attr("dtypes: list(type)") .SetIsStateful() + .SetShapeFn(shape_inference::NoOutputs) .Doc(R"doc( An op which emits multiple Tensor values from an XLA computation. -- GitLab From da5a6d86b856001c03cccace5ac74fa8f045b6ae Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Apr 2018 10:34:49 -0700 Subject: [PATCH 740/791] Disable constant folding and arithmetic optimizations for functions. PiperOrigin-RevId: 193688466 --- tensorflow/core/grappler/optimizers/meta_optimizer.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc index 22799311bc..cdc4698c34 100644 --- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc @@ -243,6 +243,10 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item, std::unordered_set optimized_funcs; bool optimize_function_library = true; + // TODO(ezhulenev): turn it on after fixing ranklab: tune_tf_test. + cfg_.set_constant_folding(RewriterConfig::OFF); + cfg_.set_arithmetic_optimization(RewriterConfig::OFF); + while (optimize_function_library) { optimize_function_library = false; -- GitLab From b3f379e907259aa166c1ef734ccfd03331eb0a94 Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Fri, 20 Apr 2018 11:10:56 -0700 Subject: [PATCH 741/791] [XLA:CPU] Use Eigen for F64 dot operations PiperOrigin-RevId: 193694613 --- tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc | 3 ++- tensorflow/compiler/xla/service/cpu/ir_emitter.cc | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc index 29afd8ea5f..495fecc4aa 100644 --- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc +++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc @@ -1070,7 +1070,8 @@ static bool AreValidGemmShapes(const Shape& lhs_shape, const Shape& rhs_shape, // 1) be matrices with no padding, and // 2) have an allowed element type. PrimitiveType output_primitive_type = output_shape.element_type(); - return (output_primitive_type == F32 || output_primitive_type == F16) && + return (output_primitive_type == F64 || output_primitive_type == F32 || + output_primitive_type == F16) && IsRank2WithNoPadding(lhs_shape) && IsRank2WithNoPadding(rhs_shape) && IsRank2WithNoPadding(output_shape); } diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc index 3405277d44..f990ee2785 100644 --- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc +++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc @@ -2076,7 +2076,7 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) { TF_RETURN_IF_ERROR(ElementTypesSameAndSupported( /*instruction=*/*root, /*operands=*/{lhs, rhs}, - /*supported_types=*/{F16, F32})); + /*supported_types=*/{F16, F32, F64})); llvm_ir::IrArray lhs_array(GetIrArrayFor(lhs)); llvm_ir::IrArray rhs_array(GetIrArrayFor(rhs)); -- GitLab From 49f3469d9533cb12d06ed3907b4ced975e2fcea4 Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Fri, 20 Apr 2018 11:13:16 -0700 Subject: [PATCH 742/791] Use CreateWorkerSession and DeleteWorkerSession for all distributed sessions. This change adds a phase to the session creation protocol: the master now contacts all workers to register a session handle and create a "WorkerSession" on each worker before it first registers or runs a graph on any worker. Subsequent requests to a worker ensure that the worker has the session handle registered before performing the request, and an AbortedError is raised if the worker has not (e.g. because it restarted after a failure). As a result, more failure cases are covered by the high-level APIs (tf.estimator, Slim, etc.) that recreate the session on receiving an AbortedError. Previously, there was a possible race condition in which a PS task could restart between variable initialization and the first step, leading to a FailedPreconditionError ("Attempting to use uninitialized value") that would not be handled by the high-level APIs. PiperOrigin-RevId: 193694958 --- .../core/distributed_runtime/master_session.cc | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tensorflow/core/distributed_runtime/master_session.cc b/tensorflow/core/distributed_runtime/master_session.cc index ebe350d313..1c67b42e76 100644 --- a/tensorflow/core/distributed_runtime/master_session.cc +++ b/tensorflow/core/distributed_runtime/master_session.cc @@ -89,6 +89,10 @@ class MasterSession::ReffedClientGraph : public core::RefCounted { ~ReffedClientGraph() override { if (should_deregister_) { DeregisterPartitions(); + } else { + for (Part& part : partitions_) { + worker_cache_->ReleaseWorker(part.name, part.worker); + } } } @@ -1174,14 +1178,8 @@ Status MasterSession::Create(GraphDef* graph_def, TF_RETURN_IF_ERROR(GraphExecutionState::MakeForBaseGraph( graph_def, execution_options, &execution_state_)); } - // TODO(b/36574172): Remove these conditions when ClusterSpec - // propagation is supported in all servers. - if (options.cluster_def != nullptr || - session_opts_.config.isolate_session_state()) { - should_delete_worker_sessions_ = true; - return CreateWorkerSessions(options); - } - return Status::OK(); + should_delete_worker_sessions_ = true; + return CreateWorkerSessions(options); } Status MasterSession::CreateWorkerSessions( -- GitLab From 570d90b9c7e6a19bc2606fdaf7ad0f85b8590c0e Mon Sep 17 00:00:00 2001 From: akindyakov Date: Fri, 20 Apr 2018 11:23:15 -0700 Subject: [PATCH 743/791] Speed up safe_strtod and safe_strtof functions by using double-conversion library Closes #12102. PiperOrigin-RevId: 193696537 --- tensorflow/contrib/cmake/CMakeLists.txt | 4 + .../cmake/external/double_conversion.cmake | 54 ++++++++++++ tensorflow/contrib/makefile/Makefile | 8 +- .../contrib/makefile/download_dependencies.sh | 4 +- tensorflow/core/BUILD | 9 +- tensorflow/core/lib/strings/numbers.cc | 51 +++++++---- tensorflow/core/lib/strings/numbers.h | 2 + tensorflow/core/lib/strings/numbers_test.cc | 87 +++++++++++++++++++ tensorflow/core/lib/strings/str_util.cc | 8 ++ tensorflow/core/lib/strings/str_util.h | 5 ++ tensorflow/core/lib/strings/str_util_test.cc | 56 ++---------- tensorflow/tools/lib_package/BUILD | 2 + tensorflow/tools/pip_package/BUILD | 1 + tensorflow/workspace.bzl | 10 +++ third_party/double_conversion.BUILD | 38 ++++++++ 15 files changed, 270 insertions(+), 69 deletions(-) create mode 100644 tensorflow/contrib/cmake/external/double_conversion.cmake create mode 100644 third_party/double_conversion.BUILD diff --git a/tensorflow/contrib/cmake/CMakeLists.txt b/tensorflow/contrib/cmake/CMakeLists.txt index 23b31ae1dc..bdf3e98635 100644 --- a/tensorflow/contrib/cmake/CMakeLists.txt +++ b/tensorflow/contrib/cmake/CMakeLists.txt @@ -193,6 +193,7 @@ include(protobuf) include(re2) include(cub) include(sqlite) +include(double_conversion) if (tensorflow_BUILD_CC_TESTS) include(googletest) endif() @@ -213,6 +214,7 @@ set(tensorflow_EXTERNAL_LIBRARIES ${protobuf_STATIC_LIBRARIES} ${re2_STATIC_LIBRARIES} ${sqlite_STATIC_LIBRARIES} + ${double_conversion_STATIC_LIBRARIES} ) if (systemlib_ZLIB) @@ -240,6 +242,7 @@ set(tensorflow_EXTERNAL_DEPENDENCIES fft2d re2 sqlite_copy_headers_to_destination + double_conversion ) include_directories( @@ -262,6 +265,7 @@ include_directories( ${PROTOBUF_INCLUDE_DIRS} ${re2_INCLUDE_DIR} ${sqlite_INCLUDE_DIR} + ${double_conversion_INCLUDE_DIR} ) if(tensorflow_ENABLE_SSL_SUPPORT) diff --git a/tensorflow/contrib/cmake/external/double_conversion.cmake b/tensorflow/contrib/cmake/external/double_conversion.cmake new file mode 100644 index 0000000000..527ccdc8d8 --- /dev/null +++ b/tensorflow/contrib/cmake/external/double_conversion.cmake @@ -0,0 +1,54 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +include (ExternalProject) + +set(double_conversion_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/double_conversion/src/double_conversion) +set(double_conversion_URL https://github.com/google/double-conversion.git) +set(double_conversion_TAG 5664746) +set(double_conversion_BUILD ${double_conversion_INCLUDE_DIR}) +set(double_conversion_LIBRARIES ${double_conversion_BUILD}/double-conversion/libdouble-conversion.so) +set(double_conversion_INCLUDES ${double_conversion_BUILD}) + +if(WIN32) + set(double_conversion_STATIC_LIBRARIES ${double_conversion_BUILD}/double-conversion/$(Configuration)/double-conversion.lib) +else() + set(double_conversion_STATIC_LIBRARIES ${double_conversion_BUILD}/double-conversion/libdouble-conversion.a) +endif() + +set(double_conversion_HEADERS + "${double_conversion_INCLUDE_DIR}/double-conversion/bignum-dtoa.h" + "${double_conversion_INCLUDE_DIR}/double-conversion/cached-powers.h" + "${double_conversion_INCLUDE_DIR}/double-conversion/double-conversion.h" + "${double_conversion_INCLUDE_DIR}/double-conversion/fixed-dtoa.h" + "${double_conversion_INCLUDE_DIR}/double-conversion/strtod.h" + "${double_conversion_INCLUDE_DIR}/double-conversion/bignum.h" + "${double_conversion_INCLUDE_DIR}/double-conversion/diy-fp.h" + "${double_conversion_INCLUDE_DIR}/double-conversion/fast-dtoa.h" + "${double_conversion_INCLUDE_DIR}/double-conversion/ieee.h" + "${double_conversion_INCLUDE_DIR}/double-conversion/utils.h" +) + +ExternalProject_Add(double_conversion + PREFIX double_conversion + GIT_REPOSITORY ${double_conversion_URL} + GIT_TAG ${double_conversion_TAG} + DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" + BUILD_IN_SOURCE 1 + INSTALL_COMMAND "" + CMAKE_CACHE_ARGS + -DCMAKE_BUILD_TYPE:STRING=Release + -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF + -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON +) diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile index 05e8d9064b..1a1ab54a53 100644 --- a/tensorflow/contrib/makefile/Makefile +++ b/tensorflow/contrib/makefile/Makefile @@ -89,6 +89,7 @@ HOST_INCLUDES := \ -I$(MAKEFILE_DIR)/downloads/gemmlowp \ -I$(MAKEFILE_DIR)/downloads/nsync/public \ -I$(MAKEFILE_DIR)/downloads/fft2d \ +-I$(MAKEFILE_DIR)/downloads/double_conversion \ -I$(HOST_GENDIR) ifeq ($(HAS_GEN_HOST_PROTOC),true) HOST_INCLUDES += -I$(MAKEFILE_DIR)/gen/protobuf-host/include @@ -125,7 +126,9 @@ PROTO_TEXT := $(HOST_BINDIR)proto_text # The list of dependencies is derived from the Bazel build file by running # the gen_file_lists.sh script on a system with a working Bazel setup. PROTO_TEXT_CC_FILES := $(shell cat $(MAKEFILE_DIR)/proto_text_cc_files.txt) -PROTO_TEXT_PB_CC_LIST := $(shell cat $(MAKEFILE_DIR)/proto_text_pb_cc_files.txt) +PROTO_TEXT_PB_CC_LIST := \ + $(shell cat $(MAKEFILE_DIR)/proto_text_pb_cc_files.txt) \ + $(wildcard tensorflow/contrib/makefile/downloads/double_conversion/double-conversion/*.cc) PROTO_TEXT_PB_H_LIST := $(shell cat $(MAKEFILE_DIR)/proto_text_pb_h_files.txt) # Locations of the intermediate files proto_text generates. @@ -171,6 +174,7 @@ INCLUDES := \ -I$(MAKEFILE_DIR)/downloads/gemmlowp \ -I$(MAKEFILE_DIR)/downloads/nsync/public \ -I$(MAKEFILE_DIR)/downloads/fft2d \ +-I$(MAKEFILE_DIR)/downloads/double_conversion \ -I$(PROTOGENDIR) \ -I$(PBTGENDIR) ifeq ($(HAS_GEN_HOST_PROTOC),true) @@ -326,6 +330,7 @@ $(MARCH_OPTION) \ -I$(MAKEFILE_DIR)/downloads/gemmlowp \ -I$(MAKEFILE_DIR)/downloads/nsync/public \ -I$(MAKEFILE_DIR)/downloads/fft2d \ +-I$(MAKEFILE_DIR)/downloads/double_conversion \ -I$(MAKEFILE_DIR)/gen/protobuf_android/$(ANDROID_ARCH)/include \ -I$(PROTOGENDIR) \ -I$(PBTGENDIR) @@ -603,6 +608,7 @@ $(wildcard tensorflow/core/platform/*/*.cc) \ $(wildcard tensorflow/core/platform/*/*/*.cc) \ $(wildcard tensorflow/core/util/*.cc) \ $(wildcard tensorflow/core/util/*/*.cc) \ +$(wildcard tensorflow/contrib/makefile/downloads/double_conversion/double-conversion/*.cc) \ tensorflow/core/util/version_info.cc # Remove duplicates (for version_info.cc) CORE_CC_ALL_SRCS := $(sort $(CORE_CC_ALL_SRCS)) diff --git a/tensorflow/contrib/makefile/download_dependencies.sh b/tensorflow/contrib/makefile/download_dependencies.sh index 8b415e6527..48953e2e38 100755 --- a/tensorflow/contrib/makefile/download_dependencies.sh +++ b/tensorflow/contrib/makefile/download_dependencies.sh @@ -32,7 +32,8 @@ GOOGLETEST_URL="https://github.com/google/googletest/archive/release-1.8.0.tar.g NSYNC_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/nsync/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)" PROTOBUF_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/protobuf/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)" RE2_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/re2/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)" -FFT2D_URL="$(grep -o 'http.*fft\.tgz' "${BZL_FILE_PATH}" | grep -v mirror.bazel | head -n1)" +FFT2D_URL="$(grep -o 'http.*fft\.tgz' "${BZL_FILE_PATH}" | grep -v bazel-mirror | head -n1)" +DOUBLE_CONVERSION_URL="$(grep -o "https.*google/double-conversion.*\.zip" "${BZL_FILE_PATH}" | head -n1)" ABSL_URL="$(grep -o 'https://github.com/abseil/abseil-cpp/.*tar.gz' "${BZL_FILE_PATH}" | head -n1)" CUB_URL="$(grep -o 'https.*cub/archive.*zip' "${BZL_FILE_PATH}" | grep -v mirror.bazel | head -n1)" @@ -87,6 +88,7 @@ download_and_extract "${NSYNC_URL}" "${DOWNLOADS_DIR}/nsync" download_and_extract "${PROTOBUF_URL}" "${DOWNLOADS_DIR}/protobuf" download_and_extract "${RE2_URL}" "${DOWNLOADS_DIR}/re2" download_and_extract "${FFT2D_URL}" "${DOWNLOADS_DIR}/fft2d" +download_and_extract "${DOUBLE_CONVERSION_URL}" "${DOWNLOADS_DIR}/double_conversion" download_and_extract "${ABSL_URL}" "${DOWNLOADS_DIR}/absl" download_and_extract "${CUB_URL}" "${DOWNLOADS_DIR}/cub/external/cub_archive" diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index c15e7de186..5b04574a4f 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -337,7 +337,9 @@ cc_library( "lib/bfloat16/bfloat16.h", ] + tf_additional_proto_hdrs() + glob(tf_env_time_hdrs()), copts = tf_copts(), - deps = tf_lib_proto_parsing_deps(), + deps = tf_lib_proto_parsing_deps() + [ + "@double_conversion//:double-conversion", + ], ) # This build rule (along with :lib_internal, :framework, and @@ -1231,6 +1233,7 @@ cc_library( deps = [ ":protos_all_cc_impl", "//third_party/eigen3", + "@double_conversion//:double-conversion", "@nsync//:nsync_cpp", "@protobuf_archive//:protobuf", ], @@ -1270,6 +1273,7 @@ cc_library( deps = [ ":protos_all_cc_impl", "//third_party/eigen3", + "@double_conversion//:double-conversion", "@nsync//:nsync_cpp", "@protobuf_archive//:protobuf", ], @@ -1333,6 +1337,7 @@ cc_library( deps = [ ":protos_all_cc_impl", "//third_party/eigen3", + "@double_conversion//:double-conversion", "@nsync//:nsync_cpp", "@protobuf_archive//:protobuf", ], @@ -1355,6 +1360,7 @@ cc_library( deps = [ ":protos_all_cc_impl", "//third_party/eigen3", + "@double_conversion//:double-conversion", "@nsync//:nsync_cpp", "@protobuf_archive//:protobuf", ], @@ -1751,6 +1757,7 @@ cc_library( "//tensorflow/core/platform/default/build_config:platformlib", "@snappy", "@zlib_archive//:zlib", + "@double_conversion//:double-conversion", "@protobuf_archive//:protobuf", ] + tf_protos_all_impl() + tf_protos_grappler_impl(), ) diff --git a/tensorflow/core/lib/strings/numbers.cc b/tensorflow/core/lib/strings/numbers.cc index c296daa95d..e4b909296e 100644 --- a/tensorflow/core/lib/strings/numbers.cc +++ b/tensorflow/core/lib/strings/numbers.cc @@ -23,6 +23,8 @@ limitations under the License. #include #include +#include "double-conversion/double-conversion.h" + #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/lib/strings/stringprintf.h" #include "tensorflow/core/platform/logging.h" @@ -110,6 +112,17 @@ T locale_independent_strtonum(const char* str, const char** endptr) { return result; } +static inline const double_conversion::StringToDoubleConverter& +StringToFloatConverter() { + static const double_conversion::StringToDoubleConverter converter( + double_conversion::StringToDoubleConverter::ALLOW_LEADING_SPACES | + double_conversion::StringToDoubleConverter::ALLOW_HEX | + double_conversion::StringToDoubleConverter::ALLOW_TRAILING_SPACES | + double_conversion::StringToDoubleConverter::ALLOW_CASE_INSENSIBILITY, + 0., 0., "inf", "nan"); + return converter; +} + } // namespace namespace strings { @@ -319,25 +332,31 @@ bool safe_strtou32(StringPiece str, uint32* value) { } bool safe_strtof(const char* str, float* value) { - const char* endptr; - *value = locale_independent_strtonum(str, &endptr); - while (isspace(*endptr)) ++endptr; - // Ignore range errors from strtod/strtof. - // The values it returns on underflow and - // overflow are the right fallback in a - // robust setting. - return *str != '\0' && *endptr == '\0'; + int processed_characters_count = -1; + auto len = str_util::Strnlen(str, kFastToBufferSize); + + // If there is no zero-termination in str, fail. + if (len == kFastToBufferSize) return false; + // If string length exceeds int max, fail. + if (len > std::numeric_limits::max()) return false; + + *value = StringToFloatConverter().StringToFloat(str, static_cast(len), + &processed_characters_count); + return processed_characters_count > 0; } bool safe_strtod(const char* str, double* value) { - const char* endptr; - *value = locale_independent_strtonum(str, &endptr); - while (isspace(*endptr)) ++endptr; - // Ignore range errors from strtod/strtof. - // The values it returns on underflow and - // overflow are the right fallback in a - // robust setting. - return *str != '\0' && *endptr == '\0'; + int processed_characters_count = -1; + auto len = str_util::Strnlen(str, kFastToBufferSize); + + // If there is no zero-termination in str, fail. + if (len == kFastToBufferSize) return false; + // If string length exceeds int max, fail. + if (len > std::numeric_limits::max()) return false; + + *value = StringToFloatConverter().StringToDouble(str, static_cast(len), + &processed_characters_count); + return processed_characters_count > 0; } size_t FloatToBuffer(float value, char* buffer) { diff --git a/tensorflow/core/lib/strings/numbers.h b/tensorflow/core/lib/strings/numbers.h index 6b7703be37..e9add42849 100644 --- a/tensorflow/core/lib/strings/numbers.h +++ b/tensorflow/core/lib/strings/numbers.h @@ -114,11 +114,13 @@ bool safe_strtou64(StringPiece str, uint64* value); // Convert strings to floating point values. // Leading and trailing spaces are allowed. // Values may be rounded on over- and underflow. +// Returns false on invalid input or if `strlen(value) >= kFastToBufferSize`. bool safe_strtof(const char* str, float* value); // Convert strings to double precision floating point values. // Leading and trailing spaces are allowed. // Values may be rounded on over- and underflow. +// Returns false on invalid input or if `strlen(value) >= kFastToBufferSize`. bool safe_strtod(const char* str, double* value); inline bool ProtoParseNumeric(StringPiece s, int32* value) { diff --git a/tensorflow/core/lib/strings/numbers_test.cc b/tensorflow/core/lib/strings/numbers_test.cc index e15161de66..0f22dac262 100644 --- a/tensorflow/core/lib/strings/numbers_test.cc +++ b/tensorflow/core/lib/strings/numbers_test.cc @@ -15,6 +15,7 @@ limitations under the License. #include "tensorflow/core/lib/strings/numbers.h" +#include #include #include "tensorflow/core/platform/test.h" @@ -277,7 +278,49 @@ TEST(safe_strtof, Float) { EXPECT_TRUE(safe_strtof("-0x2A", &result)); EXPECT_EQ(-42.0f, result); + EXPECT_TRUE(safe_strtof(" -0x2", &result)); + EXPECT_EQ(-2.0f, result); + + EXPECT_TRUE(safe_strtof("8 \t", &result)); + EXPECT_EQ(8.0f, result); + + EXPECT_TRUE(safe_strtof("\t20.0\t ", &result)); + EXPECT_EQ(20.0f, result); + EXPECT_FALSE(safe_strtof("-infinity is awesome", &result)); + + // Make sure we exit cleanly if the string is not terminated + char test_str[2 * kFastToBufferSize]; + for (int i = 0; i < 2 * kFastToBufferSize; ++i) test_str[i] = 'a'; + EXPECT_FALSE(safe_strtof(test_str, &result)); + + // Make sure we exit cleanly if the string is too long + test_str[kFastToBufferSize + 1] = '\0'; + EXPECT_FALSE(safe_strtof(test_str, &result)); + + EXPECT_TRUE(safe_strtof("-inf", &result)); + EXPECT_EQ(-std::numeric_limits::infinity(), result); + + EXPECT_TRUE(safe_strtof("+inf", &result)); + EXPECT_EQ(std::numeric_limits::infinity(), result); + + EXPECT_TRUE(safe_strtof("InF", &result)); + EXPECT_EQ(std::numeric_limits::infinity(), result); + + EXPECT_TRUE(safe_strtof("-INF", &result)); + EXPECT_EQ(-std::numeric_limits::infinity(), result); + + EXPECT_TRUE(safe_strtof("nan", &result)); + EXPECT_TRUE(std::isnan(result)); + + EXPECT_TRUE(safe_strtof("-nan", &result)); + EXPECT_TRUE(std::isnan(result)); + + EXPECT_TRUE(safe_strtof("-NaN", &result)); + EXPECT_TRUE(std::isnan(result)); + + EXPECT_TRUE(safe_strtof("+NAN", &result)); + EXPECT_TRUE(std::isnan(result)); } TEST(safe_strtod, Double) { @@ -287,6 +330,15 @@ TEST(safe_strtod, Double) { EXPECT_EQ(0.1234567890123, result); EXPECT_FALSE(safe_strtod("0.1234567890123abc", &result)); + // Make sure we exit cleanly if the string is not terminated + char test_str[2 * kFastToBufferSize]; + for (int i = 0; i < 2 * kFastToBufferSize; ++i) test_str[i] = 'a'; + EXPECT_FALSE(safe_strtod(test_str, &result)); + + // Make sure we exit cleanly if the string is too long + test_str[kFastToBufferSize + 1] = '\0'; + EXPECT_FALSE(safe_strtod(test_str, &result)); + // Overflow to infinity, underflow to 0. EXPECT_TRUE(safe_strtod("1e310", &result)); EXPECT_EQ(std::numeric_limits::infinity(), result); @@ -296,6 +348,41 @@ TEST(safe_strtod, Double) { EXPECT_TRUE(safe_strtod("1e-325", &result)); EXPECT_EQ(0, result); + + EXPECT_TRUE(safe_strtod(" -0x1c", &result)); + EXPECT_EQ(-28.0, result); + + EXPECT_TRUE(safe_strtod("50 \t", &result)); + EXPECT_EQ(50.0, result); + + EXPECT_TRUE(safe_strtod("\t82.0\t ", &result)); + EXPECT_EQ(82.0, result); + + EXPECT_FALSE(safe_strtod("infinity", &result)); + + EXPECT_TRUE(safe_strtod("-inf", &result)); + EXPECT_EQ(-std::numeric_limits::infinity(), result); + + EXPECT_TRUE(safe_strtod("+inf", &result)); + EXPECT_EQ(std::numeric_limits::infinity(), result); + + EXPECT_TRUE(safe_strtod("InF", &result)); + EXPECT_EQ(std::numeric_limits::infinity(), result); + + EXPECT_TRUE(safe_strtod("-INF", &result)); + EXPECT_EQ(-std::numeric_limits::infinity(), result); + + EXPECT_TRUE(safe_strtod("nan", &result)); + EXPECT_TRUE(std::isnan(result)); + + EXPECT_TRUE(safe_strtod("-nan", &result)); + EXPECT_TRUE(std::isnan(result)); + + EXPECT_TRUE(safe_strtod("-NaN", &result)); + EXPECT_TRUE(std::isnan(result)); + + EXPECT_TRUE(safe_strtod("+NAN", &result)); + EXPECT_TRUE(std::isnan(result)); } } // namespace strings diff --git a/tensorflow/core/lib/strings/str_util.cc b/tensorflow/core/lib/strings/str_util.cc index 2c9e98357a..4598b8ccc7 100644 --- a/tensorflow/core/lib/strings/str_util.cc +++ b/tensorflow/core/lib/strings/str_util.cc @@ -454,6 +454,14 @@ bool SplitAndParseAsFloats(StringPiece text, char delim, result); } +size_t Strnlen(const char* str, const size_t string_max_len) { + size_t len = 0; + while (len < string_max_len && str[len] != '\0') { + ++len; + } + return len; +} + bool StrContains(StringPiece haystack, StringPiece needle) { return std::search(haystack.begin(), haystack.end(), needle.begin(), needle.end()) != haystack.end(); diff --git a/tensorflow/core/lib/strings/str_util.h b/tensorflow/core/lib/strings/str_util.h index 065871c1b4..e97d00b975 100644 --- a/tensorflow/core/lib/strings/str_util.h +++ b/tensorflow/core/lib/strings/str_util.h @@ -223,6 +223,11 @@ std::vector Split(StringPiece text, char delims, Predicate p) { return Split(text, StringPiece(&delims, 1), p); } +// Returns the length of the given null-terminated byte string 'str'. +// Returns 'string_max_len' if the null character was not found in the first +// 'string_max_len' bytes of 'str'. +size_t Strnlen(const char* str, const size_t string_max_len); + } // namespace str_util } // namespace tensorflow diff --git a/tensorflow/core/lib/strings/str_util_test.cc b/tensorflow/core/lib/strings/str_util_test.cc index 63643c3e8e..3bf3e99825 100644 --- a/tensorflow/core/lib/strings/str_util_test.cc +++ b/tensorflow/core/lib/strings/str_util_test.cc @@ -430,56 +430,12 @@ TEST(StringReplace, EmptyStringReplaceAll) { EXPECT_EQ("", str_util::StringReplace("", "a", "X", /*replace_all=*/true)); } -TEST(StartsWith, Basic) { - const string s1( - "123" - "\0" - "456", - 7); - const StringPiece a("foobar"); - const StringPiece b(s1); - const StringPiece e; - EXPECT_TRUE(str_util::StartsWith(a, a)); - EXPECT_TRUE(str_util::StartsWith(a, "foo")); - EXPECT_TRUE(str_util::StartsWith(a, e)); - EXPECT_TRUE(str_util::StartsWith(b, s1)); - EXPECT_TRUE(str_util::StartsWith(b, b)); - EXPECT_TRUE(str_util::StartsWith(b, e)); - EXPECT_TRUE(str_util::StartsWith(e, "")); - EXPECT_FALSE(str_util::StartsWith(a, b)); - EXPECT_FALSE(str_util::StartsWith(b, a)); - EXPECT_FALSE(str_util::StartsWith(e, a)); -} - -TEST(EndsWith, Basic) { - const string s1( - "123" - "\0" - "456", - 7); - const StringPiece a("foobar"); - const StringPiece b(s1); - const StringPiece e; - EXPECT_TRUE(str_util::EndsWith(a, a)); - EXPECT_TRUE(str_util::EndsWith(a, "bar")); - EXPECT_TRUE(str_util::EndsWith(a, e)); - EXPECT_TRUE(str_util::EndsWith(b, s1)); - EXPECT_TRUE(str_util::EndsWith(b, b)); - EXPECT_TRUE(str_util::EndsWith(b, e)); - EXPECT_TRUE(str_util::EndsWith(e, "")); - EXPECT_FALSE(str_util::EndsWith(a, b)); - EXPECT_FALSE(str_util::EndsWith(b, a)); - EXPECT_FALSE(str_util::EndsWith(e, a)); -} - -TEST(StrContains, Basic) { - StringPiece a("abcdefg"); - StringPiece b("abcd"); - StringPiece c("efg"); - StringPiece d("gh"); - EXPECT_TRUE(str_util::StrContains(a, b)); - EXPECT_TRUE(str_util::StrContains(a, c)); - EXPECT_TRUE(!str_util::StrContains(a, d)); +TEST(Strnlen, Basic) { + EXPECT_EQ(0, str_util::Strnlen("ab", 0)); + EXPECT_EQ(1, str_util::Strnlen("a", 1)); + EXPECT_EQ(2, str_util::Strnlen("abcd", 2)); + EXPECT_EQ(3, str_util::Strnlen("abc", 10)); + EXPECT_EQ(4, str_util::Strnlen("a \t\n", 10)); } } // namespace tensorflow diff --git a/tensorflow/tools/lib_package/BUILD b/tensorflow/tools/lib_package/BUILD index 0ede8c6370..569b6678ca 100644 --- a/tensorflow/tools/lib_package/BUILD +++ b/tensorflow/tools/lib_package/BUILD @@ -118,6 +118,7 @@ genrule( "@com_googlesource_code_re2//:LICENSE", "@cub_archive//:LICENSE.TXT", "@curl//:COPYING", + "@double_conversion//:LICENSE", "@eigen_archive//:COPYING.MPL2", "@farmhash_archive//:COPYING", "@fft2d//:fft/readme.txt", @@ -155,6 +156,7 @@ genrule( "@com_googlesource_code_re2//:LICENSE", "@cub_archive//:LICENSE.TXT", "@curl//:COPYING", + "@double_conversion//:LICENSE", "@eigen_archive//:COPYING.MPL2", "@farmhash_archive//:COPYING", "@fft2d//:fft/readme.txt", diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD index 0ac5a5bb6d..7b508f87ab 100644 --- a/tensorflow/tools/pip_package/BUILD +++ b/tensorflow/tools/pip_package/BUILD @@ -128,6 +128,7 @@ filegroup( "@com_googlesource_code_re2//:LICENSE", "@cub_archive//:LICENSE.TXT", "@curl//:COPYING", + "@double_conversion//:LICENSE", "@eigen_archive//:COPYING.MPL2", "@farmhash_archive//:COPYING", "@fft2d//:fft/readme.txt", diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index aeaf8d7a24..bbef4b9e5f 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -693,6 +693,16 @@ def tf_workspace(path_prefix="", tf_repo_name=""): build_file = clean_dep("//third_party/flatbuffers:flatbuffers.BUILD"), ) + native.new_http_archive( + name = "double_conversion", + urls = [ + "https://github.com/google/double-conversion/archive/3992066a95b823efc8ccc1baf82a1cfc73f6e9b8.zip", + ], + sha256 = "2f7fbffac0d98d201ad0586f686034371a6d152ca67508ab611adc2386ad30de", + strip_prefix = "double-conversion-3992066a95b823efc8ccc1baf82a1cfc73f6e9b8", + build_file = clean_dep("//third_party:double_conversion.BUILD") + ) + tf_http_archive( name = "tflite_mobilenet", sha256 = "23f814d1c076bdf03715dfb6cab3713aa4fbdf040fd5448c43196bd2e97a4c1b", diff --git a/third_party/double_conversion.BUILD b/third_party/double_conversion.BUILD new file mode 100644 index 0000000000..9f905216c0 --- /dev/null +++ b/third_party/double_conversion.BUILD @@ -0,0 +1,38 @@ +# Bazel(http://bazel.io) BUILD file + +licenses(["notice"]) + +exports_files(["LICENSE"]) + +cc_library( + name = "double-conversion", + srcs = [ + "double-conversion/bignum.cc", + "double-conversion/bignum-dtoa.cc", + "double-conversion/cached-powers.cc", + "double-conversion/diy-fp.cc", + "double-conversion/double-conversion.cc", + "double-conversion/fast-dtoa.cc", + "double-conversion/fixed-dtoa.cc", + "double-conversion/strtod.cc", + "double-conversion/utils.h", + ], + hdrs = [ + "double-conversion/bignum.h", + "double-conversion/bignum-dtoa.h", + "double-conversion/cached-powers.h", + "double-conversion/diy-fp.h", + "double-conversion/double-conversion.h", + "double-conversion/fast-dtoa.h", + "double-conversion/fixed-dtoa.h", + "double-conversion/ieee.h", + "double-conversion/strtod.h", + ], + includes = [ + ".", + ], + linkopts = [ + "-lm", + ], + visibility = ["//visibility:public"], +) -- GitLab From 5fbb1feecd77a70b32d333b56bd13b1798b9a766 Mon Sep 17 00:00:00 2001 From: James Qin Date: Fri, 20 Apr 2018 11:23:29 -0700 Subject: [PATCH 744/791] Temporarily set cudnn Rnn math precision to fp32. Problem: When calling cudnnGetRNNLinLayerMatrixParams(), return error CUDNN_STATUS_BAD_PARAM if: * RNN descriptor set math precision = CUDNN_DATA_FLOAT * input descriptor dataType = CUDNN_DATA_HALF * weight descriptor dataType= CUDNN_DATA_HALF If updating Rnn descriptor math precision to CUDNN_DATA_HALF, then no error. cudnn 7.1.4 will fix the problem. PiperOrigin-RevId: 193696566 --- tensorflow/stream_executor/cuda/cuda_dnn.cc | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc index d673e19007..640f270323 100644 --- a/tensorflow/stream_executor/cuda/cuda_dnn.cc +++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc @@ -2529,12 +2529,20 @@ cudnnDataType_t GetConvComputeType() { } // A helper struct to decide whether to use FP32 as the internal compute type -// for rnn when the input data type is FP16. By default it is turned on, -// users can explicitly disable them (choose to use FP16 as the internal compute -// type) through an env-var "TF_FP16_RNN_USE_FP32_COMPUTE=0". +// for rnn when the input data type is FP16. At present it is turned off, +// users can explicitly control them through an env-var +// TF_FP16_RNN_USE_FP32_COMPUTE. +// After the TODO below is fixed, users should almost always use fp32 compute +// type for training. Using fp16 might suffer suboptimal accuracy due to loss +// in precision. struct RnnDoFP32ComputationFP16Input { static constexpr const char* kName = "TF_FP16_RNN_USE_FP32_COMPUTE"; - static constexpr bool kDefaultFlag = true; + // TODO(jamesqin): b/78182362 flip to true when cudnn 7.1.4 fixes the bug. + // Before cudnn 7.1.4 RNN are always done in fp32, no matter what math + // precision is set. + // Set it temporary to false s.t. no error is raised when using fp16 inputs, + // fp32 math precision. + static constexpr bool kDefaultFlag = false; }; // A helper function to return the internal compute type for -- GitLab From 712bbc5d7babd523951445f361f0e339061cd259 Mon Sep 17 00:00:00 2001 From: Akshay Modi Date: Fri, 20 Apr 2018 11:24:53 -0700 Subject: [PATCH 745/791] Allow creating tensors from numpy arrays, and other various constants - try #2 Allow type-inference from a different input tensor, similar to args_to_matching_eager. - Update TFE_Py_TensorShapeSlice to take tuples. - Update int values to allow int/long in py2 END_PUBLIC BEGIN_PUBLIC Automated g4 rollback of changelist 192184809 PiperOrigin-RevId: 193696790 --- tensorflow/python/eager/pywrap_tensor.cc | 201 ++++++++-------- tensorflow/python/eager/pywrap_tensor.h | 10 + tensorflow/python/eager/pywrap_tfe.h | 12 +- tensorflow/python/eager/pywrap_tfe_src.cc | 278 +++++++++++++++++++--- tensorflow/python/eager/tensor_test.py | 7 +- tensorflow/python/framework/ops.py | 16 ++ 6 files changed, 389 insertions(+), 135 deletions(-) diff --git a/tensorflow/python/eager/pywrap_tensor.cc b/tensorflow/python/eager/pywrap_tensor.cc index 519814b979..b5b4e394e3 100644 --- a/tensorflow/python/eager/pywrap_tensor.cc +++ b/tensorflow/python/eager/pywrap_tensor.cc @@ -60,42 +60,6 @@ TFE_TensorHandle* NumpyToTensorHandle(PyObject* obj) { } } -// Casts data referred to by `handle` from type `src_type_enum` to type -// `dst_type_enum`. -TFE_TensorHandle* EagerCast(TFE_Context* ctx, TFE_TensorHandle* handle, - TF_DataType src_type_enum, - TF_DataType dst_type_enum, TF_Status* out_status) { - if (ctx == nullptr) return nullptr; - const char* op_name = "Cast"; - const char* device_name = "/job:localhost/replica:0/task:0/device:CPU:0"; - TFE_Op* op = TFE_NewOp(ctx, op_name, out_status); -#define RETURN_ERROR \ - { \ - TFE_DeleteOp(op); \ - return nullptr; \ - } - if (TF_GetCode(out_status) != TF_OK) RETURN_ERROR - TFE_OpSetDevice(op, device_name, out_status); - if (TF_GetCode(out_status) != TF_OK) RETURN_ERROR - TFE_OpAddInput(op, handle, out_status); - if (TF_GetCode(out_status) != TF_OK) RETURN_ERROR - TFE_OpSetAttrType(op, "SrcT", src_type_enum); - TFE_OpSetAttrType(op, "DstT", dst_type_enum); - TFE_TensorHandle* output = nullptr; - int num_outputs = 1; - TFE_Execute(op, &output, &num_outputs, out_status); - if (TF_GetCode(out_status) != TF_OK || num_outputs != 1 || - output == nullptr) { - if (output != nullptr) { - TFE_DeleteTensorHandle(output); - } - RETURN_ERROR - } - TFE_DeleteOp(op); - return output; -#undef RETURN_ERROR -} - TFE_TensorHandle* CopyToDevice(TFE_TensorHandle* handle, PyObject* ctx, PyObject* dev) { const char* device = ""; @@ -161,6 +125,100 @@ PyObject* PyIntFromDataType(TF_DataType l) { } // namespace +namespace tensorflow { +// Casts data referred to by `handle` from type `src_type_enum` to type +// `dst_type_enum`. +TFE_TensorHandle* EagerCast(TFE_Context* ctx, TFE_TensorHandle* handle, + TF_DataType src_type_enum, + TF_DataType dst_type_enum, TF_Status* out_status) { + if (ctx == nullptr) return nullptr; + const char* op_name = "Cast"; + const char* device_name = "/job:localhost/replica:0/task:0/device:CPU:0"; + TFE_Op* op = TFE_NewOp(ctx, op_name, out_status); +#define RETURN_ERROR \ + { \ + TFE_DeleteOp(op); \ + return nullptr; \ + } + if (TF_GetCode(out_status) != TF_OK) RETURN_ERROR + TFE_OpSetDevice(op, device_name, out_status); + if (TF_GetCode(out_status) != TF_OK) RETURN_ERROR + TFE_OpAddInput(op, handle, out_status); + if (TF_GetCode(out_status) != TF_OK) RETURN_ERROR + TFE_OpSetAttrType(op, "SrcT", src_type_enum); + TFE_OpSetAttrType(op, "DstT", dst_type_enum); + TFE_TensorHandle* output = nullptr; + int num_outputs = 1; + TFE_Execute(op, &output, &num_outputs, out_status); + if (TF_GetCode(out_status) != TF_OK || num_outputs != 1 || + output == nullptr) { + if (output != nullptr) { + TFE_DeleteTensorHandle(output); + } + RETURN_ERROR + } + TFE_DeleteOp(op); + return output; +#undef RETURN_ERROR +} + +TFE_TensorHandle* ConvertToEagerTensor(PyObject* value, PyObject* dtype) { + int desired_dtype = -1; + if (dtype != Py_None) { + if (!PyIntToDataType(dtype, &desired_dtype)) { + PyErr_SetString(PyExc_TypeError, + tensorflow::strings::StrCat( + "Expecting a DataType value for dtype. Got ", + Py_TYPE(dtype)->tp_name) + .c_str()); + return nullptr; + } + } + if (PyArray_Check(value)) { + int desired_np_dtype = -1; + if (desired_dtype >= 0) { + if (!tensorflow::TF_DataType_to_PyArray_TYPE( + static_cast(desired_dtype), &desired_np_dtype) + .ok()) { + PyErr_SetString(PyExc_TypeError, + tensorflow::strings::StrCat( + "Invalid dtype argument value ", desired_dtype) + .c_str()); + return nullptr; + } + } + PyArrayObject* array = reinterpret_cast(value); + int current_np_dtype = PyArray_TYPE(array); + auto safe_value = tensorflow::make_safe(static_cast(nullptr)); + if ((desired_np_dtype >= 0 && desired_np_dtype != current_np_dtype) || + !PyArray_ISCARRAY(array)) { + int new_dtype = + desired_np_dtype >= 0 ? desired_np_dtype : current_np_dtype; + safe_value = tensorflow::make_safe( + PyArray_FromAny(value, PyArray_DescrFromType(new_dtype), 0, 0, + NPY_ARRAY_CARRAY | NPY_ARRAY_FORCECAST, nullptr)); + if (PyErr_Occurred()) return nullptr; + if (safe_value == nullptr) { + PyErr_SetString(PyExc_ValueError, "Error while casting a numpy value"); + return nullptr; + } + value = safe_value.get(); + } + return NumpyToTensorHandle(value); + } else { + tensorflow::Tensor t; + // TODO(josh11b): Have PySeqToTensor set python errors instead of + // returning Status. + auto cppstatus = tensorflow::PySeqToTensor(value, dtype, &t); + if (!cppstatus.ok()) { + PyErr_SetString(PyExc_ValueError, cppstatus.error_message().c_str()); + return nullptr; + } + return TFE_NewTensorHandle(t); + } +} +} // namespace tensorflow + extern "C" { static const int kMaxEagerTensorParentSize = 64; @@ -230,61 +288,16 @@ int EagerTensor_init(EagerTensor* self, PyObject* args, PyObject* kwds) { return -1; } } - tensorflow::Safe_TFE_TensorHandlePtr handle = - tensorflow::make_safe(static_cast(nullptr)); PyErr_Clear(); - if (PyArray_Check(value)) { - int desired_np_dtype = -1; - if (desired_dtype >= 0) { - if (!tensorflow::TF_DataType_to_PyArray_TYPE( - static_cast(desired_dtype), &desired_np_dtype) - .ok()) { - PyErr_SetString(PyExc_TypeError, - tensorflow::strings::StrCat( - "Invalid dtype argument value ", desired_dtype) - .c_str()); - return -1; - } - } - PyArrayObject* array = reinterpret_cast(value); - int current_np_dtype = PyArray_TYPE(array); - auto safe_value = tensorflow::make_safe(static_cast(nullptr)); - if ((desired_np_dtype >= 0 && desired_np_dtype != current_np_dtype) || - !PyArray_ISCARRAY(array)) { - int new_dtype = - desired_np_dtype >= 0 ? desired_np_dtype : current_np_dtype; - safe_value = tensorflow::make_safe( - PyArray_FromAny(value, PyArray_DescrFromType(new_dtype), 0, 0, - NPY_ARRAY_CARRAY | NPY_ARRAY_FORCECAST, nullptr)); - if (PyErr_Occurred()) return -1; - if (safe_value == nullptr) { - PyErr_SetString(PyExc_ValueError, "Error while casting a numpy value"); - return -1; - } - value = safe_value.get(); - } - handle = tensorflow::make_safe(NumpyToTensorHandle(value)); - } else { - tensorflow::Tensor t; - // TODO(josh11b): Have PySeqToTensor set python errors instead of - // returning Status. - auto cppstatus = tensorflow::PySeqToTensor(value, dtype, &t); - if (!cppstatus.ok()) { - PyErr_SetString(PyExc_ValueError, cppstatus.error_message().c_str()); - return -1; - } - handle = tensorflow::make_safe(TFE_NewTensorHandle(t)); - } - if (PyErr_Occurred()) return -1; - if (handle == nullptr) { - PyErr_SetString(PyExc_ValueError, "Error while creating an EagerTensor"); - return -1; - } + tensorflow::Safe_TFE_TensorHandlePtr handle = + tensorflow::make_safe(static_cast( + tensorflow::ConvertToEagerTensor(value, dtype))); + if (handle == nullptr) return -1; TF_DataType handle_dtype = TFE_TensorHandleDataType(handle.get()); if (desired_dtype >= 0 && desired_dtype != handle_dtype) { - handle = tensorflow::make_safe( - EagerCast(GetContext(context), handle.get(), handle_dtype, - static_cast(desired_dtype), self->status)); + handle = tensorflow::make_safe(tensorflow::EagerCast( + GetContext(context), handle.get(), handle_dtype, + static_cast(desired_dtype), self->status)); if (TF_GetCode(self->status) != TF_OK) { PyErr_SetString(PyExc_ValueError, tensorflow::strings::StrCat( @@ -701,12 +714,12 @@ PyObject* TFE_Py_InitEagerTensor(PyObject* base_class) { return reinterpret_cast(EagerTensorType); } -PyObject* TFE_Py_TensorShapeSlice(PyObject* tensor_list, int slice_dim) { - if (!PyList_Check(tensor_list)) { +PyObject* TFE_Py_TensorShapeSlice(PyObject* tensors, int slice_dim) { + if (!PyList_Check(tensors) && !PyTuple_Check(tensors)) { PyErr_SetString(PyExc_TypeError, tensorflow::strings::StrCat( - "tensor_list argument must be a list. Got \"", - Py_TYPE(tensor_list)->tp_name, "\"") + "tensors argument must be a list or a tuple. Got \"", + Py_TYPE(tensors)->tp_name, "\"") .c_str()); return nullptr; } @@ -720,14 +733,14 @@ PyObject* TFE_Py_TensorShapeSlice(PyObject* tensor_list, int slice_dim) { return nullptr; } - Py_ssize_t num_tensors = PyList_Size(tensor_list); + Py_ssize_t num_tensors = PySequence_Fast_GET_SIZE(tensors); int64_t num_tensors_int = static_cast(num_tensors); auto tensor = tensorflow::make_safe(TF_AllocateTensor( TF_INT32, &num_tensors_int, /*num_dims=*/1, /*len=*/4 * num_tensors_int)); int32_t* data = reinterpret_cast(TF_TensorData(tensor.get())); auto status = tensorflow::make_safe(TF_NewStatus()); for (Py_ssize_t i = 0; i < num_tensors; ++i) { - PyObject* tensor_obj = PyList_GET_ITEM(tensor_list, i); + PyObject* tensor_obj = PySequence_Fast_GET_ITEM(tensors, i); if (!EagerTensor_CheckExact(tensor_obj)) { PyErr_SetString(PyExc_TypeError, tensorflow::strings::StrCat( diff --git a/tensorflow/python/eager/pywrap_tensor.h b/tensorflow/python/eager/pywrap_tensor.h index aa1efdd1b8..63ab1ed84d 100644 --- a/tensorflow/python/eager/pywrap_tensor.h +++ b/tensorflow/python/eager/pywrap_tensor.h @@ -22,4 +22,14 @@ limitations under the License. bool EagerTensor_CheckExact(const PyObject* o); tensorflow::int64 EagerTensor_id(const PyObject* tensor); +namespace tensorflow { +TFE_TensorHandle* ConvertToEagerTensor(PyObject* value, PyObject* dtype); + +// TODO(nareshmodi): Move EagerCast and ReadVariableOp (which use the C API to +// execute TFE Ops) to a separate common library. +TFE_TensorHandle* EagerCast(TFE_Context* ctx, TFE_TensorHandle* handle, + TF_DataType src_type_enum, + TF_DataType dst_type_enum, TF_Status* out_status); +} + #endif // TENSORFLOW_PYTHON_EAGER_PYWRAP_TENSOR_H_ diff --git a/tensorflow/python/eager/pywrap_tfe.h b/tensorflow/python/eager/pywrap_tfe.h index 32d731d0f6..691b613e48 100644 --- a/tensorflow/python/eager/pywrap_tfe.h +++ b/tensorflow/python/eager/pywrap_tfe.h @@ -186,16 +186,16 @@ PyObject* TFE_Py_RecordGradient(PyObject* op_name, PyObject* inputs, // Returns the set of variables watched by the given tape. PyObject* TFE_Py_TapeWatchedVariables(PyObject* tape); -// Returns an EagerTensor of dimension [len(`tensor_list`)] containing -// the `slice_dim`'th dimension of each tensor in `tensor_list`. In other words, +// Returns an EagerTensor of dimension [len(`tensors`)] containing +// the `slice_dim`'th dimension of each tensor in `tensors`. In other words, // TFE_Py_TensorShapeSlice takes a slice of dimensions of tensors in -// `tensor_list`. For example, if `tensor_list` contains tensors of with shapes +// `tensors`. For example, if `tensors` contains tensors of with shapes // [1, 2, 3], [4, 5], [6, 7, 8, 9], TFE_Py_TensorShapeSlice called with // `slice_dim` equal to 1 will return [2, 5, 7]. // On error, returns nullptr and sets python exception. -// REQUIRES: `tensor_list` is a python list of EagerTensors +// REQUIRES: `tensors` is a python list/tuple of EagerTensors // REQUIRES: `slice_dim` is non-negative and smaller than the rank of all -// tensors in `tensor_list`. -PyObject* TFE_Py_TensorShapeSlice(PyObject* tensor_list, int slice_dim); +// tensors in `tensors`. +PyObject* TFE_Py_TensorShapeSlice(PyObject* tensors, int slice_dim); #endif // TENSORFLOW_PYTHON_EAGER_PYWRAP_TFE_H_ diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc index d99bd0b0ff..2bfa1f052c 100644 --- a/tensorflow/python/eager/pywrap_tfe_src.cc +++ b/tensorflow/python/eager/pywrap_tfe_src.cc @@ -38,6 +38,54 @@ using tensorflow::strings::Printf; namespace { +struct InputInfo { + InputInfo(int i, bool is_list) : i(i), is_list(is_list) {} + + int i; + bool is_list = false; +}; + +using AttrToInputsMap = + tensorflow::gtl::FlatMap>; + +tensorflow::mutex all_attr_to_input_maps_lock( + tensorflow::LINKER_INITIALIZED); +tensorflow::gtl::FlatMap* GetAllAttrToInputsMaps() { + static auto* all_attr_to_input_maps = + new tensorflow::gtl::FlatMap; + return all_attr_to_input_maps; +} + +AttrToInputsMap* GetAttrToInputsMap(const tensorflow::OpDef& op_def) { + tensorflow::mutex_lock l(all_attr_to_input_maps_lock); + auto* all_attr_to_input_maps = GetAllAttrToInputsMaps(); + + auto* output = + tensorflow::gtl::FindPtrOrNull(*all_attr_to_input_maps, op_def.name()); + if (output != nullptr) { + return output; + } + + std::unique_ptr m(new AttrToInputsMap); + + // Store a list of InputIndex -> List of corresponding inputs. + for (int i = 0; i < op_def.input_arg_size(); i++) { + if (!op_def.input_arg(i).type_attr().empty()) { + auto it = m->find(op_def.input_arg(i).type_attr()); + if (it == m->end()) { + it = m->insert({op_def.input_arg(i).type_attr(), {}}).first; + } + it->second.emplace_back(i, !op_def.input_arg(i).number_attr().empty()); + } + } + + auto* retval = m.get(); + (*all_attr_to_input_maps)[op_def.name()] = m.release(); + + return retval; +} + struct FastPathOpExecInfo { TFE_Context* ctx; const char* device_name; @@ -53,6 +101,14 @@ struct FastPathOpExecInfo { // The op type name of the main op being executed. PyObject* op_name; PyObject* callbacks; + + // All the args passed into the FastPathOpExecInfo. + PyObject* args; + + // DTypes can come from another input that has the same attr. So build that + // map. + const AttrToInputsMap* attr_to_inputs_map; + tensorflow::gtl::FlatMap cached_dtypes; }; #define PARSE_VALUE(fn_name, type, check_fn, parse_fn) \ @@ -76,12 +132,29 @@ PARSE_VALUE(ParseIntValue, int, PyLong_Check, PyLong_AsLong) PARSE_VALUE(ParseInt64Value, int64_t, PyLong_Check, PyLong_AsLong) #else PARSE_VALUE(ParseIntValue, int, PyInt_Check, PyInt_AsLong) -PARSE_VALUE(ParseInt64Value, int64_t, PyInt_Check, PyInt_AsLong) -PARSE_VALUE(ParseInt64LongValue, int64_t, PyLong_Check, PyLong_AsLong) #endif PARSE_VALUE(ParseFloatValue, float, PyFloat_Check, PyFloat_AsDouble) #undef PARSE_VALUE +#if PY_MAJOR_VERSION < 3 +bool ParseInt64Value(const string& key, PyObject* py_value, TF_Status* status, + int64_t* value) { + if (PyInt_Check(py_value)) { + *value = static_cast(PyInt_AsLong(py_value)); + return true; + } else if (PyLong_Check(py_value)) { + *value = static_cast(PyLong_AsLong(py_value)); + return true; + } + TF_SetStatus( + status, TF_INVALID_ARGUMENT, + tensorflow::strings::StrCat("Expecting int or long value for attr ", key, + ", got ", py_value->ob_type->tp_name) + .c_str()); + return false; +} +#endif + Py_ssize_t TensorShapeNumDims(PyObject* value) { const auto size = PySequence_Size(value); if (size == -1) { @@ -234,7 +307,7 @@ bool SetOpAttrList( std::unique_ptr buffer(new int64_t[total_dims]); // Copy the input dims into the buffer and set dims to point to // the start of each list's dims. - std::unique_ptr dims(new const int64_t*[num_values]); + std::unique_ptr dims(new const int64_t*[num_values]); std::unique_ptr num_dims(new int[num_values]); int64_t* offset = buffer.get(); for (int i = 0; i < num_values; ++i) { @@ -296,7 +369,7 @@ void SetOpAttrListDefault( TF_Status* status) { if (type == TF_ATTR_STRING) { int num_values = attr.default_value().list().s_size(); - std::unique_ptr values(new const char*[num_values]); + std::unique_ptr values(new const char*[num_values]); (*attr_list_sizes)[key] = num_values; for (int i = 0; i < num_values; i++) { values[i] = attr.default_value().list().s(i).data(); @@ -349,7 +422,7 @@ void SetOpAttrListDefault( std::unique_ptr buffer(new int64_t[total_dims]); // Copy the input dims into the buffer and set dims to point to // the start of each list's dims. - std::unique_ptr dims(new const int64_t*[num_values]); + std::unique_ptr dims(new const int64_t*[num_values]); std::unique_ptr num_dims(new int[num_values]); int64_t* offset = buffer.get(); for (int i = 0; i < num_values; ++i) { @@ -369,7 +442,7 @@ void SetOpAttrListDefault( } else if (type == TF_ATTR_FUNC) { int num_values = attr.default_value().list().func_size(); (*attr_list_sizes)[key] = num_values; - std::unique_ptr funcs(new const TFE_Op*[num_values]); + std::unique_ptr funcs(new const TFE_Op*[num_values]); for (int i = 0; i < num_values; i++) { funcs[i] = GetFunc(ctx, attr.default_value().list().func(i), status); } @@ -1399,10 +1472,39 @@ PyObject* GetPythonObjectFromString(const char* s) { #endif } +PyObject* GetPythonObjectFromInt(int num) { +#if PY_MAJOR_VERSION >= 3 + return PyLong_FromLong(num); +#else + return PyInt_FromLong(num); +#endif +} + bool CheckResourceVariable(PyObject* item) { return PyObject_TypeCheck(item, resource_variable_type); } +bool IsNumberType(PyObject* item) { +#if PY_MAJOR_VERSION >= 3 + return PyFloat_Check(item) || PyLong_Check(item); +#else + return PyFloat_Check(item) || PyInt_Check(item) || PyLong_Check(item); +#endif +} + +bool CheckOneInput(PyObject* item) { + if (EagerTensor_CheckExact(item) || CheckResourceVariable(item) || + PyArray_Check(item) || IsNumberType(item)) { + return true; + } + + // Sequences are not properly handled. Sequences with purely python numeric + // types work, but sequences with mixes of EagerTensors and python numeric + // types don't work. + // TODO(nareshmodi): fix + return false; +} + bool CheckInputsOk(PyObject* seq, int start_index, const tensorflow::OpDef& op_def) { for (int i = 0; i < op_def.input_arg_size(); i++) { @@ -1419,8 +1521,7 @@ bool CheckInputsOk(PyObject* seq, int start_index, } for (Py_ssize_t j = 0; j < PySequence_Fast_GET_SIZE(item); j++) { PyObject* inner_item = PySequence_Fast_GET_ITEM(item, j); - if (!EagerTensor_CheckExact(inner_item) && - !CheckResourceVariable(inner_item)) { + if (!CheckOneInput(inner_item)) { VLOG(1) << "Falling back to slow path for Op \"" << op_def.name() << "\", Input \"" << op_def.input_arg(i).name() << "\", Index " @@ -1430,7 +1531,7 @@ bool CheckInputsOk(PyObject* seq, int start_index, return false; } } - } else if (!EagerTensor_CheckExact(item) && !CheckResourceVariable(item)) { + } else if (!CheckOneInput(item)) { VLOG(1) << "Falling back to slow path for Op \"" << op_def.name() << "\", Input \"" << op_def.input_arg(i).name() @@ -1443,6 +1544,52 @@ bool CheckInputsOk(PyObject* seq, int start_index, return true; } +PyObject* MaybeGetDType(PyObject* item) { + if (EagerTensor_CheckExact(item)) { + tensorflow::Safe_PyObjectPtr py_dtype( + PyObject_GetAttrString(item, "dtype")); + return PyObject_GetAttrString(py_dtype.get(), "_type_enum"); + } + + if (CheckResourceVariable(item)) { + tensorflow::Safe_PyObjectPtr py_dtype( + PyObject_GetAttrString(item, "_dtype")); + return PyObject_GetAttrString(py_dtype.get(), "_type_enum"); + } + + return nullptr; +} + +PyObject* MaybeGetDTypeForAttr(const string& attr, + FastPathOpExecInfo* op_exec_info) { + auto cached_it = op_exec_info->cached_dtypes.find(attr); + if (cached_it != op_exec_info->cached_dtypes.end()) { + return GetPythonObjectFromInt(cached_it->second); + } + + auto it = op_exec_info->attr_to_inputs_map->find(attr); + if (it == op_exec_info->attr_to_inputs_map->end()) { + // No other inputs - this should never happen. + Py_RETURN_NONE; + } + + for (const auto& input_info : it->second) { + PyObject* item = PyTuple_GET_ITEM( + op_exec_info->args, kFastPathExecuteInputStartIndex + input_info.i); + if (input_info.is_list) { + for (int i = 0; i < PySequence_Fast_GET_SIZE(item); i++) { + auto* dtype = MaybeGetDType(PySequence_Fast_GET_ITEM(item, i)); + if (dtype != nullptr) return dtype; + } + } else { + auto* dtype = MaybeGetDType(item); + if (dtype != nullptr) return dtype; + } + } + + Py_RETURN_NONE; +} + bool OpDoesntRequireOutput(const string& op_name) { static tensorflow::gtl::FlatSet* ops_that_dont_require_outputs = new tensorflow::gtl::FlatSet({ @@ -1668,23 +1815,80 @@ bool ReadVariableOp(const FastPathOpExecInfo& parent_op_exec_info, // i) input is an EagerTensor // ii) input is a ResourceVariable - in this case, the is_variable param is set // to true. -bool ConvertToTensor(const FastPathOpExecInfo& op_exec_info, PyObject* input, - tensorflow::Safe_PyObjectPtr* output_handle, - TF_Status* status) { - if (CheckResourceVariable(input)) { +// +// NOTE: dtype_hint_getter must *always* return a PyObject that can be +// decref'd. So if no hint is found, Py_RETURN_NONE (which correctly +// increfs Py_None). +bool ConvertToTensor( + const FastPathOpExecInfo& op_exec_info, PyObject* input, + tensorflow::Safe_PyObjectPtr* output_handle, + // This gets a hint for this particular input. + const std::function& dtype_hint_getter, + // This sets the dtype after conversion is complete. + const std::function& dtype_setter, + TF_Status* status) { + if (EagerTensor_CheckExact(input)) { + Py_INCREF(input); + output_handle->reset(input); + return true; + } else if (CheckResourceVariable(input)) { return ReadVariableOp(op_exec_info, input, output_handle, status); } - Py_INCREF(input); - output_handle->reset(input); + // The hint comes from a supposedly similarly typed tensor. + tensorflow::Safe_PyObjectPtr dtype_hint(dtype_hint_getter()); + if (PyErr_Occurred()) { + return false; + } + + tensorflow::Safe_TFE_TensorHandlePtr handle = + tensorflow::make_safe(static_cast( + tensorflow::ConvertToEagerTensor(input, dtype_hint.get()))); + if (handle == nullptr) { + status->status = tensorflow::errors::InvalidArgument( + "Unable to convert value to tensor"); + return false; + } + + int desired_dtype = -1; + if (dtype_hint.get() != Py_None) { + if (!ParseTypeValue("", dtype_hint.get(), status, &desired_dtype)) { + status->status = tensorflow::errors::InvalidArgument( + "Expecting a DataType value for dtype. Got ", + Py_TYPE(dtype_hint.get())->tp_name); + } + } + + TF_DataType handle_dtype = TFE_TensorHandleDataType(handle.get()); + if (desired_dtype >= 0 && desired_dtype != handle_dtype) { + handle = tensorflow::make_safe( + tensorflow::EagerCast(op_exec_info.ctx, handle.get(), handle_dtype, + static_cast(desired_dtype), status)); + if (!status->status.ok()) return false; + + handle_dtype = TFE_TensorHandleDataType(handle.get()); + } + + if (handle_dtype != TF_INT32) { + // Note that this is a shallow copy and will share the underlying buffer + // if copying to the same device. + handle = tensorflow::make_safe(TFE_TensorHandleCopyToDevice( + handle.get(), op_exec_info.ctx, op_exec_info.device_name, status)); + if (!status->status.ok()) return false; + } + + output_handle->reset(EagerTensorFromHandle(handle.release())); + + dtype_setter(handle_dtype); return true; } // Adds input and type attr to the op, and to the list of flattened // inputs/attrs. -bool AddInputToOp(const FastPathOpExecInfo& op_exec_info, PyObject* input, - const tensorflow::OpDef::ArgDef* input_arg, +bool AddInputToOp(FastPathOpExecInfo* op_exec_info, PyObject* input, + const bool add_type_attr, + const tensorflow::OpDef::ArgDef& input_arg, std::vector* flattened_attrs, std::vector* flattened_inputs, TFE_Op* op, TF_Status* status) { @@ -1693,18 +1897,30 @@ bool AddInputToOp(const FastPathOpExecInfo& op_exec_info, PyObject* input, // out of scope in this function. tensorflow::Safe_PyObjectPtr py_eager_tensor = nullptr; - if (!ConvertToTensor(op_exec_info, input, &py_eager_tensor, status)) { + if (!ConvertToTensor( + *op_exec_info, input, &py_eager_tensor, + [&]() { + if (input_arg.type() != tensorflow::DataType::DT_INVALID) { + return GetPythonObjectFromInt(input_arg.type()); + } + return MaybeGetDTypeForAttr(input_arg.type_attr(), op_exec_info); + }, + [&](const TF_DataType dtype) { + op_exec_info->cached_dtypes[input_arg.type_attr()] = + static_cast(dtype); + }, + status)) { return false; } TFE_TensorHandle* input_handle = EagerTensor_Handle(py_eager_tensor.get()); - if (input_arg != nullptr && !input_arg->type_attr().empty()) { + if (add_type_attr && !input_arg.type_attr().empty()) { auto dtype = TFE_TensorHandleDataType(input_handle); - TFE_OpSetAttrType(op, input_arg->type_attr().data(), dtype); + TFE_OpSetAttrType(op, input_arg.type_attr().data(), dtype); if (flattened_attrs != nullptr) { flattened_attrs->emplace_back( - GetPythonObjectFromString(input_arg->type_attr().data())); + GetPythonObjectFromString(input_arg.type_attr().data())); flattened_attrs->emplace_back(PyLong_FromLong(dtype)); } } @@ -1844,6 +2060,7 @@ PyObject* TFE_Py_FastPathExecute_C(PyObject*, PyObject* args) { op_exec_info.ctx = reinterpret_cast( PyCapsule_GetPointer(PyTuple_GET_ITEM(args, 0), nullptr)); + op_exec_info.args = args; if (op_exec_info.ctx == nullptr) { // The context hasn't been initialized. It will be in the slow path. @@ -1892,6 +2109,8 @@ PyObject* TFE_Py_FastPathExecute_C(PyObject*, PyObject* args) { return nullptr; } + op_exec_info.attr_to_inputs_map = GetAttrToInputsMap(*op_def); + TF_Status* status = TF_NewStatus(); TFE_Op* op = TFE_NewOp(op_exec_info.ctx, op_def->name().c_str(), status); auto cleaner = tensorflow::gtl::MakeCleanup([status, op] { @@ -1986,17 +2205,16 @@ PyObject* TFE_Py_FastPathExecute_C(PyObject*, PyObject* args) { if (len > 0) { // First item adds the type attr. - if (!AddInputToOp(op_exec_info, PySequence_Fast_GET_ITEM(input, 0), - &input_arg, flattened_attrs.get(), + if (!AddInputToOp(&op_exec_info, PySequence_Fast_GET_ITEM(input, 0), + true, input_arg, flattened_attrs.get(), flattened_inputs.get(), op, status)) { return nullptr; } for (Py_ssize_t j = 1; j < len; j++) { // Since the list is homogeneous, we don't need to re-add the attr. - if (!AddInputToOp(op_exec_info, PySequence_Fast_GET_ITEM(input, j), - nullptr /* input_arg */, - nullptr /* flattened_attrs */, + if (!AddInputToOp(&op_exec_info, PySequence_Fast_GET_ITEM(input, j), + false, input_arg, nullptr /* flattened_attrs */, flattened_inputs.get(), op, status)) { return nullptr; } @@ -2018,7 +2236,8 @@ PyObject* TFE_Py_FastPathExecute_C(PyObject*, PyObject* args) { PyObject* py_input = PySequence_Fast_GET_ITEM(input, j); tensorflow::Safe_PyObjectPtr py_eager_tensor; if (!ConvertToTensor(op_exec_info, py_input, &py_eager_tensor, - status)) { + []() { Py_RETURN_NONE; }, + [](const TF_DataType& dtype) {}, status)) { return nullptr; } @@ -2048,8 +2267,9 @@ PyObject* TFE_Py_FastPathExecute_C(PyObject*, PyObject* args) { attr_list_sizes[attr_name] = len; } else { // The item is a single item. - if (!AddInputToOp(op_exec_info, input, &input_arg, flattened_attrs.get(), - flattened_inputs.get(), op, status)) { + if (!AddInputToOp(&op_exec_info, input, true, input_arg, + flattened_attrs.get(), flattened_inputs.get(), op, + status)) { return nullptr; } } diff --git a/tensorflow/python/eager/tensor_test.py b/tensorflow/python/eager/tensor_test.py index 0bd5a5dbaf..b044b30231 100644 --- a/tensorflow/python/eager/tensor_test.py +++ b/tensorflow/python/eager/tensor_test.py @@ -278,14 +278,9 @@ class TFETensorUtilTest(test_util.TensorFlowTestCase): with self.assertRaisesRegexp( TypeError, - r"tensor_list argument must be a list. Got \"EagerTensor\""): + r"tensors argument must be a list or a tuple. Got \"EagerTensor\""): pywrap_tensorflow.TFE_Py_TensorShapeSlice(t1, -2) - with self.assertRaisesRegexp( - TypeError, - r"tensor_list argument must be a list. Got \"tuple\""): - pywrap_tensorflow.TFE_Py_TensorShapeSlice((t1,), -2) - def testNegativeSliceDim(self): t1 = _create_tensor([1, 2], dtype=dtypes.int32) diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 662cda2a7d..8cd6820f6a 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -1385,6 +1385,22 @@ def register_tensor_conversion_function(base_type, if not callable(conversion_func): raise TypeError("conversion_func must be callable.") + # context._context is checked so that we don't inadvertently create it. + # This is because enable_eager_execution will fail when called from the main + # function if the context._context is already created, and the + # register_tensor_conversion_function calls happen when the module is + # imported. + if context._context is not None and context.executing_eagerly( + ) and isinstance(base_type, six.integer_types + ( + float, + np.ndarray, + )): + # TODO(nareshmodi): consider setting a context variable which disables the + # fastpath instead. + raise TypeError( + "Cannot register conversions for numpy arrays, python number types " + "when executing eagerly.") + try: funcs_at_priority = _tensor_conversion_func_registry[priority] except KeyError: -- GitLab From 76ea66f24d4370e6e7848b83fc0b571ba7edfa2d Mon Sep 17 00:00:00 2001 From: Akshay Modi Date: Fri, 20 Apr 2018 11:34:55 -0700 Subject: [PATCH 746/791] Move the guts of TFE_Op into EagerOperation PiperOrigin-RevId: 193698320 --- tensorflow/c/eager/BUILD | 2 + tensorflow/c/eager/c_api.cc | 230 +++++++++--------- tensorflow/c/eager/c_api_internal.h | 16 +- tensorflow/core/common_runtime/eager/BUILD | 16 ++ .../common_runtime/eager/eager_operation.cc | 33 +++ .../common_runtime/eager/eager_operation.h | 74 ++++++ 6 files changed, 242 insertions(+), 129 deletions(-) create mode 100644 tensorflow/core/common_runtime/eager/eager_operation.cc create mode 100644 tensorflow/core/common_runtime/eager/eager_operation.h diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD index 3e14c10727..d66386acbd 100644 --- a/tensorflow/c/eager/BUILD +++ b/tensorflow/c/eager/BUILD @@ -51,6 +51,7 @@ tf_cuda_library( ], "//conditions:default": [], }) + [ + "//tensorflow/core/common_runtime/eager:eager_operation", "//tensorflow/core:gpu_runtime", ], ) @@ -73,6 +74,7 @@ tf_cuda_library( "//tensorflow/core:lib_internal", "//tensorflow/core/common_runtime/eager:context", "//tensorflow/core/common_runtime/eager:eager_executor", + "//tensorflow/core/common_runtime/eager:eager_operation", "//tensorflow/core/common_runtime/eager:kernel_and_device", "//tensorflow/core/common_runtime/eager:tensor_handle", ], diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc index 369342b142..b7a3097208 100644 --- a/tensorflow/c/eager/c_api.cc +++ b/tensorflow/c/eager/c_api.cc @@ -241,21 +241,18 @@ TFE_Op* TFE_NewOp(TFE_Context* ctx, const char* op_or_function_name, void TFE_DeleteOp(TFE_Op* op) { delete op; } void TFE_OpSetDevice(TFE_Op* op, const char* device_name, TF_Status* status) { - tensorflow::Device* d = nullptr; - if (device_name != nullptr && strlen(device_name) > 0) { - status->status = op->ctx->context.FindDeviceByName(device_name, &d); - } - op->device = d; + status->status = op->operation.SetDevice(device_name); } const char* TFE_OpGetDevice(TFE_Op* op, TF_Status* status) { - tensorflow::Device* device = - (op->device == nullptr) ? op->ctx->context.HostCPU() : op->device; + tensorflow::Device* device = (op->operation.Device() == nullptr) + ? op->operation.EagerContext()->HostCPU() + : op->operation.Device(); return device->name().c_str(); } void TFE_OpSetXLACompilation(TFE_Op* op, unsigned char enable) { - op->use_xla = enable; + op->operation.SetUseXla(enable); #ifndef TENSORFLOW_EAGER_USE_XLA LOG(WARNING) << "This call is a no-op, as the TensorFlow library is not " "built with XLA support."; @@ -263,22 +260,20 @@ void TFE_OpSetXLACompilation(TFE_Op* op, unsigned char enable) { } void TFE_OpAddInput(TFE_Op* op, TFE_TensorHandle* h, TF_Status* status) { - h->handle->Ref(); - op->inputs.push_back(h->handle); - op->attrs.NumInputs(op->inputs.size()); + op->operation.AddInput(h->handle); } TF_AttrType TFE_OpGetAttrType(TFE_Op* op, const char* attr_name, unsigned char* is_list, TF_Status* status) { TF_AttrType ret; - if (op->is_function()) { + if (op->operation.is_function()) { status->status = tensorflow::errors::Unimplemented( "TODO(apassos): Support for attributes for TensorFlow functions is not " "ready yet."); return TF_ATTR_INT; // The compiler requires that we return something. } - status->status = - tensorflow::AttrTypeByName(*op->attr_types, attr_name, &ret, is_list); + status->status = tensorflow::AttrTypeByName(*op->operation.AttrTypes(), + attr_name, &ret, is_list); return ret; } @@ -297,23 +292,24 @@ TF_AttrType TFE_OpNameGetAttrType(TFE_Context* ctx, } void TFE_OpSetAttrString(TFE_Op* op, const char* attr_name, const char* value) { - op->attrs.Set(attr_name, value); + op->operation.MutableAttrs()->Set(attr_name, value); } void TFE_OpSetAttrInt(TFE_Op* op, const char* attr_name, int64_t value) { - op->attrs.Set(attr_name, static_cast(value)); + op->operation.MutableAttrs()->Set(attr_name, static_cast(value)); } void TFE_OpSetAttrFloat(TFE_Op* op, const char* attr_name, float value) { - op->attrs.Set(attr_name, value); + op->operation.MutableAttrs()->Set(attr_name, value); } void TFE_OpSetAttrBool(TFE_Op* op, const char* attr_name, unsigned char value) { - op->attrs.Set(attr_name, (value == 0) ? false : true); + op->operation.MutableAttrs()->Set(attr_name, (value == 0) ? false : true); } void TFE_OpSetAttrType(TFE_Op* op, const char* attr_name, TF_DataType value) { - op->attrs.Set(attr_name, static_cast(value)); + op->operation.MutableAttrs()->Set(attr_name, + static_cast(value)); } void TFE_OpSetAttrShape(TFE_Op* op, const char* attr_name, const int64_t* dims, @@ -335,23 +331,24 @@ void TFE_OpSetAttrShape(TFE_Op* op, const char* attr_name, const int64_t* dims, proto.add_dim()->set_size(dims[d]); } } - op->attrs.Set(attr_name, proto); + op->operation.MutableAttrs()->Set(attr_name, proto); } void TFE_OpSetAttrFunction(TFE_Op* op, const char* attr_name, const TFE_Op* value) { tensorflow::AttrValue attr_value; tensorflow::NameAttrList* func = attr_value.mutable_func(); - func->set_name(value->name); - value->attrs.FillAttrValueMap(func->mutable_attr()); - op->attrs.Set(attr_name, attr_value); + func->set_name(value->operation.Name()); + value->operation.Attrs().FillAttrValueMap(func->mutable_attr()); + op->operation.MutableAttrs()->Set(attr_name, attr_value); } #define TFE_OP_SET_ATTR_LIST(fn, type) \ void fn(TFE_Op* op, const char* attr_name, const type* values, \ int num_values) { \ - op->attrs.Set(attr_name, tensorflow::gtl::ArraySlice( \ - values, num_values)); \ + op->operation.MutableAttrs()->Set( \ + attr_name, \ + tensorflow::gtl::ArraySlice(values, num_values)); \ } TFE_OP_SET_ATTR_LIST(TFE_OpSetAttrStringList, char*) TFE_OP_SET_ATTR_LIST(TFE_OpSetAttrFloatList, float) @@ -359,14 +356,14 @@ TFE_OP_SET_ATTR_LIST(TFE_OpSetAttrFloatList, float) void TFE_OpSetAttrIntList(TFE_Op* op, const char* attr_name, const int64_t* values, int num_values) { - op->attrs.Set(attr_name, - tensorflow::gtl::ArraySlice( - reinterpret_cast(values), num_values)); + op->operation.MutableAttrs()->Set( + attr_name, tensorflow::gtl::ArraySlice( + reinterpret_cast(values), num_values)); } void TFE_OpSetAttrTypeList(TFE_Op* op, const char* attr_name, const TF_DataType* values, int num_values) { - op->attrs.Set( + op->operation.MutableAttrs()->Set( attr_name, tensorflow::gtl::ArraySlice( reinterpret_cast(values), num_values)); @@ -378,8 +375,8 @@ void TFE_OpSetAttrBoolList(TFE_Op* op, const char* attr_name, for (int i = 0; i < num_values; ++i) { b[i] = values[i]; } - op->attrs.Set(attr_name, - tensorflow::gtl::ArraySlice(b.get(), num_values)); + op->operation.MutableAttrs()->Set( + attr_name, tensorflow::gtl::ArraySlice(b.get(), num_values)); } void TFE_OpSetAttrShapeList(TFE_Op* op, const char* attr_name, @@ -409,9 +406,9 @@ void TFE_OpSetAttrShapeList(TFE_Op* op, const char* attr_name, } } } - op->attrs.Set(attr_name, - tensorflow::gtl::ArraySlice( - proto.get(), num_values)); + op->operation.MutableAttrs()->Set( + attr_name, tensorflow::gtl::ArraySlice( + proto.get(), num_values)); } void TFE_OpSetAttrFunctionList(TFE_Op* op, const char* attr_name, @@ -419,12 +416,12 @@ void TFE_OpSetAttrFunctionList(TFE_Op* op, const char* attr_name, std::unique_ptr funcs( new tensorflow::NameAttrList[num_values]); for (int i = 0; i < num_values; i++) { - funcs[i].set_name(value[i]->name); - value[i]->attrs.FillAttrValueMap(funcs[i].mutable_attr()); + funcs[i].set_name(value[i]->operation.Name()); + value[i]->operation.Attrs().FillAttrValueMap(funcs[i].mutable_attr()); } - op->attrs.Set(attr_name, - tensorflow::gtl::ArraySlice( - funcs.get(), num_values)); + op->operation.MutableAttrs()->Set( + attr_name, tensorflow::gtl::ArraySlice( + funcs.get(), num_values)); } } // extern "C" @@ -460,18 +457,19 @@ int StepStatsDeviceIndex(tensorflow::StepStats* step_stats, } tensorflow::Status ValidateInputTypeAndPlacement( - tensorflow::EagerContext* ctx, tensorflow::Device* op_device, TFE_Op* op, - const tensorflow::OpKernel* kernel, tensorflow::RunMetadata* run_metadata) { + tensorflow::EagerContext* ctx, tensorflow::Device* op_device, + tensorflow::EagerOperation* op, const tensorflow::OpKernel* kernel, + tensorflow::RunMetadata* run_metadata) { tensorflow::Device* host_device = ctx->HostCPU(); const tensorflow::MemoryTypeVector& memtypes = kernel->input_memory_types(); - if (memtypes.size() != op->inputs.size()) { + if (memtypes.size() != op->Inputs().size()) { return tensorflow::errors::InvalidArgument( - "expected ", memtypes.size(), " inputs, got ", op->inputs.size()); + "expected ", memtypes.size(), " inputs, got ", op->Inputs().size()); } - for (int i = 0; i < op->inputs.size(); ++i) { + for (int i = 0; i < op->Inputs().size(); ++i) { const tensorflow::Device* expected_device = memtypes[i] == tensorflow::HOST_MEMORY ? host_device : op_device; - tensorflow::TensorHandle* handle = op->inputs[i]; + tensorflow::TensorHandle* handle = op->Inputs()[i]; tensorflow::Device* handle_device = nullptr; TF_RETURN_IF_ERROR(handle->Device(&handle_device)); const tensorflow::Device* actual_device = @@ -491,7 +489,7 @@ tensorflow::Status ValidateInputTypeAndPlacement( return tensorflow::errors::InvalidArgument( "Tensors on conflicting devices:" " cannot compute ", - op->name, " as input #", i, " was expected to be on ", + op->Name(), " as input #", i, " was expected to be on ", expected_device->name(), " but is actually on ", actual_device->name(), " (operation running on ", op_device->name(), ")", @@ -502,7 +500,7 @@ tensorflow::Status ValidateInputTypeAndPlacement( "between devices" " may slow down your model"); case tensorflow::DEVICE_PLACEMENT_WARN: - LOG(WARNING) << "before computing " << op->name << " input #" << i + LOG(WARNING) << "before computing " << op->Name() << " input #" << i << " was expected to be on " << expected_device->name() << " but is actually on " << actual_device->name() << " (operation running on " << op_device->name() @@ -534,16 +532,16 @@ tensorflow::Status ValidateInputTypeAndPlacement( if (copied_tensor != nullptr) copied_tensor->Unref(); return tensorflow::errors::Internal( "Failed copying input tensor from ", actual_device->name(), " to ", - expected_device->name(), " in order to run ", op->name, ": ", + expected_device->name(), " in order to run ", op->Name(), ": ", status.error_message()); } handle->Unref(); handle = copied_tensor; - op->inputs[i] = copied_tensor; + (*op->MutableInputs())[i] = copied_tensor; } if (handle->dtype != kernel->input_type(i)) { return tensorflow::errors::InvalidArgument( - "cannot compute ", op->name, " as input #", i, + "cannot compute ", op->Name(), " as input #", i, " was expected to be a ", tensorflow::DataTypeString(kernel->input_type(i)), " tensor but is a ", tensorflow::DataTypeString(handle->dtype), @@ -554,9 +552,10 @@ tensorflow::Status ValidateInputTypeAndPlacement( } tensorflow::Device* SelectDevice(const tensorflow::NodeDef& ndef, - TFE_Context* ctx, TF_Status* status) { + tensorflow::EagerContext* ctx, + TF_Status* status) { tensorflow::DeviceSet ds; - for (tensorflow::Device* d : *ctx->context.devices()) { + for (tensorflow::Device* d : *ctx->devices()) { ds.AddDevice(d); } tensorflow::DeviceTypeVector final_devices; @@ -570,7 +569,7 @@ tensorflow::Device* SelectDevice(const tensorflow::NodeDef& ndef, "Could not find valid device for node ", ndef.DebugString()); return nullptr; } - for (tensorflow::Device* d : *ctx->context.devices()) { + for (tensorflow::Device* d : *ctx->devices()) { if (d->device_type() == final_devices[0].type_string()) { return d; } @@ -599,15 +598,16 @@ const tensorflow::FunctionDef* OpToFunction( std::vector* arg_input_types, tensorflow::gtl::FlatMap* op_input_to_func_input, TF_Status* status) { - DCHECK(!op->is_function()); + DCHECK(!op->operation.is_function()); tensorflow::FunctionDef fdef; // Get the OpDef of the op we are trying to encapsulate. - TFE_Context* ctx = op->ctx; + TFE_Context* ctx = op->operation.ctx; const tensorflow::OpRegistrationData* op_data; { - status->status = ctx->context.FindFunctionOpData(op->name, &op_data); + status->status = + ctx->context.FindFunctionOpData(op->operation.Name(), &op_data); if (!status->status.ok()) { return nullptr; } @@ -618,7 +618,8 @@ const tensorflow::FunctionDef* OpToFunction( // Handle constant inputs. const std::unordered_set const_inputs( - *tensorflow::XlaOpRegistry::CompileTimeConstantInputs(op->name)); + *tensorflow::XlaOpRegistry::CompileTimeConstantInputs( + op->operation.Name())); // First add place holders for the input args, so that we can refer to them by // position in the next loop. Also tally up the resource inputs. @@ -644,7 +645,7 @@ const tensorflow::FunctionDef* OpToFunction( (*op_input_to_func_input)[i] = const_index; func_input_arg = signature->mutable_input_arg(const_index++); const_input_types->push_back( - static_cast(op->inputs[i]->dtype)); + static_cast(op->operation.Inputs()[i]->dtype)); } else if (op_input_arg.type() == tensorflow::DT_RESOURCE) { VLOG(1) << "For resource input, mapping op input " << i << " to func input " << resource_index; @@ -656,11 +657,11 @@ const tensorflow::FunctionDef* OpToFunction( (*op_input_to_func_input)[i] = arg_index; func_input_arg = signature->mutable_input_arg(arg_index++); arg_input_types->push_back( - static_cast(op->inputs[i]->dtype)); + static_cast(op->operation.Inputs()[i]->dtype)); } func_input_arg->set_name(op_input_arg.name()); - func_input_arg->set_type(op->inputs[i]->dtype); + func_input_arg->set_type(op->operation.Inputs()[i]->dtype); } VLOG(1) << "Added OpDef Inputs: " << fdef.DebugString(); @@ -673,7 +674,8 @@ const tensorflow::FunctionDef* OpToFunction( op_def.name(), func_id_generator.fetch_add(1))); // Add the node def and set its input names to match op_def's names. - const tensorflow::NodeDef& ndef = op->attrs.BuildNodeDef(); + const tensorflow::NodeDef& ndef = + op->operation.MutableAttrs()->BuildNodeDef(); DCHECK_EQ(signature->input_arg_size(), ndef.input_size()); *fdef.add_node_def() = ndef; for (int i = 0; i < op_def.input_arg_size(); ++i) { @@ -713,17 +715,18 @@ const tensorflow::FunctionDef* OpToFunction( // Builds an _XLALaunchOp as a wrapper over 'op', so that 'op' can be executed // via XLA. std::unique_ptr BuildXlaLaunch(TFE_Op* op, TF_Status* status) { - VLOG(1) << "Creating _XlaLaunchOp for TFE_Op " << op->name; - auto launch_op = - std::unique_ptr(TFE_NewOp(op->ctx, "_XlaLaunch", status)); + VLOG(1) << "Creating _XlaLaunchOp for TFE_Op " << op->operation.Name(); + auto launch_op = std::unique_ptr( + TFE_NewOp(op->operation.ctx, "_XlaLaunch", status)); if (TF_GetCode(status) != TF_OK) return nullptr; - if (op->device) { - TFE_OpSetDevice(launch_op.get(), op->device->name().c_str(), status); + if (op->operation.device) { + TFE_OpSetDevice(launch_op.get(), op->operation.device->name().c_str(), + status); if (TF_GetCode(status) != TF_OK) return nullptr; } const tensorflow::FunctionDef* fdef; - { fdef = op->ctx->context.FindFunctionDef(op->name); } + { fdef = op->operation.ctx->FindFunctionDef(op->operation.Name()); } std::vector const_input_types; std::vector arg_input_types; tensorflow::gtl::FlatMap op_input_to_func_input; @@ -748,20 +751,21 @@ std::unique_ptr BuildXlaLaunch(TFE_Op* op, TF_Status* status) { // Copy inputs and their devices. // Since input param reordering may have occurred between `op` and `launch_op` // via `op_input_to_func_input`, adjust the actual inputs accordingly. - launch_op->inputs = op->inputs; - for (tensorflow::TensorHandle* h : launch_op->inputs) { + *launch_op->operation.MutableInputs() = op->operation.Inputs(); + for (tensorflow::TensorHandle* h : launch_op->operation.Inputs()) { h->Ref(); } if (!op_input_to_func_input.empty()) { - DCHECK_EQ(op->inputs.size(), op_input_to_func_input.size()); + DCHECK_EQ(op->operation.Inputs().size(), op_input_to_func_input.size()); for (int i = 0; i < op_input_to_func_input.size(); ++i) { VLOG(1) << "mapping op input " << i << " to func input " << op_input_to_func_input[i]; - launch_op->inputs[op_input_to_func_input[i]] = op->inputs[i]; + (*launch_op->operation.MuableInputs())[op_input_to_func_input[i]] = + op->operation.Inputs()[i]; } } - launch_op->attrs.NumInputs(op->inputs.size()); + launch_op->operation.MutableAttrs()->NumInputs(op->operation.Inputs().size()); TFE_OpSetAttrTypeList(launch_op.get(), "Tconstants", const_input_types.data(), const_input_types.size()); @@ -796,16 +800,17 @@ std::unique_ptr BuildXlaLaunch(TFE_Op* op, TF_Status* status) { extern "C" { -void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals, +void TFE_Execute(TFE_Op* tfe_op, TFE_TensorHandle** retvals, int* num_retvals, TF_Status* status) { - TFE_Context* ctx = op->ctx; - status->status = ctx->context.GetStatus(); + tensorflow::EagerOperation* op = &tfe_op->operation; + tensorflow::EagerContext* ctx = op->EagerContext(); + status->status = ctx->GetStatus(); if (!status->status.ok()) { return; } #ifdef TENSORFLOW_EAGER_USE_XLA std::unique_ptr xla_launch_op; - if (op->use_xla && op->name != "_XlaLaunch") { + if (op->UseXla() && op->Name() != "_XlaLaunch") { xla_launch_op = BuildXlaLaunch(op, status); if (!status->status.ok()) { return; @@ -816,31 +821,31 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals, // Ensure all resource-touching ops run in the device the resource is, // regardless of anything else that has been specified. This is identical to // the graph mode behavior. - for (int i = 0; i < op->inputs.size(); ++i) { + for (int i = 0; i < op->Inputs().size(); ++i) { tensorflow::Device* input_op_device = nullptr; - status->status = op->inputs[i]->OpDevice(&input_op_device); + status->status = op->Inputs()[i]->OpDevice(&input_op_device); if (!status->status.ok()) return; - VLOG(2) << "for op " << op->name << " input " << i << " " - << tensorflow::DataTypeString(op->inputs[i]->dtype) << " " + VLOG(2) << "for op " << op->Name() << " input " << i << " " + << tensorflow::DataTypeString(op->Inputs()[i]->dtype) << " " << (input_op_device == nullptr ? "cpu" : input_op_device->name()) - << " " << (op->device == nullptr ? "cpu" : op->device->name()); - if (op->inputs[i]->dtype == tensorflow::DT_RESOURCE && - (input_op_device != op->device || input_op_device == nullptr)) { + << " " << (op->Device() == nullptr ? "cpu" : op->Device()->name()); + if (op->Inputs()[i]->dtype == tensorflow::DT_RESOURCE && + (input_op_device != op->Device() || input_op_device == nullptr)) { tensorflow::Device* d = - input_op_device == nullptr ? ctx->context.HostCPU() : input_op_device; - VLOG(1) << "Changing device of operation " << op->name << " to " + input_op_device == nullptr ? ctx->HostCPU() : input_op_device; + VLOG(1) << "Changing device of operation " << op->Name() << " to " << d->name() << " because input #" << i << " is a resource in this device."; - op->device = d; + op->SetDevice(d); } } - tensorflow::Device* device = op->device; + tensorflow::Device* device = op->Device(); - tensorflow::Fprint128 cache_key = - op->attrs.CacheKey(device == nullptr ? "unspecified" : device->name()); - tensorflow::KernelAndDevice* kernel = ctx->context.GetCachedKernel(cache_key); + tensorflow::Fprint128 cache_key = op->MutableAttrs()->CacheKey( + device == nullptr ? "unspecified" : device->name()); + tensorflow::KernelAndDevice* kernel = ctx->GetCachedKernel(cache_key); if (kernel == nullptr) { - const tensorflow::NodeDef& ndef = op->attrs.BuildNodeDef(); + const tensorflow::NodeDef& ndef = op->MutableAttrs()->BuildNodeDef(); if (device == nullptr) { device = SelectDevice(ndef, ctx, status); if (!status->status.ok()) { @@ -848,19 +853,19 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals, } } CHECK(device != nullptr); - if (ctx->context.LogDevicePlacement()) { + if (ctx->LogDevicePlacement()) { LOG(INFO) << "Executing op " << ndef.op() << " in device " << device->name(); } - kernel = new tensorflow::KernelAndDevice(ctx->context.GetRendezvous()); + kernel = new tensorflow::KernelAndDevice(ctx->GetRendezvous()); // Knowledge of the implementation of Init (and in-turn // FunctionLibraryRuntime::CreateKernel) tells us that ctx->func_lib_def // will be accessed, so grab on to the lock. // See WARNING comment in Execute (before kernel->Run) - would be nice to // rework to avoid this subtlety. - tensorflow::tf_shared_lock l(*ctx->context.FunctionsMu()); - status->status = tensorflow::KernelAndDevice::Init( - ndef, ctx->context.func_lib(device), kernel); + tensorflow::tf_shared_lock l(*ctx->FunctionsMu()); + status->status = + tensorflow::KernelAndDevice::Init(ndef, ctx->func_lib(device), kernel); if (!status->status.ok()) { delete kernel; return; @@ -868,7 +873,7 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals, // Update output_dtypes inside `kernel`. const tensorflow::OpDef* op_def = nullptr; const tensorflow::FunctionDef* function_def = - ctx->context.FuncLibDef()->Find(ndef.op()); + ctx->FuncLibDef()->Find(ndef.op()); if (function_def != nullptr) { op_def = &(function_def->signature()); } @@ -884,7 +889,7 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals, if (!status->status.ok()) { return; } - ctx->context.AddKernelToCache(cache_key, kernel); + ctx->AddKernelToCache(cache_key, kernel); } const tensorflow::DataTypeVector& output_dtypes = kernel->output_dtypes(); const int output_dtypes_size = output_dtypes.size(); @@ -903,43 +908,42 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals, device = kernel->device(); } status->status = ValidateInputTypeAndPlacement( - &ctx->context, device, op, kernel->kernel(), - ctx->context.ShouldStoreMetadata() ? ctx->context.RunMetadataProto() - : nullptr); + ctx, device, op, kernel->kernel(), + ctx->ShouldStoreMetadata() ? ctx->RunMetadataProto() : nullptr); if (!status->status.ok()) return; std::unique_ptr maybe_stats; - if (ctx->context.ShouldStoreMetadata()) { + if (ctx->ShouldStoreMetadata()) { maybe_stats.reset(new tensorflow::NodeExecStats); - maybe_stats->set_node_name(op->name); + maybe_stats->set_node_name(op->Name()); maybe_stats->set_all_start_micros(tensorflow::Env::Default()->NowMicros()); maybe_stats->set_op_start_rel_micros(0); maybe_stats->set_scheduled_micros(tensorflow::Env::Default()->NowMicros()); // TODO(apassos) track referenced tensors } - if (ctx->context.Async()) { + if (ctx->Async()) { // Note that for async mode, execution order will make sure that all // input handles are ready before executing them. // TODO(agarwal): Consider executing "cheap" kernels inline for performance. tensorflow::gtl::InlinedVector handle_retvals( *num_retvals); - tensorflow::uint64 id = op->ctx->context.NextId(); + tensorflow::uint64 id = ctx->NextId(); for (int i = 0; i < *num_retvals; ++i) { tensorflow::TensorHandle* h = - new tensorflow::TensorHandle(id, output_dtypes[i], &op->ctx->context); + new tensorflow::TensorHandle(id, output_dtypes[i], ctx); retvals[i] = new TFE_TensorHandle(h); handle_retvals[i] = h; } tensorflow::EagerNode* node = new tensorflow::ExecuteNode( - id, &op->ctx->context, op->device, op->inputs, kernel, - maybe_stats.release(), output_dtypes, handle_retvals); - ctx->context.ExecutorAdd(node); + id, ctx, op->Device(), op->Inputs(), kernel, maybe_stats.release(), + output_dtypes, handle_retvals); + ctx->ExecutorAdd(node); } else { // Execute checks if retvals[i] is nullptr or not to figure if it needs to // allocate it. tensorflow::gtl::InlinedVector handle_retvals( *num_retvals); status->status = tensorflow::EagerExecute( - &op->ctx->context, op->device, op->inputs, kernel, maybe_stats.get(), + ctx, op->Device(), op->Inputs(), kernel, maybe_stats.get(), handle_retvals.data(), *num_retvals); for (int i = 0; i < *num_retvals; ++i) { retvals[i] = new TFE_TensorHandle(handle_retvals[i]); @@ -1142,9 +1146,3 @@ void SetOpAttrValueScalar(TFE_Context* ctx, TFE_Op* op, } } } // namespace tensorflow - -TFE_Op::~TFE_Op() { - for (tensorflow::TensorHandle* h : inputs) { - h->Unref(); - } -} diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h index 05dc64f521..49e1aab1ce 100644 --- a/tensorflow/c/eager/c_api_internal.h +++ b/tensorflow/c/eager/c_api_internal.h @@ -32,6 +32,7 @@ limitations under the License. #include "tensorflow/core/common_runtime/device_factory.h" #include "tensorflow/core/common_runtime/eager/context.h" #include "tensorflow/core/common_runtime/eager/eager_executor.h" +#include "tensorflow/core/common_runtime/eager/eager_operation.h" #include "tensorflow/core/common_runtime/eager/kernel_and_device.h" #include "tensorflow/core/common_runtime/eager/tensor_handle.h" #include "tensorflow/core/common_runtime/function.h" @@ -45,7 +46,6 @@ limitations under the License. #include "tensorflow/core/platform/thread_annotations.h" #include "tensorflow/core/public/version.h" - struct TFE_ContextOptions { TF_SessionOptions session_options; // true if async execution is enabled. @@ -85,19 +85,9 @@ struct TFE_Op { // t is NULL iff the TFE_Op corresponds to a TensorFlow function instead of a // primitive operation. TFE_Op(TFE_Context* ctx, const char* op, const tensorflow::AttrTypeMap* t) - : ctx(ctx), name(op), attrs(op), attr_types(t), device(nullptr) {} - - ~TFE_Op(); - - bool const is_function() const { return attr_types == nullptr; } + : operation(&ctx->context, op, t) {} - TFE_Context* ctx; // Must outlive the TFE_Op. - const tensorflow::string name; - tensorflow::AttrBuilder attrs; - const tensorflow::AttrTypeMap* attr_types; - tensorflow::gtl::InlinedVector inputs; - tensorflow::Device* device; - bool use_xla = false; + tensorflow::EagerOperation operation; }; namespace tensorflow { diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD index 941a0e61c7..00ac4a4e47 100644 --- a/tensorflow/core/common_runtime/eager/BUILD +++ b/tensorflow/core/common_runtime/eager/BUILD @@ -54,6 +54,22 @@ tf_cuda_library( ], ) +tf_cuda_library( + name = "eager_operation", + srcs = [ + "eager_operation.cc", + ], + hdrs = [ + "eager_operation.h", + ], + visibility = ["//tensorflow:internal"], + deps = [ + ":context", + ":tensor_handle", + "//tensorflow/c/eager:runtime", + ], +) + tf_cuda_library( name = "tensor_handle", srcs = [ diff --git a/tensorflow/core/common_runtime/eager/eager_operation.cc b/tensorflow/core/common_runtime/eager/eager_operation.cc new file mode 100644 index 0000000000..381b05ada8 --- /dev/null +++ b/tensorflow/core/common_runtime/eager/eager_operation.cc @@ -0,0 +1,33 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/core/common_runtime/eager/eager_operation.h" + +namespace tensorflow { +tensorflow::Status EagerOperation::SetDevice(const char* device) { + auto status = Status::OK(); + tensorflow::Device* d = nullptr; + if (device != nullptr && strlen(device) > 0) { + status.Update(ctx_->FindDeviceByName(device, &d)); + } + device_ = d; + return status; +} + +void EagerOperation::AddInput(tensorflow::TensorHandle* h) { + h->Ref(); + inputs_.push_back(h); + attrs_.NumInputs(static_cast(inputs_.size())); +} +} // namespace tensorflow diff --git a/tensorflow/core/common_runtime/eager/eager_operation.h b/tensorflow/core/common_runtime/eager/eager_operation.h new file mode 100644 index 0000000000..6b6e53da87 --- /dev/null +++ b/tensorflow/core/common_runtime/eager/eager_operation.h @@ -0,0 +1,74 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_EAGER_OPERATION_H_ +#define TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_EAGER_OPERATION_H_ + +#include "tensorflow/c/eager/runtime.h" +#include "tensorflow/core/common_runtime/eager/context.h" +#include "tensorflow/core/common_runtime/eager/tensor_handle.h" + +namespace tensorflow { +class EagerOperation { + public: + // t is NULL iff the EagerOperation corresponds to a TensorFlow function + // instead of a primitive operation. + EagerOperation(tensorflow::EagerContext* ctx, const char* op, + const tensorflow::AttrTypeMap* t) + : ctx_(ctx), name_(op), attrs_(op), attr_types_(t), device_(nullptr) {} + + ~EagerOperation() { + for (tensorflow::TensorHandle* h : inputs_) { + h->Unref(); + } + } + + bool is_function() const { return attr_types_ == nullptr; } + + tensorflow::EagerContext* EagerContext() { return ctx_; } + + tensorflow::AttrBuilder* MutableAttrs() { return &attrs_; } + const tensorflow::AttrBuilder& Attrs() const { return attrs_; } + + const tensorflow::gtl::InlinedVector& Inputs() + const { + return inputs_; + } + tensorflow::gtl::InlinedVector* + MutableInputs() { + return &inputs_; + } + void AddInput(tensorflow::TensorHandle* h); + + const tensorflow::string& Name() const { return name_; } + const tensorflow::AttrTypeMap* AttrTypes() const { return attr_types_; } + + tensorflow::Device* Device() const { return device_; } + tensorflow::Status SetDevice(const char* device); + void SetDevice(tensorflow::Device* device) { device_ = device; } + + void SetUseXla(bool use_xla) { use_xla_ = use_xla; } + + private: + tensorflow::EagerContext* ctx_; // Must outlive the EagerOperation. + const tensorflow::string name_; + tensorflow::AttrBuilder attrs_; + const tensorflow::AttrTypeMap* attr_types_; + tensorflow::gtl::InlinedVector inputs_; + tensorflow::Device* device_; + bool use_xla_ = false; +}; +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_EAGER_OPERATION_H_ -- GitLab From 2b0b015ebb1c33a409836bd1c9c98124dfd841ec Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Apr 2018 11:43:48 -0700 Subject: [PATCH 747/791] [XLA] Fix a bug in ToProto: don't add gather attributes twice. PiperOrigin-RevId: 193699745 --- tensorflow/compiler/xla/service/hlo_instruction.cc | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index a638d54d85..a714d0e114 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -2451,12 +2451,6 @@ HloInstructionProto HloInstruction::ToProto() const { proto.add_fft_length(fft_len); } - if (gather_dimension_numbers_ != nullptr) { - *proto.mutable_gather_dimension_numbers() = *gather_dimension_numbers_; - } - for (int64 bound : gather_window_bounds_) { - proto.add_gather_window_bounds(bound); - } proto.set_channel_name(channel_name_); proto.set_cost_estimate_ns(cost_estimate_ns_); -- GitLab From 0074dffd076e0faf4da5913aebfa594ef925d6c7 Mon Sep 17 00:00:00 2001 From: Anna R Date: Fri, 20 Apr 2018 12:01:21 -0700 Subject: [PATCH 748/791] Prefix compat import with underscore in meta_graph_transform.py so that it doesn't get exported as part of API: https://www.tensorflow.org/versions/r1.8/api_docs/python/tf/contrib/meta_graph_transform/meta_graph_transform PiperOrigin-RevId: 193702570 --- .../meta_graph_transform/meta_graph_transform.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorflow/contrib/meta_graph_transform/meta_graph_transform.py b/tensorflow/contrib/meta_graph_transform/meta_graph_transform.py index ff88b4fa84..4090c1ff3e 100644 --- a/tensorflow/contrib/meta_graph_transform/meta_graph_transform.py +++ b/tensorflow/contrib/meta_graph_transform/meta_graph_transform.py @@ -30,7 +30,7 @@ from tensorflow.python.framework import importer as _importer from tensorflow.python.framework import ops as _ops from tensorflow.python.saved_model import constants as _saved_model_constants from tensorflow.python.training import saver as _saver_lib -from tensorflow.python.util import compat +from tensorflow.python.util import compat as _compat from tensorflow.tools import graph_transforms as _graph_transforms @@ -161,7 +161,7 @@ def _clean_save_and_restore(graph_def, op, removed_op_names): shapes = [] dtypes = [] for index, value in enumerate(name_op_value_tensor.string_val): - if not _is_removed(compat.as_str(value), removed_op_names): + if not _is_removed(_compat.as_str(value), removed_op_names): names.append(value) shapes.append(shape_op_value_tensor.string_val[index]) dtypes.append(op.attr['dtypes'].list.type[index]) @@ -651,7 +651,7 @@ def _is_removed_mentioned(s, removed_op_names): # /foo/bar. This regex ensures that we handle these two nodes # as separate entities. It matches on nodes having names in the form of # '/foo/bar_x' as well as nodes having names in the form of 'foo.' - s_names = _re.findall(r'((?:[\/]?[a-zA-Z0-9\_]*)*)', compat.as_str_any(s)) + s_names = _re.findall(r'((?:[\/]?[a-zA-Z0-9\_]*)*)', _compat.as_str_any(s)) for removed_op_name in removed_op_names: for s_name in s_names: if s_name.endswith(removed_op_name): @@ -737,9 +737,9 @@ def meta_graph_transform( for tag in tags: meta_graph_def.meta_info_def.tags.append(tag) - base_op_names = [compat.as_str(node.name) + base_op_names = [_compat.as_str(node.name) for node in base_meta_graph_def.graph_def.node] - retained_op_names = [compat.as_str(node.name) + retained_op_names = [_compat.as_str(node.name) for node in meta_graph_def.graph_def.node] removed_op_names = set(base_op_names) - set(retained_op_names) -- GitLab From 1b5839e6acad5d360ea9e5b94226b30047924cb9 Mon Sep 17 00:00:00 2001 From: Dimitris Vardoulakis Date: Fri, 20 Apr 2018 12:02:56 -0700 Subject: [PATCH 749/791] [TF:XLA] Now that the compiler no longer introduces implicit broadcasts, forbid them in the HLO verifier. PiperOrigin-RevId: 193702874 --- tensorflow/compiler/xla/service/BUILD | 1 + .../compiler/xla/service/hlo_verifier.cc | 21 ++++++++ .../compiler/xla/service/hlo_verifier.h | 4 ++ .../xla/service/reshape_mover_test.cc | 51 ------------------- 4 files changed, 26 insertions(+), 51 deletions(-) diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index 9009cbf845..9555d91817 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -2032,6 +2032,7 @@ cc_library( srcs = ["hlo_verifier.cc"], hdrs = ["hlo_verifier.h"], deps = [ + ":hlo", ":hlo_pass", ":shape_inference", "//tensorflow/compiler/xla:status_macros", diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc index 80ed6d6832..8a30cbf9cd 100644 --- a/tensorflow/compiler/xla/service/hlo_verifier.cc +++ b/tensorflow/compiler/xla/service/hlo_verifier.cc @@ -15,6 +15,7 @@ limitations under the License. #include +#include "tensorflow/compiler/xla/service/hlo_opcode.h" #include "tensorflow/compiler/xla/service/hlo_verifier.h" #include "tensorflow/compiler/xla/status_macros.h" #include "tensorflow/core/lib/core/errors.h" @@ -780,6 +781,24 @@ Status HloVerifier::CheckWhileInstruction(HloInstruction* instruction) { return tensorflow::Status::OK(); } +Status HloVerifier::CheckElementwiseInstruction(HloInstruction* instruction) { + const Shape& out_shape = instruction->shape(); + for (HloInstruction* operand : instruction->operands()) { + const Shape& operand_shape = operand->shape(); + if (!ShapeUtil::IsScalar(operand_shape) && + !ShapeUtil::CompatibleIgnoringElementType(operand_shape, out_shape)) { + return FailedPrecondition( + "Implicit broadcast is not allowed in HLO." + "Found non-compatible shapes for instruction %s.\n" + "output: %s\noperand: %s\n", + HloOpcodeString(instruction->opcode()).c_str(), + ShapeUtil::HumanString(out_shape).c_str(), + ShapeUtil::HumanString(operand_shape).c_str()); + } + } + return tensorflow::Status::OK(); +} + StatusOr HloVerifier::Run(HloModule* module) { TF_RETURN_IF_ERROR(VerifyHloStructure(module)); @@ -821,6 +840,8 @@ StatusOr HloVerifier::Run(HloModule* module) { << " != " << ShapeUtil::Rank(instruction->operand(0)->shape()); } else if (instruction->opcode() == HloOpcode::kWhile) { TF_RETURN_IF_ERROR(CheckWhileInstruction(instruction)); + } else if (instruction->IsElementwise()) { + TF_RETURN_IF_ERROR(CheckElementwiseInstruction(instruction)); } auto previous = instructions.find(instruction->name()); diff --git a/tensorflow/compiler/xla/service/hlo_verifier.h b/tensorflow/compiler/xla/service/hlo_verifier.h index 1ec55a9bdc..6208887547 100644 --- a/tensorflow/compiler/xla/service/hlo_verifier.h +++ b/tensorflow/compiler/xla/service/hlo_verifier.h @@ -146,6 +146,10 @@ class HloVerifier : public HloPassInterface { Status CheckWhileInstruction(HloInstruction* instruction); + // Checks that the non-scalar operand shapes are compatible to the output + // shape, i.e., that there are no implicit broadcasts of size-one dimensions. + Status CheckElementwiseInstruction(HloInstruction* instruction); + // Creates a ShapeVerifier that checks that shapes match inferred // expectations. This is a factory function because ShapeVerifier, // being a DfsHloVisitor, is stateful. We want a clean object diff --git a/tensorflow/compiler/xla/service/reshape_mover_test.cc b/tensorflow/compiler/xla/service/reshape_mover_test.cc index 094f7319f4..13e2d3258e 100644 --- a/tensorflow/compiler/xla/service/reshape_mover_test.cc +++ b/tensorflow/compiler/xla/service/reshape_mover_test.cc @@ -458,57 +458,6 @@ TEST_F(ReshapeMoverTest, ScalarReshapeNotMovedAcrossSelect) { EXPECT_EQ(select, computation->root_instruction()); } -// Tree looks like: -// -// param0 [1,128,1] -// | -// reshape [128,1] constant [128,1024] -// \ / -// multiply w/implicit broadcast [128,1024] -// -// The reshape mover would like to sink the reshape below the multiply. -// -// Previously we would attempt to insert a reshape of the constant to [1,128,1] -// (which is unsound, because it has a different number of elements) as -// preparation for sinking the reshape. -// -// To eliminate the unsoundness, we outlaw reshape sinking when one of the -// operands is implicitly broadcast in the elementwise consumer. -// -// TODO(b/37799338) However, it would be possible in this case to do a more -// in-depth analysis to get reshape movement to occur: -// -// 1. Note that the broadcast dimension (logical dimension 1) in the operands -// would map back to logical dimension 2 in the param0 node. -// 2. Match rank of the constant to the param0 node (by prepending a trivial 1 -// dimension). -// 3. Reshape to [128,1024] at the root. -// -// But this is not currently done. -TEST_F(ReshapeMoverTest, ImplicitlyBroadcastReshapeIsNotMovedBug37787999) { - HloComputation::Builder builder(TestName()); - auto param0 = builder.AddInstruction(HloInstruction::CreateParameter( - 0, ShapeUtil::MakeShape(F32, {1, 128, 1}), "param0")); - auto reshape = builder.AddInstruction(HloInstruction::CreateReshape( - ShapeUtil::MakeShape(F32, {128, 1}), param0)); - Array2D a(128, 1024); - auto literal = Literal::CreateR2FromArray2D(a); - auto constant = builder.AddInstruction( - HloInstruction::CreateConstant(std::move(literal))); - auto multiply = builder.AddInstruction(HloInstruction::CreateBinary( - constant->shape(), HloOpcode::kMultiply, constant, reshape)); - - auto computation = module().AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), - op::Multiply(op::Constant(), op::Reshape(param0))); - - EXPECT_FALSE(ReshapeMover().Run(&module()).ValueOrDie()); - - EXPECT_THAT(computation->root_instruction(), - op::Multiply(op::Constant(), op::Reshape(param0))); - EXPECT_EQ(multiply, computation->root_instruction()); -} - // Tree looks like this: // // add1 -- GitLab From ceed923d600584ade8d159271422b4a08f728cbb Mon Sep 17 00:00:00 2001 From: Yangzihao Wang Date: Fri, 20 Apr 2018 12:05:11 -0700 Subject: [PATCH 750/791] Add native dilated support for conv3d and its gradients in cudnn v>=6. PiperOrigin-RevId: 193703316 --- tensorflow/core/framework/common_shape_fns.cc | 32 ++- .../core/framework/common_shape_fns_test.cc | 55 ++++- tensorflow/core/kernels/conv_grad_ops_3d.cc | 115 +++++++++- tensorflow/core/kernels/conv_ops_3d.cc | 52 ++++- tensorflow/core/ops/nn_ops.cc | 2 + .../python/kernel_tests/conv_ops_3d_test.py | 196 +++++++++++++++++- tensorflow/python/ops/nn_grad.py | 6 + 7 files changed, 426 insertions(+), 32 deletions(-) diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc index 72eeda7a43..0916c9b7a8 100644 --- a/tensorflow/core/framework/common_shape_fns.cc +++ b/tensorflow/core/framework/common_shape_fns.cc @@ -487,6 +487,15 @@ Status Conv3DShape(shape_inference::InferenceContext* c) { string data_format; Status s = c->GetAttr("data_format", &data_format); + std::vector dilations; + TF_RETURN_IF_ERROR(c->GetAttr("dilations", &dilations)); + + if (dilations.size() != 5) { + return errors::InvalidArgument( + "Conv3D requires the dilation attribute to contain 5 values, but got: ", + dilations.size()); + } + std::vector strides; TF_RETURN_IF_ERROR(c->GetAttr("strides", &strides)); if (strides.size() != 5) { @@ -496,6 +505,7 @@ Status Conv3DShape(shape_inference::InferenceContext* c) { } int32 stride_planes, stride_rows, stride_cols; + int32 dilation_planes, dilation_rows, dilation_cols; if (s.ok() && data_format == "NCDHW") { // Convert input_shape to NDHWC. auto dim = [&](char dimension) { @@ -506,10 +516,16 @@ Status Conv3DShape(shape_inference::InferenceContext* c) { stride_planes = strides[2]; stride_rows = strides[3]; stride_cols = strides[4]; + dilation_planes = dilations[2]; + dilation_cols = dilations[3]; + dilation_rows = dilations[4]; } else { stride_planes = strides[1]; stride_rows = strides[2]; stride_cols = strides[3]; + dilation_planes = dilations[1]; + dilation_cols = dilations[2]; + dilation_rows = dilations[3]; } DimensionHandle batch_size_dim = c->Dim(input_shape, 0); @@ -530,13 +546,15 @@ Status Conv3DShape(shape_inference::InferenceContext* c) { TF_RETURN_IF_ERROR(c->GetAttr("padding", &padding)); DimensionHandle output_planes, output_rows, output_cols; - TF_RETURN_IF_ERROR( - GetWindowedOutputSizeFromDims(c, in_planes_dim, filter_planes_dim, - stride_planes, padding, &output_planes)); - TF_RETURN_IF_ERROR(GetWindowedOutputSizeFromDims( - c, in_rows_dim, filter_rows_dim, stride_rows, padding, &output_rows)); - TF_RETURN_IF_ERROR(GetWindowedOutputSizeFromDims( - c, in_cols_dim, filter_cols_dim, stride_cols, padding, &output_cols)); + TF_RETURN_IF_ERROR(GetWindowedOutputSizeFromDimsV2( + c, in_planes_dim, filter_planes_dim, dilation_planes, stride_planes, + padding, &output_planes)); + TF_RETURN_IF_ERROR(GetWindowedOutputSizeFromDimsV2( + c, in_rows_dim, filter_rows_dim, dilation_rows, stride_rows, padding, + &output_rows)); + TF_RETURN_IF_ERROR(GetWindowedOutputSizeFromDimsV2( + c, in_cols_dim, filter_cols_dim, dilation_cols, stride_cols, padding, + &output_cols)); ShapeHandle output_shape; if (data_format == "NCDHW") { diff --git a/tensorflow/core/framework/common_shape_fns_test.cc b/tensorflow/core/framework/common_shape_fns_test.cc index 13d429b895..919e0967c0 100644 --- a/tensorflow/core/framework/common_shape_fns_test.cc +++ b/tensorflow/core/framework/common_shape_fns_test.cc @@ -644,15 +644,19 @@ TEST(CommonShapeFnsTest, Conv3DShapeTest) { .Finalize(&op.node_def)); }; - // 1x1x1 filter - set_op({{1, 1, 1, 1, 1}}, "VALID"); - INFER_OK(op, "[1,2,2,2,1];[1,1,1,1,1]", "[d0_0,2,2,2,d1_4]"); - // Invalid rank for input INFER_ERROR("must be rank 5", op, "[4,4];[2,1,1,1]"); // Invalid rank for filter INFER_ERROR("must be rank 5", op, "[1,4,4,1];[2,1,1]"); + // Invalid value for strides + set_op({{1, 1, 1, 0, 1}}, "VALID"); + INFER_ERROR("must be > 0", op, "[1,2,2,2,1];[1,1,1,1,1]"); + + // 1x1x1 filter + set_op({{1, 1, 1, 1, 1}}, "VALID"); + INFER_OK(op, "[1,2,2,2,1];[1,1,1,1,1]", "[d0_0,2,2,2,d1_4]"); + // unknown dims in the critical fields give partial inference. INFER_OK(op, "[1,2,2,2,1];[1,1,1,1,1]", "[d0_0,2,2,2,d1_4]"); INFER_OK(op, "[1,?,2,2,1];[1,1,1,1,1]", "[d0_0,?,2,2,d1_4]"); @@ -712,6 +716,49 @@ TEST(CommonShapeFnsTest, Conv3DShapeTest) { INFER_OK(op, "[1,4,9,4,1];[2,2,2,1,?]", "[d0_0,2,3,1,d1_4]"); } +TEST(CommonShapeFnsTest, Conv3DDilatedShapeTest) { + ShapeInferenceTestOp op("Conv3D"); + auto set_op = [&op](const std::vector& dilations, + const std::vector& strides, + const string& padding) { + TF_CHECK_OK(NodeDefBuilder("test", "Conv3D") + .Input("input", 0, DT_FLOAT) + .Input("filter", 0, DT_FLOAT) + .Attr("dilations", dilations) + .Attr("strides", strides) + .Attr("padding", padding) + .Finalize(&op.node_def)); + }; + + // Invalid rank for dilation + set_op({{1, 2, 1, 1}}, {{1, 1, 1, 1, 1}}, "VALID"); + INFER_ERROR("contain 5 values", op, "[1,2,2,2,1];[1,1,1,1,1]"); + + // Invalid value for dilation + set_op({{1, 2, 0, 1, 1}}, {{1, 1, 1, 1, 1}}, "VALID"); + INFER_ERROR("must be >= 1", op, "[1,2,2,2,1];[1,1,1,1,1]"); + + // 2x1x1 dilation 1x1x1 filter + set_op({{1, 2, 1, 1, 1}}, {{1, 1, 1, 1, 1}}, "VALID"); + INFER_OK(op, "[1,2,2,2,1];[1,1,1,1,1]", "[d0_0,2,2,2,d1_4]"); + + // 2x1x1 dilation 2x2x2 filter + set_op({{1, 2, 1, 1, 1}}, {{1, 1, 1, 1, 1}}, "VALID"); + INFER_OK(op, "[1,3,2,2,1];[2,2,2,1,1]", "[d0_0,1,1,1,d1_4]"); + + // 2x1x1 dilation 3x3x3 input, 1x1x1 filter, 2x2x2 stride + set_op({{1, 2, 1, 1, 1}}, {{1, 2, 2, 2, 1}}, "VALID"); + INFER_OK(op, "[1,3,3,3,1];[1,1,1,1,1]", "[d0_0,2,2,2,d1_4]"); + + // 2x1x1 dilation 3x3x3 input, 1x1x1 filter, 2x1x1 stride + set_op({{1, 2, 1, 1, 1}}, {{1, 2, 1, 1, 1}}, "VALID"); + INFER_OK(op, "[1,3,3,3,1];[1,1,1,1,1]", "[d0_0,2,3,3,d1_4]"); + + // 2x1x1 dilation 4x4x4 input, 2x2x2 filter, 1x1x1 stride + set_op({{1, 2, 1, 1, 1}}, {{1, 1, 1, 1, 1}}, "SAME"); + INFER_OK(op, "[1,4,4,4,1];[2,2,2,1,1]", "[d0_0,d0_1,d0_2,d0_3,d1_4]"); +} + TEST(CommonShapeFnsTest, DepthwiseConv2DShapeTest) { ShapeInferenceTestOp op("DepthwiseConv2dNative"); std::vector strides = {{1, 1, 1, 1}}; diff --git a/tensorflow/core/kernels/conv_grad_ops_3d.cc b/tensorflow/core/kernels/conv_grad_ops_3d.cc index 1234997bc5..092e859a5b 100644 --- a/tensorflow/core/kernels/conv_grad_ops_3d.cc +++ b/tensorflow/core/kernels/conv_grad_ops_3d.cc @@ -79,13 +79,18 @@ typedef Eigen::GpuDevice GPUDevice; context, out_depth == GetTensorDim(out_backprop, data_format_, 'C'), \ errors::InvalidArgument( \ label, ": filter and out_backprop must have the same out_depth")); \ + const std::array dilations = { \ + {GetTensorDim(dilation_, data_format_, '0'), \ + GetTensorDim(dilation_, data_format_, '1'), \ + GetTensorDim(dilation_, data_format_, '2')}}; \ const std::array strides = { \ {GetTensorDim(stride_, data_format_, '0'), \ GetTensorDim(stride_, data_format_, '1'), \ GetTensorDim(stride_, data_format_, '2')}}; \ std::array out, padding; \ - OP_REQUIRES_OK(context, Get3dOutputSize(input_size, filter_size, strides, \ - padding_, &out, &padding)); \ + OP_REQUIRES_OK( \ + context, Get3dOutputSizeV2(input_size, filter_size, dilations, strides, \ + padding_, &out, &padding)); \ OP_REQUIRES(context, output_planes == out[0], \ errors::InvalidArgument( \ label, \ @@ -151,6 +156,26 @@ class Conv3DBackpropInputOp : public OpKernel { "Conv3DBackpropInputOpV2 only supports NDHWC on the CPU.")); } + OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilation_)); + OP_REQUIRES(context, dilation_.size() == 5, + errors::InvalidArgument("Dilation rates field must " + "specify 5 dimensions")); + OP_REQUIRES(context, + (GetTensorDim(dilation_, data_format_, 'C') == 1 && + GetTensorDim(dilation_, data_format_, 'N') == 1), + errors::InvalidArgument( + "Current implementation does not yet support " + "dilation rates in the batch and depth dimensions.")); + + // TODO(yangzihao): Add CPU version of dilated conv 3D. + OP_REQUIRES(context, + (GetTensorDim(dilation_, data_format_, '0') == 1 && + GetTensorDim(dilation_, data_format_, '1') == 1 && + GetTensorDim(dilation_, data_format_, '2') == 1), + errors::InvalidArgument( + "Current CPU implementation does not yet support " + "dilation rates larger than 1.")); + OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_)); OP_REQUIRES(context, stride_.size() == 5, errors::InvalidArgument("Sliding window strides field must " @@ -223,6 +248,7 @@ class Conv3DBackpropInputOp : public OpKernel { } private: + std::vector dilation_; std::vector stride_; Padding padding_; TensorFormat data_format_; @@ -261,6 +287,26 @@ class Conv3DBackpropFilterOp : public OpKernel { "Conv3DBackpropFilterOpV2 only supports NDHWC on the CPU.")); } + OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilation_)); + OP_REQUIRES(context, dilation_.size() == 5, + errors::InvalidArgument("Dilation rates field must " + "specify 5 dimensions")); + OP_REQUIRES(context, + (GetTensorDim(dilation_, data_format_, 'C') == 1 && + GetTensorDim(dilation_, data_format_, 'N') == 1), + errors::InvalidArgument( + "Current implementation does not yet support " + "dilation rates in the batch and depth dimensions.")); + + // TODO(yangzihao): Add CPU version of dilated conv 3D. + OP_REQUIRES(context, + (GetTensorDim(dilation_, data_format_, '0') == 1 && + GetTensorDim(dilation_, data_format_, '1') == 1 && + GetTensorDim(dilation_, data_format_, '2') == 1), + errors::InvalidArgument( + "Current CPU implementation does not yet support " + "dilation rates larger than 1.")); + OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_)); OP_REQUIRES(context, stride_.size() == 5, errors::InvalidArgument("Sliding window strides field must " @@ -370,6 +416,7 @@ class Conv3DBackpropFilterOp : public OpKernel { } private: + std::vector dilation_; std::vector stride_; Padding padding_; TensorFormat data_format_; @@ -438,6 +485,22 @@ class Conv3DBackpropInputOp : public OpKernel { OP_REQUIRES(context, FormatFromString(data_format, &data_format_), errors::InvalidArgument("Invalid data format")); } + OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilation_)); + OP_REQUIRES(context, dilation_.size() == 5, + errors::InvalidArgument("Dilation rates field must " + "specify 5 dimensions")); + OP_REQUIRES(context, + (GetTensorDim(dilation_, data_format_, 'C') == 1 && + GetTensorDim(dilation_, data_format_, 'N') == 1), + errors::InvalidArgument( + "Current implementation does not yet support " + "dilation rates in the batch and depth dimensions.")); + OP_REQUIRES( + context, + (GetTensorDim(dilation_, data_format_, '0') > 0 && + GetTensorDim(dilation_, data_format_, '1') > 0 && + GetTensorDim(dilation_, data_format_, '2') > 0), + errors::InvalidArgument("Dilated rates should be larger than 0.")); OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_)); OP_REQUIRES(context, stride_.size() == 5, errors::InvalidArgument("Sliding window strides field must " @@ -448,6 +511,12 @@ class Conv3DBackpropInputOp : public OpKernel { GetTensorDim(stride_, data_format_, 'N') == 1), errors::InvalidArgument("Current implementation does not yet support " "strides in the batch and depth dimensions.")); + OP_REQUIRES( + context, + (GetTensorDim(stride_, data_format_, '0') > 0 && + GetTensorDim(stride_, data_format_, '1') > 0 && + GetTensorDim(stride_, data_format_, '2') > 0), + errors::InvalidArgument("Spatial strides should be larger than 0.")); OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_)); cudnn_use_autotune_ = CudnnUseAutotune(); } @@ -471,6 +540,7 @@ class Conv3DBackpropInputOp : public OpKernel { OP_REQUIRES(context, stream, errors::Internal("No GPU stream available.")); if (filter_size[0] == 1 && filter_size[1] == 1 && filter_size[2] == 1 && + dilation_[0] == 1 && dilation_[1] == 1 && dilation_[2] == 1 && stride_[0] == 1 && stride_[1] == 1 && stride_[2] == 1 && data_format_ == FORMAT_NHWC) { const uint64 m = batch * input_size[0] * input_size[1] * input_size[2]; @@ -580,7 +650,10 @@ class Conv3DBackpropInputOp : public OpKernel { .set_input_feature_map_count(in_depth) .set_output_feature_map_count(out_depth); perftools::gputools::dnn::ConvolutionDescriptor conv_desc(3); - conv_desc.set_filter_stride(DimIndex::X, strides[2]) + conv_desc.set_dilation_rate(DimIndex::X, dilations[2]) + .set_dilation_rate(DimIndex::Y, dilations[1]) + .set_dilation_rate(DimIndex::Z, dilations[0]) + .set_filter_stride(DimIndex::X, strides[2]) .set_filter_stride(DimIndex::Y, strides[1]) .set_filter_stride(DimIndex::Z, strides[0]) .set_zero_padding(DimIndex::X, padding_cols / 2) @@ -645,9 +718,7 @@ class Conv3DBackpropInputOp : public OpKernel { {{input_size[0], input_size[1], input_size[2]}}, out_depth, {{filter_size[0], filter_size[1], filter_size[2]}}, - // TODO(yangzihao): Send in arbitrary dilation rates after the dilated - // conv is supported. - /*dilation=*/{{1, 1, 1}}, + {{dilations[0], dilations[1], dilations[2]}}, {{strides[0], strides[1], strides[2]}}, {{padding_planes, padding_rows, padding_cols}}, dtype, @@ -755,6 +826,7 @@ class Conv3DBackpropInputOp : public OpKernel { } private: + std::vector dilation_; std::vector stride_; Padding padding_; TensorFormat data_format_; @@ -784,6 +856,22 @@ class Conv3DBackpropFilterOp : public OpKernel { OP_REQUIRES(context, FormatFromString(data_format, &data_format_), errors::InvalidArgument("Invalid data format")); } + OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilation_)); + OP_REQUIRES(context, dilation_.size() == 5, + errors::InvalidArgument("Dilation rates field must " + "specify 5 dimensions")); + OP_REQUIRES(context, + (GetTensorDim(dilation_, data_format_, 'C') == 1 && + GetTensorDim(dilation_, data_format_, 'N') == 1), + errors::InvalidArgument( + "Current implementation does not yet support " + "dilation rates in the batch and depth dimensions.")); + OP_REQUIRES( + context, + (GetTensorDim(dilation_, data_format_, '0') > 0 && + GetTensorDim(dilation_, data_format_, '1') > 0 && + GetTensorDim(dilation_, data_format_, '2') > 0), + errors::InvalidArgument("Dilated rates should be larger than 0.")); OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_)); OP_REQUIRES(context, stride_.size() == 5, errors::InvalidArgument("Sliding window strides field must " @@ -794,6 +882,12 @@ class Conv3DBackpropFilterOp : public OpKernel { GetTensorDim(stride_, data_format_, 'N') == 1), errors::InvalidArgument("Current implementation does not yet support " "strides in the batch and depth dimensions.")); + OP_REQUIRES( + context, + (GetTensorDim(stride_, data_format_, '0') > 0 && + GetTensorDim(stride_, data_format_, '1') > 0 && + GetTensorDim(stride_, data_format_, '2') > 0), + errors::InvalidArgument("Spatial strides should be larger than 0.")); OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_)); cudnn_use_autotune_ = CudnnUseAutotune(); } @@ -820,6 +914,7 @@ class Conv3DBackpropFilterOp : public OpKernel { OP_REQUIRES(context, stream, errors::Internal("No GPU stream available.")); if (filter_size[1] == 1 && filter_size[2] == 1 && filter_size[0] == 1 && + dilations[2] == 1 && dilations[1] == 1 && dilations[0] == 1 && strides[2] == 1 && strides[1] == 1 && strides[0] == 1 && data_format_ == FORMAT_NHWC) { const uint64 m = in_depth; @@ -943,7 +1038,10 @@ class Conv3DBackpropFilterOp : public OpKernel { .set_input_feature_map_count(in_depth) .set_output_feature_map_count(out_depth); perftools::gputools::dnn::ConvolutionDescriptor conv_desc(3); - conv_desc.set_filter_stride(DimIndex::X, strides[2]) + conv_desc.set_dilation_rate(DimIndex::X, dilations[2]) + .set_dilation_rate(DimIndex::Y, dilations[1]) + .set_dilation_rate(DimIndex::Z, dilations[0]) + .set_filter_stride(DimIndex::X, strides[2]) .set_filter_stride(DimIndex::Y, strides[1]) .set_filter_stride(DimIndex::Z, strides[0]) .set_zero_padding(DimIndex::X, padding_cols / 2) @@ -1016,7 +1114,7 @@ class Conv3DBackpropFilterOp : public OpKernel { {{input_size[0], input_size[1], input_size[2]}}, out_depth, {{filter_size[0], filter_size[1], filter_size[2]}}, - {{1, 1, 1}}, + {{dilations[0], dilations[1], dilations[2]}}, {{strides[0], strides[1], strides[2]}}, {{padding_planes, padding_rows, padding_cols}}, dtype, @@ -1102,6 +1200,7 @@ class Conv3DBackpropFilterOp : public OpKernel { } private: + std::vector dilation_; std::vector stride_; Padding padding_; TensorFormat data_format_; diff --git a/tensorflow/core/kernels/conv_ops_3d.cc b/tensorflow/core/kernels/conv_ops_3d.cc index 0b7c1524e6..48dd3c9eb0 100644 --- a/tensorflow/core/kernels/conv_ops_3d.cc +++ b/tensorflow/core/kernels/conv_ops_3d.cc @@ -49,12 +49,18 @@ template struct LaunchConvOp { static void launch(OpKernelContext* context, bool cudnn_use_autotune, const Tensor& input, const Tensor& filter, + const std::array& dilations, const std::array& strides, const Padding padding, TensorFormat data_format, Tensor* output) { OP_REQUIRES(context, data_format == FORMAT_NHWC, errors::InvalidArgument("CPU implementation of Conv3D " "currently only supports the NHWC " "tensor format.")); + OP_REQUIRES(context, + dilations[0] == 1 && dilations[1] == 1 && dilations[2] == 1, + errors::InvalidArgument("CPU implementation of Conv3D " + "currently only supports dilated rates " + "of 1.")); functor::CuboidConvolution()( context->eigen_device(), output->tensor(), input.tensor(), filter.tensor(), strides[2], strides[1], @@ -80,6 +86,28 @@ class Conv3DOp : public BinaryOp { GetTensorDim(stride_, data_format_, 'C') == 1), errors::InvalidArgument("Current implementation does not yet support " "strides in the batch and depth dimensions.")); + OP_REQUIRES( + context, + (GetTensorDim(stride_, data_format_, '0') > 0 && + GetTensorDim(stride_, data_format_, '1') > 0 && + GetTensorDim(stride_, data_format_, '2') > 0), + errors::InvalidArgument("Spatial strides should be larger than 0.")); + OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilation_)); + OP_REQUIRES(context, dilation_.size() == 5, + errors::InvalidArgument("Dilation rates field must " + "specify 5 dimensions")); + OP_REQUIRES(context, + (GetTensorDim(dilation_, data_format_, 'N') == 1 && + GetTensorDim(dilation_, data_format_, 'C') == 1), + errors::InvalidArgument( + "Current implementation does not yet support " + "dilation rates in the batch and depth dimensions.")); + OP_REQUIRES( + context, + (GetTensorDim(dilation_, data_format_, '0') > 0 && + GetTensorDim(dilation_, data_format_, '1') > 0 && + GetTensorDim(dilation_, data_format_, '2') > 0), + errors::InvalidArgument("Dilated rates should be larger than 0.")); OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_)); cudnn_use_autotune_ = CudnnUseAutotune(); } @@ -115,13 +143,18 @@ class Conv3DOp : public BinaryOp { GetTensorDim(input, data_format_, '2')}}; std::array filter_size = { {filter.dim_size(0), filter.dim_size(1), filter.dim_size(2)}}; + std::array dilations = { + {GetTensorDim(dilation_, data_format_, '0'), + GetTensorDim(dilation_, data_format_, '1'), + GetTensorDim(dilation_, data_format_, '2')}}; std::array strides = {{GetTensorDim(stride_, data_format_, '0'), GetTensorDim(stride_, data_format_, '1'), GetTensorDim(stride_, data_format_, '2')}}; std::array out, padding; - OP_REQUIRES_OK(context, Get3dOutputSize(input_size, filter_size, strides, - padding_, &out, &padding)); + OP_REQUIRES_OK( + context, Get3dOutputSizeV2(input_size, filter_size, dilations, strides, + padding_, &out, &padding)); TensorShape out_shape = ShapeFromFormat( data_format_, in_batch, {{out[0], out[1], out[2]}}, out_depth); Tensor* output; @@ -131,10 +164,12 @@ class Conv3DOp : public BinaryOp { if (out_shape.num_elements() == 0) return; LaunchConvOp::launch(context, cudnn_use_autotune_, input, filter, - strides, padding_, data_format_, output); + dilations, strides, padding_, data_format_, + output); } private: + std::vector dilation_; std::vector stride_; Padding padding_; TensorFormat data_format_; @@ -165,6 +200,7 @@ template struct LaunchConvOp { static void launch(OpKernelContext* ctx, bool cudnn_use_autotune, const Tensor& input_param, const Tensor& filter, + const std::array& dilations, const std::array& strides, const Padding padding, TensorFormat data_format, Tensor* output) { auto* stream = ctx->op_device_context()->stream(); @@ -199,6 +235,7 @@ struct LaunchConvOp { // NOTE: This only works in NHWC. if (filter_planes == 1 && filter_rows == 1 && filter_cols == 1 && + dilations[0] == 1 && dilations[1] == 1 && dilations[2] == 1 && strides[0] == 1 && strides[1] == 1 && strides[2] == 1 && data_format == FORMAT_NHWC) { // 1x1 filter, so call cublas directly. @@ -330,7 +367,10 @@ struct LaunchConvOp { .set_input_feature_map_count(in_depth) .set_output_feature_map_count(out_depth); perftools::gputools::dnn::ConvolutionDescriptor conv_desc(3); - conv_desc.set_filter_stride(DimIndex::X, strides[2]) + conv_desc.set_dilation_rate(DimIndex::X, dilations[2]) + .set_dilation_rate(DimIndex::Y, dilations[1]) + .set_dilation_rate(DimIndex::Z, dilations[0]) + .set_filter_stride(DimIndex::X, strides[2]) .set_filter_stride(DimIndex::Y, strides[1]) .set_filter_stride(DimIndex::Z, strides[0]) .set_zero_padding(DimIndex::X, pad_cols / 2) @@ -377,9 +417,7 @@ struct LaunchConvOp { {{in_planes, in_rows, in_cols}}, out_depth, {{filter_planes, filter_rows, filter_cols}}, - // TODO(yangzihao): Send in arbitrary dilation rates after the dilated - // conv is supported. - /*dilation=*/{{1, 1, 1}}, + {{dilations[0], dilations[1], dilations[2]}}, {{strides[0], strides[1], strides[2]}}, {{pad_planes, pad_rows, pad_cols}}, dtype, diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc index 12d6dc5eaf..6dc3d9df31 100644 --- a/tensorflow/core/ops/nn_ops.cc +++ b/tensorflow/core/ops/nn_ops.cc @@ -524,6 +524,7 @@ REGISTER_OP("Conv3DBackpropInput") .Attr("strides: list(int) >= 5") .Attr(GetPaddingAttrString()) .Deprecated(10, "Use Conv3DBackpropInputV2") + .Attr("dilations: list(int) = [1, 1, 1, 1, 1]") .SetShapeFn([](InferenceContext* c) { return UnchangedShapeWithRank(c, 5); }); @@ -537,6 +538,7 @@ REGISTER_OP("Conv3DBackpropFilter") .Attr("strides: list(int) >= 5") .Attr(GetPaddingAttrString()) .Deprecated(10, "Use Conv3DBackpropFilterV2") + .Attr("dilations: list(int) = [1, 1, 1, 1, 1]") .SetShapeFn([](InferenceContext* c) { ShapeHandle out; TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 5, &out)); diff --git a/tensorflow/python/kernel_tests/conv_ops_3d_test.py b/tensorflow/python/kernel_tests/conv_ops_3d_test.py index f4616fd661..0b531125f3 100644 --- a/tensorflow/python/kernel_tests/conv_ops_3d_test.py +++ b/tensorflow/python/kernel_tests/conv_ops_3d_test.py @@ -28,6 +28,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import test_util from tensorflow.python.ops import gradient_checker +from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import nn_ops import tensorflow.python.ops.nn_grad # pylint: disable=unused-import from tensorflow.python.platform import test @@ -61,18 +62,18 @@ class Conv3DTest(test.TestCase): def _SetupValuesForDevice(self, tensor_in_sizes, filter_in_sizes, stride, padding, data_format, dtype, use_gpu): - total_size_1 = 1 - total_size_2 = 1 + total_size_tensor = 1 + total_size_filter = 1 for s in tensor_in_sizes: - total_size_1 *= s + total_size_tensor *= s for s in filter_in_sizes: - total_size_2 *= s + total_size_filter *= s # Initializes the input tensor with array containing numbers from 0 to 1. # We keep the input tensor values fairly small to avoid overflowing float16 # during the conv3d. - x1 = [f * 1.0 / total_size_1 for f in range(1, total_size_1 + 1)] - x2 = [f * 1.0 / total_size_2 for f in range(1, total_size_2 + 1)] + x1 = [f * 1.0 / total_size_tensor for f in range(1, total_size_tensor + 1)] + x2 = [f * 1.0 / total_size_filter for f in range(1, total_size_filter + 1)] with self.test_session(use_gpu=use_gpu): t1 = constant_op.constant(x1, shape=tensor_in_sizes, dtype=dtype) t2 = constant_op.constant(x2, shape=filter_in_sizes, dtype=dtype) @@ -118,6 +119,79 @@ class Conv3DTest(test.TestCase): self.assertAllClose(expected, value.flatten(), atol=tol, rtol=tol) + def _ComputeReferenceDilatedConv(self, tensor_in_sizes, filter_in_sizes, + stride, dilation, padding, data_format, + use_gpu): + total_size_tensor = 1 + total_size_filter = 1 + for s in tensor_in_sizes: + total_size_tensor *= s + for s in filter_in_sizes: + total_size_filter *= s + + # Initializes the input tensor with array containing incrementing + # numbers from 1. + x1 = [f * 1.0 for f in range(1, total_size_tensor + 1)] + x2 = [f * 1.0 for f in range(1, total_size_filter + 1)] + with self.test_session(use_gpu=use_gpu): + t1 = constant_op.constant(x1, shape=tensor_in_sizes) + t2 = constant_op.constant(x2, shape=filter_in_sizes) + if isinstance(stride, collections.Iterable): + strides = list(stride) + else: + strides = [stride, stride, stride] + if data_format == "NCDHW": + t1 = test_util.NHWCToNCHW(t1) + full_strides = [1, 1] + strides + full_dilation = [1, 1] + dilation + else: + full_strides = [1] + strides + [1] + full_dilation = [1] + dilation + [1] + expected = nn_ops.convolution( + t1, + t2, + padding=padding, + strides=strides, + dilation_rate=dilation, + data_format=data_format) + computed = nn_ops.conv3d( + t1, + t2, + strides=full_strides, + dilations=full_dilation, + padding=padding, + data_format=data_format) + if data_format == "NCDHW": + expected = test_util.NCHWToNHWC(expected) + computed = test_util.NCHWToNHWC(computed) + return expected, computed + + def _VerifyDilatedConvValues(self, tensor_in_sizes, filter_in_sizes, stride, + padding, dilations): + expected_results = [] + computed_results = [] + default_dilations = ( + dilations[0] == 1 and dilations[1] == 1 and dilations[2] == 1) + for data_format, use_gpu in GetTestConfigs(): + # If any dilation rate is larger than 1, only do test on the GPU + # because we currently do not have a CPU implementation for arbitrary + # dilation rates. + if default_dilations or use_gpu: + expected, computed = self._ComputeReferenceDilatedConv( + tensor_in_sizes, filter_in_sizes, stride, dilations, padding, + data_format, use_gpu) + expected_results.append(expected) + computed_results.append(computed) + tolerance = 1e-2 if use_gpu else 1e-5 + with self.test_session() as sess: + expected_values = sess.run(expected_results) + computed_values = sess.run(computed_results) + for e_value, c_value in zip(expected_values, computed_values): + print("expected = ", e_value) + print("actual = ", c_value) + self.assertAllClose( + e_value.flatten(), c_value.flatten(), atol=tolerance, rtol=1e-6) + def testConv3D1x1x1Filter(self): expected_output = [ 0.18518519, 0.22222222, 0.25925926, 0.40740741, 0.5, 0.59259259, @@ -145,6 +219,15 @@ class Conv3DTest(test.TestCase): padding="VALID", expected=expected_output) + def testConv3D1x1x1Filter2x1x1Dilation(self): + if test.is_gpu_available(cuda_only=True): + self._VerifyDilatedConvValues( + tensor_in_sizes=[1, 3, 6, 1, 1], + filter_in_sizes=[1, 1, 1, 1, 1], + stride=1, + padding="VALID", + dilations=[2, 1, 1]) + # Expected values computed using scipy's correlate function. def testConv3D2x2x2Filter(self): expected_output = [ @@ -161,6 +244,15 @@ class Conv3DTest(test.TestCase): padding="VALID", expected=expected_output) + def testConv3D2x2x2Filter1x2x1Dilation(self): + if test.is_gpu_available(cuda_only=True): + self._VerifyDilatedConvValues( + tensor_in_sizes=[1, 4, 6, 3, 1], + filter_in_sizes=[2, 2, 2, 1, 1], + stride=1, + padding="VALID", + dilations=[1, 2, 1]) + def testConv3DStrides(self): expected_output = [ 0.06071429, 0.08988095, 0.10238095, 0.11488095, 0.12738095, 0.13988095, @@ -546,6 +638,98 @@ class Conv3DTest(test.TestCase): padding="SAME", test_input=False) + # Testing for backprops + def _RunAndVerifyBackprop(self, input_sizes, filter_sizes, output_sizes, + strides, dilations, padding, data_format, use_gpu, + err, mode): + total_input_size = 1 + total_filter_size = 1 + for s in input_sizes: + total_input_size *= s + for s in filter_sizes: + total_filter_size *= s + # Initializes the input tensor with array containing incrementing + # numbers from 1. + x1 = [f * 1.0 for f in range(1, total_input_size + 1)] + x2 = [f * 1.0 for f in range(1, total_filter_size + 1)] + default_dilations = ( + dilations[0] == 1 and dilations[1] == 1 and dilations[2] == 1) + + # If any dilation rate is larger than 1, only do test on the GPU + # because we currently do not have a CPU implementation for arbitrary + # dilation rates. + if default_dilations or use_gpu: + with self.test_session(use_gpu=use_gpu) as sess: + if data_format == "NCDHW": + input_sizes = test_util.NHWCToNCHW(input_sizes) + t1 = constant_op.constant(x1, shape=input_sizes) + t2 = constant_op.constant(x2, shape=filter_sizes) + full_strides = [1] + strides + [1] + full_dilations = [1] + dilations + [1] + if data_format == "NCDHW": + full_strides = test_util.NHWCToNCHW(full_strides) + full_dilations = test_util.NHWCToNCHW(full_dilations) + actual = nn_ops.conv3d( + t1, + t2, + strides=full_strides, + dilations=full_dilations, + padding=padding, + data_format=data_format) + expected = nn_ops.convolution( + t1, + t2, + padding=padding, + strides=strides, + dilation_rate=dilations, + data_format=data_format) + if data_format == "NCDHW": + actual = test_util.NCHWToNHWC(actual) + expected = test_util.NCHWToNHWC(expected) + actual_grad = gradients_impl.gradients(actual, t1 + if mode == "input" else t2)[0] + expected_grad = gradients_impl.gradients(expected, t1 + if mode == "input" else t2)[0] + # "values" consists of two tensors for two backprops + actual_value = sess.run(actual_grad) + expected_value = sess.run(expected_grad) + self.assertShapeEqual(actual_value, actual_grad) + self.assertShapeEqual(expected_value, expected_grad) + print("expected = ", expected_value) + print("actual = ", actual_value) + self.assertArrayNear(expected_value.flatten(), actual_value.flatten(), + err) + + def testConv3D2x2Depth3ValidBackpropFilterStride1x1Dilation2x1(self): + if test.is_gpu_available(cuda_only=True): + for (data_format, use_gpu) in GetTestConfigs(): + self._RunAndVerifyBackprop( + input_sizes=[1, 3, 6, 1, 1], + filter_sizes=[2, 2, 1, 1, 1], + output_sizes=[1, 1, 5, 1, 1], + strides=[1, 1, 1], + dilations=[2, 1, 1], + padding="VALID", + data_format=data_format, + use_gpu=use_gpu, + err=1e-5, + mode="filter") + + def testConv3D2x2Depth3ValidBackpropInputStride1x1Dilation2x1(self): + if test.is_gpu_available(cuda_only=True): + for (data_format, use_gpu) in GetTestConfigs(): + self._RunAndVerifyBackprop( + input_sizes=[1, 3, 6, 1, 1], + filter_sizes=[2, 2, 1, 1, 1], + output_sizes=[1, 1, 5, 1, 1], + strides=[1, 1, 1], + dilations=[2, 1, 1], + padding="VALID", + data_format=data_format, + use_gpu=use_gpu, + err=1e-5, + mode="input") + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/ops/nn_grad.py b/tensorflow/python/ops/nn_grad.py index 4af5bd26dd..3a41391340 100644 --- a/tensorflow/python/ops/nn_grad.py +++ b/tensorflow/python/ops/nn_grad.py @@ -94,6 +94,7 @@ def _Conv3DGrad(op, grad): array_ops.shape(op.inputs[0]), op.inputs[1], grad, + dilations=op.get_attr("dilations"), strides=op.get_attr("strides"), padding=op.get_attr("padding"), data_format=data_format), @@ -101,6 +102,7 @@ def _Conv3DGrad(op, grad): op.inputs[0], array_ops.shape(op.inputs[1]), grad, + dilations=op.get_attr("dilations"), strides=op.get_attr("strides"), padding=op.get_attr("padding"), data_format=data_format) @@ -116,12 +118,14 @@ def _Conv3DBackpropInputGrad(op, grad): grad, array_ops.shape(op.inputs[1]), op.inputs[2], + dilations=op.get_attr("dilations"), strides=op.get_attr("strides"), padding=op.get_attr("padding"), data_format=data_format), nn_ops.conv3d( grad, op.inputs[1], + dilations=op.get_attr("dilations"), strides=op.get_attr("strides"), padding=op.get_attr("padding"), data_format=data_format) @@ -136,12 +140,14 @@ def _Conv3DBackpropFilterGrad(op, grad): array_ops.shape(op.inputs[0]), grad, op.inputs[2], + dilations=op.get_attr("dilations"), strides=op.get_attr("strides"), padding=op.get_attr("padding"), data_format=data_format), None, nn_ops.conv3d( op.inputs[0], grad, + dilations=op.get_attr("dilations"), strides=op.get_attr("strides"), padding=op.get_attr("padding"), data_format=data_format) -- GitLab From a175841eb549f069ac205fb32bf55314a387fe6d Mon Sep 17 00:00:00 2001 From: jinghuangintel Date: Fri, 20 Apr 2018 12:20:00 -0700 Subject: [PATCH 751/791] [INTEL MKLDNN]: Upgrade mkldnn version to v13 (#18508) * upgrade mkldnn version to v13 * upgrade mkldnn version to v13 for all platforms --- tensorflow/workspace.bzl | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index c58ef87338..f0a81f7754 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -50,31 +50,31 @@ def tf_workspace(path_prefix="", tf_repo_name=""): mkl_repository( name = "mkl_linux", urls = [ - "https://mirror.bazel.build/github.com/intel/mkl-dnn/releases/download/v0.12/mklml_lnx_2018.0.1.20171227.tgz", - "https://github.com/intel/mkl-dnn/releases/download/v0.12/mklml_lnx_2018.0.1.20171227.tgz", + "https://mirror.bazel.build/github.com/intel/mkl-dnn/releases/download/v0.13/mklml_lnx_2018.0.2.20180127.tgz", + "https://github.com/intel/mkl-dnn/releases/download/v0.13/mklml_lnx_2018.0.2.20180127.tgz", ], - sha256 = "feacc3d82565c1231470359b42c696236fae873704e0b013436afba5fd4fd30f", - strip_prefix = "mklml_lnx_2018.0.1.20171227", + sha256 = "74844bd77294742bf2396ff040369d1aa4cdd9e826fcd38cf8398ae83564d146", + strip_prefix = "mklml_lnx_2018.0.2.20180127", build_file = clean_dep("//third_party/mkl:mkl.BUILD") ) mkl_repository( name = "mkl_windows", urls = [ - "https://mirror.bazel.build/github.com/intel/mkl-dnn/releases/download/v0.12/mklml_win_2018.0.1.20171227.zip", - "https://github.com/intel/mkl-dnn/releases/download/v0.12/mklml_win_2018.0.1.20171227.zip" + "https://mirror.bazel.build/github.com/intel/mkl-dnn/releases/download/v0.13/mklml_win_2018.0.2.20180127.zip", + "https://github.com/intel/mkl-dnn/releases/download/v0.13/mklml_win_2018.0.2.20180127.zip" ], - sha256 = "24bae8d7b22b431a654acadea43f2243c46ae6b1e5a73a4a936825f31d284ee4", - strip_prefix = "mklml_win_2018.0.1.20171227", + sha256 = "d8fbf0faa0684bffa3548005d05fe5cfe56ff9dbc0e15e7612d7ac01055a6ded", + strip_prefix = "mklml_win_2018.0.2.20180127", build_file = clean_dep("//third_party/mkl:mkl.BUILD") ) mkl_repository( name = "mkl_darwin", urls = [ - "https://mirror.bazel.build/github.com/intel/mkl-dnn/releases/download/v0.12/mklml_mac_2018.0.1.20171227.tgz", - "https://github.com/intel/mkl-dnn/releases/download/v0.12/mklml_mac_2018.0.1.20171227.tgz" + "https://mirror.bazel.build/github.com/intel/mkl-dnn/releases/download/v0.13/mklml_mac_2018.0.2.20180127.tgz", + "https://github.com/intel/mkl-dnn/releases/download/v0.13/mklml_mac_2018.0.2.20180127.tgz" ], - sha256 = "0e954ec6fd3dc5e37f64c4043f6b5613dd687558da3df1028b3b7c29ff5cf77f", - strip_prefix = "mklml_mac_2018.0.1.20171227", + sha256 = "aa740d71e14562bfea56e6829e6dc186e7487cbcf6748a88dec73826b7ec1943", + strip_prefix = "mklml_mac_2018.0.2.20180127", build_file = clean_dep("//third_party/mkl:mkl.BUILD") ) @@ -85,11 +85,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""): tf_http_archive( name = "mkl_dnn", urls = [ - "https://mirror.bazel.build/github.com/intel/mkl-dnn/archive/v0.12.tar.gz", - "https://github.com/intel/mkl-dnn/archive/v0.12.tar.gz", + "https://mirror.bazel.build/github.com/intel/mkl-dnn/archive/v0.13.tar.gz", + "https://github.com/intel/mkl-dnn/archive/v0.13.tar.gz", ], - sha256 = "86fa2a8c12a56e3b725945acedeaa82492746be02545aba6d710f097e013e19e", - strip_prefix = "mkl-dnn-0.12", + sha256 = "d2cfd93a70cfe86ebe054477c530c9b5c1218b70f75856eb6d1956c68ee89e8f", + strip_prefix = "mkl-dnn-0.13", build_file = clean_dep("//third_party/mkl_dnn:mkldnn.BUILD"), ) -- GitLab From b23e91d247368f2046dae035b5c7bdda56512077 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Apr 2018 12:37:39 -0700 Subject: [PATCH 752/791] Changed tf_to_tflite build rule. PiperOrigin-RevId: 193707628 --- tensorflow/contrib/lite/build_def.bzl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tensorflow/contrib/lite/build_def.bzl b/tensorflow/contrib/lite/build_def.bzl index b8f6b7fd59..8521677682 100644 --- a/tensorflow/contrib/lite/build_def.bzl +++ b/tensorflow/contrib/lite/build_def.bzl @@ -124,19 +124,19 @@ def tf_to_tflite(name, src, options, out): out: name of the output flatbuffer file. """ - toco = "//tensorflow/contrib/lite/toco:toco" + toco_cmdline = " ".join([ + "//tensorflow/contrib/lite/toco:toco", + "--input_format=TENSORFLOW_GRAPHDEF", + "--output_format=TFLITE", + ("--input_file=$(location %s)" % src), + ("--output_file=$(location %s)" % out), + ] + options ) native.genrule( name = name, - srcs=[src, options], + srcs=[src], outs=[out], - cmd = ("$(location %s) " + - " --input_file=$(location %s) " + - " --output_file=$(location %s) " + - " --input_format=TENSORFLOW_GRAPHDEF" + - " --output_format=TFLITE" + - " `cat $(location %s)`") - % (toco, src, out, options), - tools= [toco], + cmd = toco_cmdline, + tools= ["//tensorflow/contrib/lite/toco:toco"], ) def tflite_to_json(name, src, out): -- GitLab From 517d1912f4ec71180944320350a3694332a1dedc Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Fri, 20 Apr 2018 12:40:57 -0700 Subject: [PATCH 753/791] Add a utility to visualize object-based checkpoints Useful for generating a warm fuzzy feeling that everything you think should be saved was saved, and for explaining what object-based checkpointing is. (Also useful on the former front will be a planned "assert that all of this Graph's trainable variables are accessible from object X" function.) Somewhat hacky since it generates strings rather than using the pydot bindings (and so works without a pydot dependency). PiperOrigin-RevId: 193708003 --- tensorflow/contrib/BUILD | 1 + tensorflow/contrib/checkpoint/__init__.py | 3 + tensorflow/contrib/checkpoint/python/BUILD | 32 +++++ .../contrib/checkpoint/python/visualize.py | 111 ++++++++++++++++++ .../checkpoint/python/visualize_test.py | 97 +++++++++++++++ 5 files changed, 244 insertions(+) create mode 100644 tensorflow/contrib/checkpoint/python/visualize.py create mode 100644 tensorflow/contrib/checkpoint/python/visualize_test.py diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD index 7e47516550..d28392a62c 100644 --- a/tensorflow/contrib/BUILD +++ b/tensorflow/contrib/BUILD @@ -25,6 +25,7 @@ py_library( "//tensorflow/contrib/batching:batch_py", "//tensorflow/contrib/bayesflow:bayesflow_py", "//tensorflow/contrib/boosted_trees:init_py", + "//tensorflow/contrib/checkpoint/python:checkpoint", "//tensorflow/contrib/cloud:cloud_py", "//tensorflow/contrib/cluster_resolver:cluster_resolver_pip", "//tensorflow/contrib/cluster_resolver:cluster_resolver_py", diff --git a/tensorflow/contrib/checkpoint/__init__.py b/tensorflow/contrib/checkpoint/__init__.py index 70d7d2d8d7..1192cc44a1 100644 --- a/tensorflow/contrib/checkpoint/__init__.py +++ b/tensorflow/contrib/checkpoint/__init__.py @@ -16,6 +16,7 @@ For creating and managing dependencies: +@@dot_graph_from_checkpoint @@split_dependency """ @@ -24,6 +25,8 @@ from __future__ import division from __future__ import print_function from tensorflow.contrib.checkpoint.python.split_dependency import split_dependency +from tensorflow.contrib.checkpoint.python.visualize import dot_graph_from_checkpoint + from tensorflow.python.util.all_util import remove_undocumented remove_undocumented(module_name=__name__) diff --git a/tensorflow/contrib/checkpoint/python/BUILD b/tensorflow/contrib/checkpoint/python/BUILD index d57b01aab2..a5681ffa61 100644 --- a/tensorflow/contrib/checkpoint/python/BUILD +++ b/tensorflow/contrib/checkpoint/python/BUILD @@ -4,6 +4,15 @@ package(default_visibility = ["//tensorflow:internal"]) load("//tensorflow:tensorflow.bzl", "py_test") +py_library( + name = "checkpoint", + srcs_version = "PY2AND3", + deps = [ + ":split_dependency", + ":visualize", + ], +) + py_library( name = "split_dependency", srcs = ["split_dependency.py"], @@ -27,3 +36,26 @@ py_test( "//tensorflow/python/eager:test", ], ) + +py_library( + name = "visualize", + srcs = ["visualize.py"], + srcs_version = "PY2AND3", + visibility = ["//tensorflow:internal"], + deps = [ + "//tensorflow/python:pywrap_tensorflow", + ], +) + +py_test( + name = "visualize_test", + srcs = ["visualize_test.py"], + deps = [ + ":visualize", + "//tensorflow/python:array_ops", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:resource_variable_ops", + "//tensorflow/python:training", + "//tensorflow/python/eager:test", + ], +) diff --git a/tensorflow/contrib/checkpoint/python/visualize.py b/tensorflow/contrib/checkpoint/python/visualize.py new file mode 100644 index 0000000000..86fbdb41d2 --- /dev/null +++ b/tensorflow/contrib/checkpoint/python/visualize.py @@ -0,0 +1,111 @@ +"""Utilities for visualizing dependency graphs.""" +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.core.protobuf import checkpointable_object_graph_pb2 +from tensorflow.python import pywrap_tensorflow +from tensorflow.python.framework import errors_impl +from tensorflow.python.training import checkpointable + + +def dot_graph_from_checkpoint(save_path): + r"""Visualizes an object-based checkpoint (from `tf.train.Checkpoint`). + + Useful for inspecting checkpoints and debugging loading issues. + + Example usage from Python (requires pydot): + ```python + import tensorflow as tf + import pydot + + dot_string = tf.contrib.checkpoint.dot_graph_from_checkpoint('/path/to/ckpt') + parsed, = pydot.graph_from_dot_data(dot_string) + parsed.write_svg('/tmp/tensorflow/visualized_checkpoint.svg') + ``` + + Example command line usage: + ```sh + python -c "import tensorflow as tf;\ + print(tf.contrib.checkpoint.dot_graph_from_checkpoint('/path/to/ckpt'))"\ + | dot -Tsvg > /tmp/tensorflow/checkpoint_viz.svg + ``` + + Args: + save_path: The checkpoint prefix, as returned by `tf.train.Checkpoint.save` + or `tf.train.latest_checkpoint`. + Returns: + A graph in DOT format as a string. + """ + reader = pywrap_tensorflow.NewCheckpointReader(save_path) + try: + object_graph_string = reader.get_tensor( + checkpointable.OBJECT_GRAPH_PROTO_KEY) + except errors_impl.NotFoundError: + raise ValueError( + ('The specified checkpoint "%s" does not appear to be object-based (it ' + 'is missing the key "%s"). Likely it was created with a name-based ' + 'saver and does not contain an object dependency graph.') % ( + save_path, checkpointable.OBJECT_GRAPH_PROTO_KEY)) + shape_map = reader.get_variable_to_shape_map() + dtype_map = reader.get_variable_to_dtype_map() + object_graph = ( + checkpointable_object_graph_pb2.CheckpointableObjectGraph()) + object_graph.ParseFromString(object_graph_string) + graph = 'digraph {\n' + def _escape(name): + return name.replace('"', '\\"') + slot_ids = set() + for node in object_graph.nodes: + for slot_reference in node.slot_variables: + slot_ids.add(slot_reference.slot_variable_node_id) + for node_id, node in enumerate(object_graph.nodes): + if (len(node.attributes) == 1 + and node.attributes[0].name == checkpointable.VARIABLE_VALUE_KEY): + if node_id in slot_ids: + color = 'orange' + tooltip_prefix = 'Slot variable' + else: + color = 'blue' + tooltip_prefix = 'Variable' + attribute = node.attributes[0] + graph += ('N_%d [shape=point label="" color=%s width=.25' + ' tooltip="%s %s shape=%s %s"]\n') % ( + node_id, + color, + tooltip_prefix, + _escape(attribute.full_name), + shape_map[attribute.checkpoint_key], + dtype_map[attribute.checkpoint_key].name) + elif node.slot_variables: + graph += ('N_%d [shape=point label="" width=.25 color=red,' + 'tooltip="Optimizer"]\n') % node_id + else: + graph += 'N_%d [shape=point label="" width=.25]\n' % node_id + for reference in node.children: + graph += 'N_%d -> N_%d [label="%s"]\n' % ( + node_id, reference.node_id, _escape(reference.local_name)) + for slot_reference in node.slot_variables: + graph += 'N_%d -> N_%d [label="%s" style=dotted]\n' % ( + node_id, + slot_reference.slot_variable_node_id, + _escape(slot_reference.slot_name)) + graph += 'N_%d -> N_%d [style=dotted]\n' % ( + slot_reference.original_variable_node_id, + slot_reference.slot_variable_node_id) + graph += '}\n' + return graph diff --git a/tensorflow/contrib/checkpoint/python/visualize_test.py b/tensorflow/contrib/checkpoint/python/visualize_test.py new file mode 100644 index 0000000000..1d9ab78923 --- /dev/null +++ b/tensorflow/contrib/checkpoint/python/visualize_test.py @@ -0,0 +1,97 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools +import os + +from tensorflow.contrib.checkpoint.python import visualize + +from tensorflow.python.eager import context +from tensorflow.python.eager import test +from tensorflow.python.framework import constant_op +from tensorflow.python.keras._impl.keras.engine import training +from tensorflow.python.keras._impl.keras.layers import core +from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.training import adam +from tensorflow.python.training import checkpointable_utils + +try: + import pydot # pylint: disable=g-import-not-at-top +except ImportError: + pydot = None + + +class MyModel(training.Model): + """A concrete Model for testing.""" + + def __init__(self): + super(MyModel, self).__init__() + self._named_dense = core.Dense(1, use_bias=True) + self._second = core.Dense(1, use_bias=False) + + def call(self, values): + ret = self._second(self._named_dense(values)) + return ret + + +class DotGraphTests(test.TestCase): + + def testMakeDotGraph(self): + with context.eager_mode(): + input_value = constant_op.constant([[3.]]) + model = MyModel() + optimizer = adam.AdamOptimizer(0.001) + optimizer_step = resource_variable_ops.ResourceVariable(12) + save_checkpoint = checkpointable_utils.Checkpoint( + optimizer=optimizer, model=model, optimizer_step=optimizer_step) + optimizer.minimize(functools.partial(model, input_value)) + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, 'ckpt') + save_path = save_checkpoint.save(checkpoint_prefix) + prefix = save_checkpoint.save(save_path) + + dot_graph_string = visualize.dot_graph_from_checkpoint(prefix) + + # The remainder of this test is more-or-less optional since it's so + # dependent on pydot/platform/Python versions. + if pydot is None: + self.skipTest('pydot is required for the remainder of this test.') + try: + parsed, = pydot.graph_from_dot_data(dot_graph_string) + except NameError as e: + if "name 'dot_parser' is not defined" in str(e): + self.skipTest("pydot isn't working") + else: + raise + # Check that the graph isn't completely trivial + self.assertEqual( + '"model"', + parsed.obj_dict['edges'][('N_0', 'N_1')][0]['attributes']['label']) + image_path = os.path.join(self.get_temp_dir(), 'saved.svg') + try: + parsed.write_svg(image_path) + except Exception as e: # pylint: disable=broad-except + # For some reason PyDot's "dot not available" error is an Exception, not + # something more specific. + if '"dot" not found in path' in str(e): + self.skipTest("pydot won't save SVGs (dot not available)") + else: + raise + +if __name__ == '__main__': + test.main() -- GitLab From 0b6ca72332735fe460da23fbcca5c8c24d838f28 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Apr 2018 13:18:02 -0700 Subject: [PATCH 754/791] Update ops-related pbtxt files. PiperOrigin-RevId: 193712839 --- .../core/ops/compat/ops_history.v1.pbtxt | 124 ++++++++++++++++++ tensorflow/core/ops/ops.pbtxt | 26 ++++ 2 files changed, 150 insertions(+) diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt index dbd6f859c4..247f9edf5b 100644 --- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt +++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt @@ -13445,6 +13445,68 @@ op { version: 10 } } +op { + name: "Conv3DBackpropFilter" + input_arg { + name: "input" + type_attr: "T" + } + input_arg { + name: "filter" + type_attr: "T" + } + input_arg { + name: "out_backprop" + type_attr: "T" + } + output_arg { + name: "output" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + } + } + } + attr { + name: "strides" + type: "list(int)" + has_minimum: true + minimum: 5 + } + attr { + name: "padding" + type: "string" + allowed_values { + list { + s: "SAME" + s: "VALID" + } + } + } + attr { + name: "dilations" + type: "list(int)" + default_value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + deprecation { + version: 10 + } +} op { name: "Conv3DBackpropFilterV2" input_arg { @@ -13718,6 +13780,68 @@ op { version: 10 } } +op { + name: "Conv3DBackpropInput" + input_arg { + name: "input" + type_attr: "T" + } + input_arg { + name: "filter" + type_attr: "T" + } + input_arg { + name: "out_backprop" + type_attr: "T" + } + output_arg { + name: "output" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + } + } + } + attr { + name: "strides" + type: "list(int)" + has_minimum: true + minimum: 5 + } + attr { + name: "padding" + type: "string" + allowed_values { + list { + s: "SAME" + s: "VALID" + } + } + } + attr { + name: "dilations" + type: "list(int)" + default_value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + deprecation { + version: 10 + } +} op { name: "Conv3DBackpropInputV2" input_arg { diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index 46afe357f0..d1773daebe 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -5651,6 +5651,19 @@ op { } } } + attr { + name: "dilations" + type: "list(int)" + default_value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } deprecation { version: 10 explanation: "Use Conv3DBackpropFilterV2" @@ -5774,6 +5787,19 @@ op { } } } + attr { + name: "dilations" + type: "list(int)" + default_value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } deprecation { version: 10 explanation: "Use Conv3DBackpropInputV2" -- GitLab From 02075fa2456d951ff3b7bdb8fee76a1b9c6d8716 Mon Sep 17 00:00:00 2001 From: Guozhong Zhuang Date: Fri, 20 Apr 2018 13:43:06 -0700 Subject: [PATCH 755/791] MKLDNN: conv2d forward DNN primitive reuse enhancement (#17943) * Enable conv2d fwd primitive reuse * coding style change based on suggestions from TF team * minor code style fix * refactor conv2d primitive reuse class and enhance key creation utility * refactor by introducing ConvFwdDimensions structure * change 'Execute' method to be a template one per PR review suggestion * Per PR review suggestion, update DnnOp class to declared related method as abstract ones * refactor AddAsKey method - template for scalar value and remove Execute()which is not used yet * rename padding_l/_r/pl/pr to padding_left or padding_right as recommended * parameter and variable renaming - to make them more explicit --- tensorflow/core/kernels/mkl_conv_ops.cc | 414 +++++++++++++++++------- tensorflow/core/util/mkl_util.h | 87 ++++- 2 files changed, 389 insertions(+), 112 deletions(-) diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc index f0818eb96d..f2b14f1278 100644 --- a/tensorflow/core/kernels/mkl_conv_ops.cc +++ b/tensorflow/core/kernels/mkl_conv_ops.cc @@ -20,6 +20,7 @@ limitations under the License. #include #include #include +#include #include "tensorflow/core/framework/numeric_op.h" #include "tensorflow/core/framework/op_kernel.h" @@ -42,14 +43,13 @@ limitations under the License. #include "tensorflow/core/util/mkl_util.h" #ifndef INTEL_MKL_ML - #include "mkldnn.hpp" using mkldnn::prop_kind; using mkldnn::stream; - -using mkldnn::convolution_direct; using mkldnn::convolution_forward; +using mkldnn::convolution_direct; + #else #include "mkl_dnn.h" #include "mkl_dnn_types.h" @@ -57,11 +57,232 @@ using mkldnn::convolution_forward; namespace tensorflow { +#ifndef INTEL_MKL_ML + +struct ConvFwdDimensions { + memory::dims src_dims; + memory::dims filter_dims; + memory::dims bias_dims; + memory::dims dst_dims; + memory::dims strides; + memory::dims dilations; + memory::dims padding_left; + memory::dims padding_right; + + ConvFwdDimensions(memory::dims src_dims, + memory::dims filter_dims, memory::dims bias_dims, + memory::dims dst_dims, memory::dims strides, + memory::dims dilations, memory::dims padding_left, + memory::dims padding_right) : + src_dims(src_dims), filter_dims(filter_dims), + bias_dims(bias_dims), dst_dims(dst_dims), + strides(strides), dilations(dilations), + padding_left(padding_left), padding_right(padding_right) { + } +}; + +template +class Conv2DFwd : public DnnOp { + public: + explicit Conv2DFwd(const ConvFwdDimensions& convFwdDims) { + fwd_stream_.reset(new stream(stream::kind::eager)); + // create conv primitive + if (conv_fwd_ == nullptr) { + Setup(convFwdDims); + } + } + + ~Conv2DFwd() {} + + // Convolution forward execute with bias + // src_data: input data buffer of src + // filter_data: input data buffer of filter (weights) + // bias_data: input data buffer of bias + // dst_data: output data buffer of dst + void Execute(T* src_data, T* filter_data, T* bias_data, T* dst_data) { + src_mem_->set_data_handle(static_cast(src_data)); + filter_mem_->set_data_handle(static_cast(filter_data)); + bias_mem_->set_data_handle(static_cast(bias_data)); + dst_mem_->set_data_handle(static_cast(dst_data)); + fwd_stream_->submit(fwd_primitives_); + + // after exec, set data handle back + src_mem_->set_data_handle(DummyData); + filter_mem_->set_data_handle(DummyData); + bias_mem_->set_data_handle(DummyData); + dst_mem_->set_data_handle(DummyData); + + return; + } + + // Convolution forward execute without bias + // src_data: input data buffer of src + // filter_data: input data buffer of filter (weights) + // dst_data: output data buffer of dst + void Execute(T* src_data, T* filter_data, T* dst_data) { + src_mem_->set_data_handle(static_cast(src_data)); + filter_mem_->set_data_handle(static_cast(filter_data)); + dst_mem_->set_data_handle(static_cast(dst_data)); + fwd_stream_->submit(fwd_primitives_); + + // after exec, set data handle back + src_mem_->set_data_handle(DummyData); + filter_mem_->set_data_handle(DummyData); + dst_mem_->set_data_handle(DummyData); + + return; + } + + // expected memory format for this primitive instance + memory::format src_fmt_; + memory::format filter_fmt_; + + // convolution primitive + std::shared_ptr fwd_pd_; + std::shared_ptr conv_fwd_; + + private: + void Setup(const ConvFwdDimensions& convFwdDims) { + // create memory descriptors for convolution data w/ no specified format + src_md_.reset(new memory::desc({convFwdDims.src_dims}, + MklDnnType(), memory::format::any)); + + filter_md_.reset(new memory::desc({convFwdDims.filter_dims}, + MklDnnType(), memory::format::any)); + + dst_md_.reset(new memory::desc({convFwdDims.dst_dims}, + MklDnnType(), memory::format::any)); + + if (!convFwdDims.bias_dims.empty()) + bias_md_.reset(new memory::desc({convFwdDims.bias_dims}, + MklDnnType(), memory::format::any)); + + // create a convolution + if (!convFwdDims.bias_dims.empty()) { + fwd_desc_.reset(new convolution_forward::desc(prop_kind::forward, + convolution_direct, *src_md_, *filter_md_, *bias_md_, *dst_md_, + convFwdDims.strides, convFwdDims.dilations, convFwdDims.padding_left, + convFwdDims.padding_right, padding_kind::zero)); + } else { + fwd_desc_.reset(new convolution_forward::desc(prop_kind::forward, + convolution_direct, *src_md_, *filter_md_, *dst_md_, + convFwdDims.strides, convFwdDims.dilations, convFwdDims.padding_left, + convFwdDims.padding_right, padding_kind::zero)); + } + + fwd_pd_.reset(new convolution_forward::primitive_desc( + *fwd_desc_, cpu_engine_)); + + // store the expected memory format + src_fmt_ = static_cast( + fwd_pd_.get()->src_primitive_desc().desc().data.format); + + filter_fmt_ = static_cast( + fwd_pd_.get()->weights_primitive_desc().desc().data.format); + + // create memory primitive based on dummy data + src_mem_.reset(new memory(fwd_pd_.get()->src_primitive_desc(), DummyData)); + filter_mem_.reset(new memory(fwd_pd_.get()->weights_primitive_desc(), + DummyData)); + dst_mem_.reset(new memory(fwd_pd_.get()->dst_primitive_desc(), DummyData)); + + // create convolution primitive and add it to net + if (!convFwdDims.bias_dims.empty()) { + bias_mem_.reset(new memory({{{convFwdDims.bias_dims}, MklDnnType(), + memory::format::x}, cpu_engine_}, DummyData)); + conv_fwd_.reset(new convolution_forward(*fwd_pd_, *src_mem_, + *filter_mem_, *bias_mem_, *dst_mem_)); + } else { + conv_fwd_.reset(new convolution_forward(*fwd_pd_, *src_mem_, + *filter_mem_, *dst_mem_)); + } + + fwd_primitives_.push_back(*conv_fwd_); + return; + } + + // MKLDNN memory + std::shared_ptr src_mem_; + std::shared_ptr filter_mem_; + std::shared_ptr bias_mem_; + std::shared_ptr dst_mem_; + + std::shared_ptr fwd_stream_; + std::vector fwd_primitives_; + + // desc & prmitive desc + std::shared_ptr fwd_desc_; + + // memory desc + std::shared_ptr src_md_; + std::shared_ptr filter_md_; + std::shared_ptr bias_md_; + std::shared_ptr dst_md_; + + engine cpu_engine_ = engine(engine::cpu, 0); +}; + +template +class Conv2DFwdFactory : public DnnOpFactory { + public: + static Conv2DFwd* Get(const ConvFwdDimensions& convFwdDims) { + Conv2DFwd* conv2d_fwd = nullptr; + + // try to find a suitable one in pool + conv2d_fwd = dynamic_cast*> ( + Conv2DFwdFactory::GetInstance().GetConv2DFwd(convFwdDims)); + + if (conv2d_fwd == nullptr) { + conv2d_fwd = new Conv2DFwd(convFwdDims); + Conv2DFwdFactory::GetInstance().SetConv2DFwd( + convFwdDims, conv2d_fwd); + } + return conv2d_fwd; + } + + private: + Conv2DFwdFactory() {} + ~Conv2DFwdFactory() {} + + static const int kDilationH = 0, kDilationW = 1; + + static Conv2DFwdFactory& GetInstance() { + static Conv2DFwdFactory instance_; + return instance_; + } + + static std::string CreateKey(const ConvFwdDimensions& convFwdDims) { + std::string prefix = "conv2d_fwd_"; + FactoryKeyCreator key_creator; + key_creator.AddAsKey(prefix); + key_creator.AddAsKey(convFwdDims.src_dims); + key_creator.AddAsKey(convFwdDims.filter_dims); + key_creator.AddAsKey(convFwdDims.bias_dims); + key_creator.AddAsKey(convFwdDims.dst_dims); + key_creator.AddAsKey(convFwdDims.strides); + key_creator.AddAsKey(convFwdDims.dilations); + key_creator.AddAsKey(convFwdDims.padding_left); + key_creator.AddAsKey(convFwdDims.padding_right); + return key_creator.GetKey(); + } + + DnnOp* GetConv2DFwd(const ConvFwdDimensions& convFwdDims) { + std::string key = CreateKey(convFwdDims); + return this->GetOp(key); + } + + void SetConv2DFwd(const ConvFwdDimensions& convFwdDims, DnnOp *op) { + std::string key = CreateKey(convFwdDims); + this->SetOp(key, op); + } +}; + +#endif + typedef Eigen::ThreadPoolDevice CPUDevice; -// MKL-DNN is now default. MKL-ML must be specified explicitly. +// For now, MKL-ML is default. So making MKL-DNN not a default choice. #ifdef INTEL_MKL_ML - template class MklConv2DOp : public OpKernel { public: @@ -528,8 +749,6 @@ class MklConv2DOp : public OpKernel { void Compute(OpKernelContext* context) override { try { - auto cpu_engine = engine(engine::cpu, 0); - // Input tensors const Tensor& src_tensor = MklGetInput(context, kInputIndex_Src); const Tensor& filter_tensor = MklGetInput(context, kInputIndex_Filter); @@ -538,16 +757,16 @@ class MklConv2DOp : public OpKernel { GetMklShape(context, kInputIndex_Src, &src_mkl_shape); GetMklShape(context, kInputIndex_Filter, &filter_mkl_shape); OP_REQUIRES(context, filter_mkl_shape.IsMklTensor() == false, - errors::InvalidArgument("Filter should not be in " - "Mkl Layout")); + errors::InvalidArgument("Filter should not be in " + "Mkl Layout")); MklDnnData src(&cpu_engine); MklDnnData filter(&cpu_engine); - MklDnnData output(&cpu_engine); + MklDnnData dst(&cpu_engine); // output - memory::dims src_dims, filter_dims, padding_l, padding_r, + memory::dims src_dims, filter_dims, padding_left, padding_right, dilations, strides; - memory::dims output_dims_tf_order, output_dims_mkl_order; + memory::dims dst_dims_tf_order, dst_dims_mkl_order; // Get shapes of input tensors in MKL-DNN order MklDnnConvUtil conv_utl(context, strides_, padding_, data_format_, @@ -555,31 +774,29 @@ class MklConv2DOp : public OpKernel { auto src_tf_shape = GetTfShape(context, kInputIndex_Src); auto filter_tf_shape = GetTfShape(context, kInputIndex_Filter); conv_utl.GetConvFwdSizesInMklOrder( - src_tf_shape, filter_tf_shape, &src_dims, &filter_dims, &strides, - &dilations, &output_dims_tf_order, &output_dims_mkl_order, - &padding_l, &padding_r); + src_tf_shape, filter_tf_shape, &src_dims, &filter_dims, + &strides, &dilations, &dst_dims_tf_order, &dst_dims_mkl_order, + &padding_left, &padding_right); if (!context->status().ok()) return; // Check for corner case - if there is nothing to compute, return. - TensorShape output_tf_shape = MklDnnDimsToTFShape(output_dims_tf_order); + TensorShape dst_tf_shape = MklDnnDimsToTFShape(dst_dims_tf_order); // Corner cases: output with 0 elements and 0 batch size. - Tensor* output_tensor = nullptr; - if (output_tf_shape.num_elements() == 0 || output_dims_tf_order[0] == 0) { - // TODO(jbobba): Verify correctness here - // Need semantics for Null MKL tensor - MklDnnShape output_mkl_shape; - output_mkl_shape.SetMklTensor(false); - - AllocateOutputSetMklShape(context, kOutputIndex_Dst, &output_tensor, - src_tf_shape, output_mkl_shape); + Tensor* dst_tensor = nullptr; + if (dst_tf_shape.num_elements() == 0 || + dst_dims_tf_order[0] == 0) { + MklDnnShape dst_mkl_shape; + dst_mkl_shape.SetMklTensor(false); + AllocateOutputSetMklShape(context, kOutputIndex_Dst, + &dst_tensor, src_tf_shape, dst_mkl_shape); // MklConv2D also outputs converted filter as 2nd output of Conv2D. filter_mkl_shape.SetMklTensor(false); Tensor* output_filter_tensor = nullptr; AllocateOutputSetMklShape(context, kOutputIndex_Filter, - &output_filter_tensor, filter_tf_shape, - filter_mkl_shape); + &output_filter_tensor, + filter_tf_shape, filter_mkl_shape); return; } @@ -587,6 +804,7 @@ class MklConv2DOp : public OpKernel { // Describe how the inputs and outputs of Convolution look like. Also // specify buffers containing actual input and output data. auto tf_fmt = TFDataFormatToMklDnnDataFormat(data_format_); + // If input is in MKL layout, then simply grab input layout; otherwise, // construct input Tf layout. For TF layout, although input shape // (src_dims) required is in MKL-DNN order, the layout is Tensorflow's @@ -595,6 +813,7 @@ class MklConv2DOp : public OpKernel { ? src_mkl_shape.GetMklLayout() : memory::desc(src_dims, MklDnnType(), tf_fmt); src.SetUsrMem(src_md, &src_tensor); + // Although filter shape (filter_dims) required is in MKL-DNN order, // the layout is Tensorflow's layout (HWIO). auto filter_md = filter_mkl_shape.IsMklTensor() // Should NEVER be true @@ -603,98 +822,70 @@ class MklConv2DOp : public OpKernel { memory::format::hwio); filter.SetUsrMem(filter_md, &filter_tensor); - // Set output shape (output_dims) required in MKL-DNN order. - // Currently, we set output layout as Tensorflow's layout (NHWC or NCHW - // depending on data format). But later we propagate Mkl layout of the - // output to the next op directly. - output.SetUsrMem(output_dims_mkl_order, tf_fmt); - - // Create memory descriptors for convolution data w/ no specified format. - src.SetOpMemDesc(src_dims, memory::format::any); - filter.SetOpMemDesc(filter_dims, memory::format::any); - output.SetOpMemDesc(output_dims_mkl_order, memory::format::any); - // MKLDNN dilation starts from 0. dilations[kDilationH] -= 1; dilations[kDilationW] -= 1; + // get a conv2d fwd from primitive pool + Conv2DFwd *conv2d_fwd = nullptr; + if (biasEnabled) { + memory::dims bias_dims = {}; + conv_utl.GetBiasSizeInMklOrder(kInputIndex_Bias, &bias_dims); + ConvFwdDimensions convFwdDims(src_dims, filter_dims, bias_dims, + dst_dims_mkl_order, strides, dilations, padding_left, padding_right); + conv2d_fwd = Conv2DFwdFactory::Get(convFwdDims); + } else { + ConvFwdDimensions convFwdDims(src_dims, filter_dims, NONE_DIMS, + dst_dims_mkl_order, strides, dilations, padding_left, padding_right); + conv2d_fwd = Conv2DFwdFactory::Get(convFwdDims); + } + + // allocate output tensors output_tensor and filter_out_tensor + std::shared_ptr + conv_fwd_pd = conv2d_fwd->fwd_pd_; + AllocateOutputTensor(context, *conv_fwd_pd, + dst_dims_mkl_order, tf_fmt, &dst_tensor); + Tensor* filter_out_tensor = nullptr; + AllocateFilterOutputTensor(context, *conv_fwd_pd, + TFShapeToMklDnnDims(filter_tf_shape), + &filter_out_tensor); + + T* dst_data = static_cast(dst_tensor->flat().data()); + + // check whether src/filter need reorder + std::vector net; + if (src_md.data.format != conv2d_fwd->src_fmt_) + src.CheckReorderToOpMem( + conv_fwd_pd.get()->src_primitive_desc(), &net); + + if (filter_md.data.format != conv2d_fwd->filter_fmt_) + filter.CheckReorderToOpMem( + conv_fwd_pd.get()->weights_primitive_desc(), + filter.GetTensorBuffer(filter_out_tensor), &net); + stream(stream::kind::eager).submit(net).wait(); + + T* src_data = static_cast( + src.GetOpMem().get_data_handle()); + T* filter_data = static_cast( + filter.GetOpMem().get_data_handle()); + + // execute convolution if (biasEnabled) { - // Create convolution primitive with Bias. - MklDnnData bias(&cpu_engine); - memory::dims bias_size; - conv_utl.GetBiasSizeInMklOrder(kInputIndex_Bias, &bias_size); - const Tensor& bias_tensor = MklGetInput(context, kInputIndex_Bias); - bias.SetUsrMem(bias_size, memory::format::x, &bias_tensor); - bias.SetOpMemDesc(bias_size, memory::format::any); - - // Create convolution primitive with Bias. - // Use MKLDNN dilated convolution in case of dilated rate (>0). - auto conv_desc = (dilations[kDilationH] > 0 || - dilations[kDilationW] > 0) ? - convolution_forward::desc(prop_kind::forward, - convolution_direct, src.GetOpMemDesc(), - filter.GetOpMemDesc(), bias.GetOpMemDesc(), - output.GetOpMemDesc(), strides, dilations, - padding_l, padding_r, - TFPaddingToMklDnnPadding(padding_)): - convolution_forward::desc(prop_kind::forward, - convolution_direct, src.GetOpMemDesc(), - filter.GetOpMemDesc(), bias.GetOpMemDesc(), - output.GetOpMemDesc(), strides, - padding_l, padding_r, - TFPaddingToMklDnnPadding(padding_)); - - auto conv_prim_desc = convolution_forward::primitive_desc(conv_desc, - cpu_engine); - AllocateOutputTensor(context, conv_prim_desc, - output_dims_mkl_order, tf_fmt, &output_tensor); - // Set data handle for output. - output.SetUsrMemDataHandle(output_tensor); - - Tensor* filter_out_tensor = nullptr; - AllocateFilterOutputTensor(context, conv_prim_desc, - TFShapeToMklDnnDims(filter_tf_shape), - &filter_out_tensor); - - PrepareAndExecuteNet(conv_prim_desc, &src, &filter, &bias, &output, - filter_out_tensor); + const Tensor& bias_tensor = MklGetInput(context, kInputIndex_Bias); + T* bias_data = static_cast(const_cast( + bias_tensor.flat().data())); + + conv2d_fwd->Execute(src_data, filter_data, bias_data, dst_data); } else { - // Create convolution primitive without Bias. - // Use MKLDNN dilated convolution in case of dilated rate (>0). - auto conv_desc = (dilations[kDilationH] > 0 || - dilations[kDilationW] > 0) ? - convolution_forward::desc(prop_kind::forward, - convolution_direct, src.GetOpMemDesc(), - filter.GetOpMemDesc(), output.GetOpMemDesc(), - strides, dilations, padding_l, padding_r, - TFPaddingToMklDnnPadding(padding_)): - convolution_forward::desc(prop_kind::forward, - convolution_direct, src.GetOpMemDesc(), - filter.GetOpMemDesc(), output.GetOpMemDesc(), - strides, padding_l, padding_r, - TFPaddingToMklDnnPadding(padding_)); - - auto conv_prim_desc = convolution_forward::primitive_desc(conv_desc, - cpu_engine); - AllocateOutputTensor(context, conv_prim_desc, output_dims_mkl_order, - tf_fmt, &output_tensor); - // Set data handle for output. - output.SetUsrMemDataHandle(output_tensor); - - Tensor* filter_out_tensor = nullptr; - AllocateFilterOutputTensor(context, conv_prim_desc, - TFShapeToMklDnnDims(filter_tf_shape), - &filter_out_tensor); - PrepareAndExecuteNet(conv_prim_desc, &src, &filter, - nullptr, &output, filter_out_tensor); + conv2d_fwd->Execute(src_data, filter_data, dst_data); } - } catch (mkldnn::error& e) { + } catch (mkldnn::error &e) { string error_msg = "Status: " + std::to_string(e.status) + - ", message: " + std::string(e.message) + ", in file " + - std::string(__FILE__) + ":" + std::to_string(__LINE__); - OP_REQUIRES_OK( - context, - errors::Aborted("Operation received an exception:", error_msg)); + ", message: " + std::string(e.message) + + ", in file " + std::string(__FILE__) + ":" + + std::to_string(__LINE__); + OP_REQUIRES_OK(context, + errors::Aborted("Operation received an exception:", error_msg)); } } @@ -706,6 +897,7 @@ class MklConv2DOp : public OpKernel { const int kInputIndex_Src = 0, kInputIndex_Filter = 1, kInputIndex_Bias = 2; const int kOutputIndex_Dst = 0, kOutputIndex_Filter = 1; const int kDilationH = 0, kDilationW = 1; + engine cpu_engine = engine(engine::cpu, 0); // Allocate output tensor. void AllocateOutputTensor( diff --git a/tensorflow/core/util/mkl_util.h b/tensorflow/core/util/mkl_util.h index bc6d2d77a4..50a8e30574 100644 --- a/tensorflow/core/util/mkl_util.h +++ b/tensorflow/core/util/mkl_util.h @@ -19,6 +19,8 @@ limitations under the License. #include #include +#include +#include #include "mkl_dnn.h" #include "mkl_dnn_types.h" @@ -1759,7 +1761,90 @@ class MklDnnData { } }; -#endif // INTEL_MKL_ML +/// Base class for operations with reuse of DNN primitives +/// +class DnnOp { + public: + virtual ~DnnOp() {} + + // Dummy data. Its size, hard-coded as 256 here, does + // not matter since MKL should never operate on this buffer. + unsigned char DummyData[256]; +}; + +const mkldnn::memory::dims NONE_DIMS = {}; +// This constant is used to declare dummy buffer (size), for MKL primitives +template +class DnnOpFactory { + public: + DnnOpFactory() {} + ~DnnOpFactory() {} + + DnnOp* GetOp(const std::string& key) { + auto stream_iter = DnnOpFactory::GetHashMap().find(key); + if (stream_iter == DnnOpFactory::GetHashMap().end()) { + return nullptr; + } else { + return stream_iter->second; + } + } + + void SetOp(const std::string& key, DnnOp* op) { + auto stream_iter = DnnOpFactory::GetHashMap().find(key); + + CHECK(stream_iter == DnnOpFactory::GetHashMap().end()); + + DnnOpFactory::GetHashMap()[key] = op; + } + + private: + static inline std::unordered_map &GetHashMap() { + static thread_local std::unordered_map map_; + return map_; + } +}; + +// utility class for creating keys of MKL primitive pool. +class FactoryKeyCreator { + public: + FactoryKeyCreator() { + key_.reserve(kMaxKeyLength); + } + + ~FactoryKeyCreator() {} + + void AddAsKey(const string &str) { + auto buffer = reinterpret_cast(str.c_str()); + Append(buffer, str.length()); + } + + void AddAsKey(const mkldnn::memory::dims &dims) { + for (unsigned int i = 0; i < dims.size(); i++) { + AddAsKey(dims[i]); + } + } + + template + void AddAsKey(const T data) { + auto buffer = reinterpret_cast(&data); + Append(buffer, sizeof(T)); + } + + std::string GetKey() { + return key_; + } + + private: + string key_; + const char delimiter = 'x'; + const int kMaxKeyLength = 256; + void Append(const char* data, int len) { + key_.append(data, len); + key_.append(1, delimiter); + } +}; + +#endif // INTEL_MKL_DNN } // namespace tensorflow #endif // INTEL_MKL -- GitLab From 99167d3a6393ac47c2e01b6f620a03adeb9ac3e4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Apr 2018 13:48:37 -0700 Subject: [PATCH 756/791] Merged commit includes the following changes: 193717076 by yifeif: Automated g4 rollback of changelist 193713153. -- 193716750 by fchollet: Refactor `tf.keras.layers.Embedding` layer to use `embedding_lookup` instead of `gather`. This makes the layer TPU-compatible. -- 193716664 by A. Unique TensorFlower: Go: Update generated wrapper functions for TensorFlow ops. -- 193713153 by power: Experimental Keras TPU compatibility layer. -- PiperOrigin-RevId: 193717076 --- tensorflow/go/op/wrappers.go | 32 +++++++++++++++++-- tensorflow/python/keras/BUILD | 1 + .../keras/_impl/keras/layers/embeddings.py | 4 +-- .../_impl/keras/layers/embeddings_test.py | 13 ++++++++ 4 files changed, 46 insertions(+), 4 deletions(-) diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index 3b3dff0573..ec7d9dcc4f 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -5917,6 +5917,17 @@ func Sqrt(scope *Scope, x tf.Output) (y tf.Output) { return op.Output(0) } +// Conv3DBackpropFilterAttr is an optional argument to Conv3DBackpropFilter. +type Conv3DBackpropFilterAttr func(optionalAttr) + +// Conv3DBackpropFilterDilations sets the optional dilations attribute to value. +// If not specified, defaults to +func Conv3DBackpropFilterDilations(value []int64) Conv3DBackpropFilterAttr { + return func(m optionalAttr) { + m["dilations"] = value + } +} + // Computes the gradients of 3-D convolution with respect to the filter. // // DEPRECATED at GraphDef version 10: Use Conv3DBackpropFilterV2 @@ -5930,11 +5941,14 @@ func Sqrt(scope *Scope, x tf.Output) (y tf.Output) { // strides: 1-D tensor of length 5. The stride of the sliding window for each // dimension of `input`. Must have `strides[0] = strides[4] = 1`. // padding: The type of padding algorithm to use. -func Conv3DBackpropFilter(scope *Scope, input tf.Output, filter tf.Output, out_backprop tf.Output, strides []int64, padding string) (output tf.Output) { +func Conv3DBackpropFilter(scope *Scope, input tf.Output, filter tf.Output, out_backprop tf.Output, strides []int64, padding string, optional ...Conv3DBackpropFilterAttr) (output tf.Output) { if scope.Err() != nil { return } attrs := map[string]interface{}{"strides": strides, "padding": padding} + for _, a := range optional { + a(attrs) + } opspec := tf.OpSpec{ Type: "Conv3DBackpropFilter", Input: []tf.Input{ @@ -12306,6 +12320,17 @@ func MaxPool3D(scope *Scope, input tf.Output, ksize []int64, strides []int64, pa return op.Output(0) } +// Conv3DBackpropInputAttr is an optional argument to Conv3DBackpropInput. +type Conv3DBackpropInputAttr func(optionalAttr) + +// Conv3DBackpropInputDilations sets the optional dilations attribute to value. +// If not specified, defaults to +func Conv3DBackpropInputDilations(value []int64) Conv3DBackpropInputAttr { + return func(m optionalAttr) { + m["dilations"] = value + } +} + // Computes the gradients of 3-D convolution with respect to the input. // // DEPRECATED at GraphDef version 10: Use Conv3DBackpropInputV2 @@ -12319,11 +12344,14 @@ func MaxPool3D(scope *Scope, input tf.Output, ksize []int64, strides []int64, pa // strides: 1-D tensor of length 5. The stride of the sliding window for each // dimension of `input`. Must have `strides[0] = strides[4] = 1`. // padding: The type of padding algorithm to use. -func Conv3DBackpropInput(scope *Scope, input tf.Output, filter tf.Output, out_backprop tf.Output, strides []int64, padding string) (output tf.Output) { +func Conv3DBackpropInput(scope *Scope, input tf.Output, filter tf.Output, out_backprop tf.Output, strides []int64, padding string, optional ...Conv3DBackpropInputAttr) (output tf.Output) { if scope.Err() != nil { return } attrs := map[string]interface{}{"strides": strides, "padding": padding} + for _, a := range optional { + a(attrs) + } opspec := tf.OpSpec{ Type: "Conv3DBackpropInput", Input: []tf.Input{ diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD index 70040b7e74..1c58553156 100755 --- a/tensorflow/python/keras/BUILD +++ b/tensorflow/python/keras/BUILD @@ -208,6 +208,7 @@ py_library( "//tensorflow/python:array_ops", "//tensorflow/python:distribute", "//tensorflow/python:dtypes", + "//tensorflow/python:embedding_ops", "//tensorflow/python:framework_ops", "//tensorflow/python:logging_ops", "//tensorflow/python:math_ops", diff --git a/tensorflow/python/keras/_impl/keras/layers/embeddings.py b/tensorflow/python/keras/_impl/keras/layers/embeddings.py index 591bab7cd8..07b8726b85 100644 --- a/tensorflow/python/keras/_impl/keras/layers/embeddings.py +++ b/tensorflow/python/keras/_impl/keras/layers/embeddings.py @@ -24,7 +24,7 @@ from tensorflow.python.keras._impl.keras import initializers from tensorflow.python.keras._impl.keras import regularizers from tensorflow.python.keras._impl.keras.engine import Layer from tensorflow.python.keras._impl.keras.engine.base_layer import shape_type_conversion -from tensorflow.python.ops import array_ops +from tensorflow.python.ops import embedding_ops from tensorflow.python.ops import math_ops from tensorflow.python.util.tf_export import tf_export @@ -155,7 +155,7 @@ class Embedding(Layer): def call(self, inputs): if K.dtype(inputs) != 'int32': inputs = math_ops.cast(inputs, 'int32') - out = array_ops.gather(self.embeddings, inputs) + out = embedding_ops.embedding_lookup(self.embeddings, inputs) return out def get_config(self): diff --git a/tensorflow/python/keras/_impl/keras/layers/embeddings_test.py b/tensorflow/python/keras/_impl/keras/layers/embeddings_test.py index 9f6793eac8..6ebf5dc94a 100644 --- a/tensorflow/python/keras/_impl/keras/layers/embeddings_test.py +++ b/tensorflow/python/keras/_impl/keras/layers/embeddings_test.py @@ -18,6 +18,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import numpy as np + from tensorflow.python.framework import test_util as tf_test_util from tensorflow.python.keras._impl import keras from tensorflow.python.keras._impl.keras import testing_utils @@ -65,6 +67,17 @@ class EmbeddingTest(test.TestCase): input_dtype='int32', expected_output_dtype='float32') + def test_embedding_correctness(self): + with self.test_session(): + layer = keras.layers.Embedding(output_dim=2, input_dim=2) + layer.build((None, 2)) + matrix = np.array([[1, 1], [2, 2]]) + layer.set_weights([matrix]) + + inputs = keras.backend.constant([[0, 1, 0]], dtype='int32') + outputs = keras.backend.eval(layer(inputs)) + self.assertAllClose(outputs, [[[1, 1], [2, 2], [1, 1]]]) + if __name__ == '__main__': test.main() -- GitLab From 5a4356be6822dfe0b0f973852b9b65d69e4c169c Mon Sep 17 00:00:00 2001 From: Brian Patton Date: Fri, 20 Apr 2018 13:54:00 -0700 Subject: [PATCH 757/791] Fix for: Suggest braces around initialization of subobject. PiperOrigin-RevId: 193717872 --- tensorflow/python/lib/core/bfloat16.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/lib/core/bfloat16.cc b/tensorflow/python/lib/core/bfloat16.cc index 7f07deebef..77fa2c1f66 100644 --- a/tensorflow/python/lib/core/bfloat16.cc +++ b/tensorflow/python/lib/core/bfloat16.cc @@ -616,8 +616,8 @@ bool Initialize() { }; // Comparisons - const std::array compare_types = {npy_bfloat16_, npy_bfloat16_, - NPY_BOOL}; + const std::array compare_types = { + {npy_bfloat16_, npy_bfloat16_, NPY_BOOL}}; if (!register_ufunc("equal", CompareUFunc, compare_types)) { -- GitLab From 1cd64d57143814fc0652c09165735be62d96124f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Apr 2018 13:56:55 -0700 Subject: [PATCH 758/791] Track dependencies between outside_compilation clusters so that control edges can be correctly added to sequence compiled computations. PiperOrigin-RevId: 193718295 --- .../jit/encapsulate_subgraphs_pass.cc | 378 ++++++++++- .../jit/encapsulate_subgraphs_pass_test.cc | 590 +++++++++++++++++- tensorflow/compiler/tf2xla/xla_compiler.cc | 25 + tensorflow/compiler/tf2xla/xla_compiler.h | 20 + 4 files changed, 1005 insertions(+), 8 deletions(-) diff --git a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc index 9465385b58..7507e193b5 100644 --- a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc +++ b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc @@ -23,6 +23,7 @@ limitations under the License. #include #include "tensorflow/compiler/jit/graph_to_functiondef.h" +#include "tensorflow/compiler/jit/graphcycles/graphcycles.h" #include "tensorflow/compiler/jit/legacy_flags/encapsulate_subgraphs_pass_flags.h" #include "tensorflow/compiler/jit/mark_for_compilation_pass.h" #include "tensorflow/compiler/jit/shape_inference_helpers.h" @@ -160,6 +161,11 @@ class Encapsulator { std::move(outside_compilation_attribute)), graph_in_(graph_in) {} + // Find dependencies between subgraphs and outside_compilation clusters that + // only manifest via edges between outside_compilation clusters in the outer + // (non-compiled) graph. + Status FindClusterDependencies(); + // Find subgraphs marked with 'group_attribute', and build a new // subgraph, one for each value of 'group_attribute'. Status SplitIntoSubgraphs(); @@ -230,6 +236,19 @@ class Encapsulator { // the shapes of any ancestor RAH outputs. If it can be determined that the // shape of the SFH inputs will not be inferrable even once the shapes of the // RAH outputs are known, an error is returned by the rewriter. + // + // Once edges between compiled and outside_compilation clusters have been + // replaced by send/recv ops, some dependencies may no longer be apparent. + // A clustering pass finds all the dependencies between HC nodes that are only + // present as a result of edges between nodes in outside_compilaton clusters. + // Suppose there is a path from outside_compilation cluster C in subgraph S + // to outside_compilation cluster D in subgraph T. If S != T then a control + // edge is added from the call node for S to the call node for T, which + // ensures that C will execute before D because S executes before T. If S==T + // then a control dependency is added between the HC nodes for C and D in S, + // and the HC node for C is added to an 'ancestors' attr in the HC node for D + // so that during compilation of the HC node for D, an XLA control dependency + // can be added to ensure C's SendToHost executes before D's RecvFromHost. class Subgraph { public: // Creates a graph to build the subgraph in, if it doesn't already exist, @@ -324,6 +343,18 @@ class Encapsulator { void RecordOutsideCompilationOutputOrControl( const string& outside_compilation_id, const Edge* edge); + // Records the fact that there is a path from a node in outside_compilation + // cluster ancestor to node in cluster successor that does not go through + // the subgraph. + void RecordOutsideCompilationDependency(const string& successor, + const string& ancestor); + + // Returns the mapping from outside_compilation cluster C to the set of + // outside_compilation clusters that have a path to C entirely outside + // compiled subgraphs. + const std::unordered_map> + OutsideCompilationAncestorMap() const; + // Adds the HostCompute nodes for each outside_compilation subgraph. Status AddHostComputes( const string& subgraph_name, @@ -406,6 +437,13 @@ class Encapsulator { Status AddHostComputeKeyPlaceholder(OutsideCompilationSubgraph* oc_subgraph, Graph* graph_out); + // Get the set of outside_compilation clusters and the dependency edges + // between them. + void GetActiveClusterDependencyGraph( + std::unordered_set* clusters, + std::unordered_set* has_successor, + std::unordered_map>* ancestors_map); + // Builds a _RecvAtHost node producing all the inputs of an // outside_compilation subgraph and stores it in oc_subgraph.recv_at_host. Status AddRecvAtHostNode(const string& group_attribute, @@ -468,6 +506,14 @@ class Encapsulator { // The outside_compilation clusters in this subgraph. std::unordered_map outside_compilation_subgraphs_; + // For each outside_compilation cluster C, the outside_compilation clusters + // that have a path to C outside the compiled graph. + std::unordered_map> + outside_compilation_ancestors_; + // For each outside_compilation cluster C, the outside_compilation clusters + // that have a path from C outside the compiled graph. + std::unordered_map> + outside_compilation_successors_; // NoOp node in the output graph that is sequenced after the call node and // used to prevent host-side outside_compilation sends and recvs from being @@ -556,6 +602,10 @@ class Encapsulator { std::unordered_set, NodeSlot::PairHasher>* edges_added); + // Adds control dependencies between subgraph call nodes that have + // dependencies via outside_compilation edges. + Status AddCallNodeDependencies(Graph* graph_out); + // Adds all edges to the output graph. Status AddEdgesToOutputGraph( const std::unordered_map& node_images, @@ -620,10 +670,65 @@ class Encapsulator { const Graph* graph_in_; std::unordered_map subgraphs_; + // For each subgraph S the subgraphs S' such that there is a path in some + // outside_compilation cluster C in S to some outside_compilation cluster C' + // in S', that goes only through the uncompiled graph. + std::unordered_map> subgraph_ancestors_; TF_DISALLOW_COPY_AND_ASSIGN(Encapsulator); }; +namespace { + +// Return in 'sorted' a topological sort of clusters according to the +// dependencies encoded in ancestors. clusters is the list of all clusters +// including clusters that are not present in the ancestors map. has_successors +// is the set of clusters that are ancestors of some other cluster. +void TopologicalClusterSort( + const std::unordered_set& clusters, + const std::unordered_set& has_successors, + const std::unordered_map>& ancestors, + std::vector* sorted) { + // The nodes are placed in 'sorted' in topological order. + sorted->clear(); + // We don't use the standard DFS because we are not operating on Node* + // objects. + struct Work { + string cluster; + bool leave; + }; + std::set visited; + std::vector stack; + // Seed the processing list with clusters that have no successors. + for (const auto& cluster : clusters) { + if (has_successors.find(cluster) == has_successors.end()) { + stack.push_back({cluster, false}); + } + } + while (!stack.empty()) { + const Work item = stack.back(); + stack.pop_back(); + if (item.leave) { + sorted->push_back(item.cluster); + continue; + } + + if (visited.find(item.cluster) != visited.end()) continue; + visited.insert(item.cluster); + + stack.push_back({item.cluster, true}); + const auto& iter = ancestors.find(item.cluster); + if (iter != ancestors.end()) { + for (const auto& ancestor : iter->second) { + stack.push_back({ancestor, false}); + } + } + } + CHECK(sorted->size() == clusters.size()); +} + +} // namespace + Node* Encapsulator::Subgraph::GetCallNodeForInputs() const { return call_node_inputs_; } @@ -786,12 +891,71 @@ void Encapsulator::Subgraph::RecordOutsideCompilationOutputOrControl( } } +void Encapsulator::Subgraph::RecordOutsideCompilationDependency( + const string& successor, const string& ancestor) { + outside_compilation_ancestors_[successor].insert(ancestor); + outside_compilation_successors_[ancestor].insert(successor); +} + +const std::unordered_map> +Encapsulator::Subgraph::OutsideCompilationAncestorMap() const { + return outside_compilation_ancestors_; +} + +void Encapsulator::Subgraph::GetActiveClusterDependencyGraph( + std::unordered_set* clusters, + std::unordered_set* has_successor, + std::unordered_map>* ancestors_map) { + // During initial clustering the ancestor and successor datastructures may + // have been built including oc_cluster names that never turned into subgraphs + // because they had no edges into or out of the compiled cluster. Remove them + // before proceeding to simplify the logic. Get the set of clusters that was + // actually added, then remove references to the others. + for (const auto& oc_subgraph : outside_compilation_subgraphs_) { + clusters->insert(oc_subgraph.first); + } + for (const auto& cluster : outside_compilation_successors_) { + if (clusters->find(cluster.first) != clusters->end()) { + for (const auto& successor : cluster.second) { + if (clusters->find(successor) != clusters->end()) { + has_successor->insert(cluster.first); + break; + } + } + } + } + for (const auto& cluster : outside_compilation_ancestors_) { + if (clusters->find(cluster.first) != clusters->end()) { + std::unordered_set& ancestors = (*ancestors_map)[cluster.first]; + for (const auto& ancestor : cluster.second) { + if (clusters->find(ancestor) != clusters->end()) { + ancestors.insert(ancestor); + } + } + } + } +} + Status Encapsulator::Subgraph::AddHostComputes( const string& subgraph_name, const std::unordered_map& node_images) { - for (auto& oc_subgraph_iter : outside_compilation_subgraphs_) { - const string& oc_subgraph_name = oc_subgraph_iter.first; - OutsideCompilationSubgraph& oc_subgraph = oc_subgraph_iter.second; + // Get the set of outside_compilation clusters and the dependency edges + // between them. + std::unordered_set clusters; + std::unordered_set has_successor; + std::unordered_map> ancestors_map; + GetActiveClusterDependencyGraph(&clusters, &has_successor, &ancestors_map); + // Topologically sort the outside_compilation clusters according to their + // dependency relation. + std::vector sorted_clusters; + TopologicalClusterSort(clusters, has_successor, ancestors_map, + &sorted_clusters); + + // The host compute nodes added for each outside_compilation_cluster; + std::unordered_map host_compute_node; + for (const string& oc_subgraph_name : sorted_clusters) { + OutsideCompilationSubgraph& oc_subgraph = + outside_compilation_subgraphs_[oc_subgraph_name]; if (!oc_subgraph.inputs.empty() || !oc_subgraph.control_inputs.empty() || !oc_subgraph.outputs_by_src.empty() || !oc_subgraph.control_outputs.empty()) { @@ -811,13 +975,22 @@ Status Encapsulator::Subgraph::AddHostComputes( inputs[input_index].Reset(src_image->name(), src_slot, dtype); input_dtypes[input_index] = dtype; } - for (const auto& output : oc_subgraph.outputs_by_src) { DataType dtype = output.first.dtype; int output_index = output.second; output_dtypes[output_index] = dtype; } + std::vector host_compute_ancestors; + const auto iter = ancestors_map.find(oc_subgraph_name); + if (iter != ancestors_map.end()) { + for (const string& ancestor_cluster : iter->second) { + host_compute_ancestors.push_back( + outside_compilation_subgraphs_[ancestor_cluster] + .host_compute_name); + } + } + NodeDef host_compute_def; NodeDefBuilder builder(strings::StrCat("outside_compilation_", oc_subgraph_name, "_host_compute"), @@ -825,6 +998,7 @@ Status Encapsulator::Subgraph::AddHostComputes( builder.Input(inputs); builder.Attr("Tinputs", input_dtypes); builder.Attr("Toutputs", output_dtypes); + builder.Attr("ancestors", host_compute_ancestors); builder.Attr("key", strings::StrCat("host_compute_channel_", subgraph_name, "_", oc_subgraph_name)); @@ -834,6 +1008,7 @@ Status Encapsulator::Subgraph::AddHostComputes( Node* host_compute = graph_->AddNode(host_compute_def, &s); if (!s.ok()) return s; + host_compute_node[host_compute->name()] = host_compute; oc_subgraph.host_compute_name = host_compute->name(); // Connect the _HostCompute node to its producers in the subgraph. @@ -852,6 +1027,12 @@ Status Encapsulator::Subgraph::AddHostComputes( graph_->AddControlEdge(src_image, host_compute); } + // Connect the _HostCompute node to its ancestor host compute nodes. + for (const auto& ancestor_name : host_compute_ancestors) { + Node* ancestor = host_compute_node[ancestor_name]; + graph_->AddControlEdge(ancestor, host_compute); + } + // Connect the consumers in the subgraph to the _HostCompute node. for (const auto& output : oc_subgraph.outputs_by_dst) { const Node* dst_node = output.first.node; @@ -1654,6 +1835,17 @@ Status Encapsulator::CopyEdgeToOutputGraph( return Status::OK(); } +Status Encapsulator::AddCallNodeDependencies(Graph* graph_out) { + for (const auto& ancestors : subgraph_ancestors_) { + const string& subgraph = ancestors.first; + for (const string& ancestor : ancestors.second) { + graph_out->AddControlEdge(subgraphs_[ancestor].GetCallNodeForOutputs(), + subgraphs_[subgraph].GetCallNodeForInputs()); + } + } + return Status::OK(); +} + Status Encapsulator::AddEdgesToOutputGraph( const std::unordered_map& node_images, bool parallel_checking, Graph* graph_out) { @@ -1703,6 +1895,7 @@ Status Encapsulator::AddEdgesToOutputGraph( Subgraph& subgraph = subgraph_entry.second; subgraph.ConnectSequencerToCallNode(graph_out); } + TF_RETURN_IF_ERROR(AddCallNodeDependencies(graph_out)); return Status::OK(); } @@ -1960,6 +2153,182 @@ Status Encapsulator::DoStaticShapeInferenceForOutsideCompilationSend( return Status::OK(); } +namespace { + +// Helper struct for building cluster dependencies and also debugging cycles in +// the dependencies. While computing dependencies we construct a mapping from +// Node* to PathDetails. +struct PathDetails { + struct SubgraphAndCluster { + string subgraph; + string outside_compilation_cluster; + bool operator==(const SubgraphAndCluster& other) const { + return subgraph == other.subgraph && + outside_compilation_cluster == other.outside_compilation_cluster; + } + }; + + struct SubgraphAndClusterHash { + inline std::size_t operator()(const SubgraphAndCluster& v) const { + return hash()( + strings::StrCat(v.subgraph, v.outside_compilation_cluster)); + } + }; + + typedef std::unordered_set + SubgraphAndClusterSet; + + // Returns the set of (subgraph, oc_cluster) pairs that should be recorded as + // ancestors for any successor of this node. If the node is in the outer + // graph, it returns the transitive union of the ancestors of the node's + // inputs. If the node is in an outside_compilation cluster, it returns just + // that cluster. If the node is compiled, it returns the empty set. + SubgraphAndClusterSet AncestorsForSuccessor() { + if (subgraph.empty()) { + return ancestor_clusters; + } else if (outside_compilation_cluster.empty()) { + return SubgraphAndClusterSet(); + } else { + SubgraphAndCluster entry; + entry.subgraph = subgraph; + entry.outside_compilation_cluster = outside_compilation_cluster; + return SubgraphAndClusterSet({entry}); + } + } + + // The transitive union of the ancestor's of this node's inputs. This is only + // saved for debugging in order to print out enough information to debug a + // discovered cycle. + SubgraphAndClusterSet ancestor_clusters; + // The subgraph attr on this node. + string subgraph; + // The outside_compilation attr on this node. + string outside_compilation_cluster; +}; + +// Adds an edge from ancestor to successor to the cycle detector, and returns an +// error if that edge causes the formation of a cycle. In the error case, logs +// the contents of the node_ancestors_map to facilitate debugging. +Status CheckClusterDependencyForCycles( + const string& ancestor, const string& successor, + const std::unordered_map>& ancestors, + const std::unordered_map& node_ancestors_map, + GraphCycles* cycle_detector, std::map* cycle_detector_map) { + if (cycle_detector_map->find(ancestor) == cycle_detector_map->end()) { + (*cycle_detector_map)[ancestor] = cycle_detector->NewNode(); + } + if (cycle_detector_map->find(successor) == cycle_detector_map->end()) { + (*cycle_detector_map)[successor] = cycle_detector->NewNode(); + } + + if (!cycle_detector->InsertEdge((*cycle_detector_map)[ancestor], + (*cycle_detector_map)[successor])) { + LOG(ERROR) << "Cycle in outside_compilation clusters"; + for (const auto& cluster : ancestors) { + LOG(ERROR) << "Cluster " << cluster.first << " depends on:"; + for (const auto& ancestor : cluster.second) { + LOG(ERROR) << " " << ancestor; + } + } + for (const auto& node_ancestors : node_ancestors_map) { + LOG(ERROR) << "Node " << node_ancestors.first->name() << " (" + << node_ancestors.second.subgraph << ";" + << node_ancestors.second.outside_compilation_cluster + << ") has ancestor clusters:"; + for (const auto& ancestor : node_ancestors.second.ancestor_clusters) { + LOG(ERROR) << " " << ancestor.subgraph << ";" + << ancestor.outside_compilation_cluster; + } + } + return errors::InvalidArgument( + "Can't compile outside_compilation clusters because there is a " + "dependency cycle: see error log for details."); + } + return Status::OK(); +} + +} // namespace + +Status Encapsulator::FindClusterDependencies() { + // Map from nodes to ancestor details. A node is entered into the map if it is + // in a compilation subgraph, and outside_compilation cluster, or appears on a + // path in the outer graph leading from an outside_compilation subgraph. + std::unordered_map node_ancestors_map; + // We check that clusters are acyclic using this cycle detector. + GraphCycles cycle_detector; + // Map from cluster name to cycle detector node id. + std::map cycle_detector_map; + // Process the nodes in topologically-sorted order. + std::vector nodes; + GetReversePostOrder(*graph_in_, &nodes); + for (Node* node : nodes) { + string subgraph_name; + string oc_cluster; + TF_RETURN_IF_ERROR(GetFunctionNameAttr(node, &subgraph_name, &oc_cluster)); + // First create an entry in the ancestors map if the node is in a compiled + // subgraph or outside_compilation cluster, or if any incoming edge is from + // a node with an ancestor map entry; and find the union of all the + // ancestors. + if (!subgraph_name.empty()) { + node_ancestors_map[node].subgraph = subgraph_name; + node_ancestors_map[node].outside_compilation_cluster = oc_cluster; + } + for (Node* src : node->in_nodes()) { + const auto iter = node_ancestors_map.find(src); + if (iter != node_ancestors_map.end()) { + const auto& ancestors_to_follow = iter->second.AncestorsForSuccessor(); + for (const auto& ancestor : ancestors_to_follow) { + if (ancestor.subgraph != subgraph_name || + ancestor.outside_compilation_cluster != oc_cluster) { + node_ancestors_map[node].ancestor_clusters.insert(ancestor); + } + } + } + } + if (!subgraph_name.empty()) { + // The node is in a compiled subgraph or an outside_compilation cluster. + if (oc_cluster.empty()) { + // The node is not in an outside_compilation cluster. Record the + // subgraph's ancestor dependencies. + for (const auto& cluster : node_ancestors_map[node].ancestor_clusters) { + if (cluster.subgraph != subgraph_name) { + subgraph_ancestors_[subgraph_name].insert(cluster.subgraph); + TF_RETURN_IF_ERROR(CheckClusterDependencyForCycles( + cluster.subgraph, subgraph_name, subgraph_ancestors_, + node_ancestors_map, &cycle_detector, &cycle_detector_map)); + } + } + } else { + Subgraph& subgraph = subgraphs_[subgraph_name]; + // The node is in an outside_compilation cluster. Record the cluster + // and/or subgraph ancestor dependencies. + for (const auto& cluster : node_ancestors_map[node].ancestor_clusters) { + if (cluster.subgraph == subgraph_name) { + // The ancestor is in the same subgraph. + if (cluster.outside_compilation_cluster != oc_cluster) { + // But not in the same oc_cluster, so record the dependency. + subgraph.RecordOutsideCompilationDependency( + oc_cluster, cluster.outside_compilation_cluster); + TF_RETURN_IF_ERROR(CheckClusterDependencyForCycles( + cluster.outside_compilation_cluster, oc_cluster, + subgraph.OutsideCompilationAncestorMap(), node_ancestors_map, + &cycle_detector, &cycle_detector_map)); + } + } else { + // The ancestor is in a different subgraph, so record the + // dependency. + subgraph_ancestors_[subgraph_name].insert(cluster.subgraph); + TF_RETURN_IF_ERROR(CheckClusterDependencyForCycles( + cluster.subgraph, subgraph_name, subgraph_ancestors_, + node_ancestors_map, &cycle_detector, &cycle_detector_map)); + } + } + } + } + } + return Status::OK(); +} + Status Encapsulator::MakePrunedGraphCopyAndInline( const Graph& graph, const std::vector& sink_nodes, std::unique_ptr* pruned_graph, @@ -2166,6 +2535,7 @@ Status EncapsulateSubgraphsInFunctions( Encapsulator encapsulator(std::move(group_attribute), std::move(outside_compilation_attribute), &graph_in); + TF_RETURN_IF_ERROR(encapsulator.FindClusterDependencies()); TF_RETURN_IF_ERROR(encapsulator.SplitIntoSubgraphs()); TF_RETURN_IF_ERROR(encapsulator.BuildFunctionDefs( diff --git a/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc b/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc index 8599a7038a..3502d1bb45 100644 --- a/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc +++ b/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc @@ -74,7 +74,7 @@ bool EqualProtoMap(const ::tensorflow::protobuf::Map& a, if (!compare(elt_a.first, elt_a.second, iter->second)) { if (diff) { *diff = strings::StrCat(map_name, " expected: element with key '", - key_to_string(elt_a.first), " has value '", + key_to_string(elt_a.first), "' has value '", value_to_string(elt_a.second), "' got: '", value_to_string(iter->second), "'"); } @@ -121,8 +121,22 @@ bool EqualFunctionNodeDef(const NodeDef& a, const NodeDef& b, } return false; } + std::unordered_set control_input_a; + std::unordered_set control_input_b; for (int i = 0; i < a.input_size(); ++i) { - if (a.input(i) != b.input(i)) { + if (str_util::StartsWith(a.input(i), "^")) { + if (!str_util::StartsWith(b.input(i), "^")) { + if (diff) { + *diff = strings::StrCat( + diff_preamble, " mismatch for node ", a.name(), " input ", i, + ", expected control input ", a.input(i), " got ", b.input(i), + " expected:\n", a.DebugString(), "\ngot:\n", b.DebugString()); + } + return false; + } + control_input_a.insert(a.input(i)); + control_input_b.insert(b.input(i)); + } else if (a.input(i) != b.input(i)) { if (diff) { *diff = strings::StrCat(diff_preamble, " mismatch for node ", a.name(), " input ", i, ", expected ", a.input(i), @@ -132,11 +146,29 @@ bool EqualFunctionNodeDef(const NodeDef& a, const NodeDef& b, return false; } } + if (control_input_a != control_input_b) { + if (diff) { + *diff = strings::StrCat(diff_preamble, " mismatch for node ", a.name(), + " control inputs differ expected:\n", + a.DebugString(), "\ngot:\n", b.DebugString()); + } + return false; + } return EqualProtoMap( a.attr(), b.attr(), [](const string& s) { return s; }, [](const AttrValue& v) { return v.DebugString(); }, [](const string& key, const AttrValue& av, const AttrValue& bv) { - return av.DebugString() == bv.DebugString(); + if (key == "ancestors") { + // The ancestors are added from a set so the order is unpredictable; + // just compare set equality not list equality. + std::unordered_set a_set(av.list().s().begin(), + av.list().s().end()); + std::unordered_set b_set(bv.list().s().begin(), + bv.list().s().end()); + return a_set == b_set; + } else { + return av.DebugString() == bv.DebugString(); + } }, strings::StrCat(diff_preamble, " attr mismatch for node ", a.name()), diff); @@ -261,6 +293,7 @@ REGISTER_OP("XlaHostCompute") .Output("outputs: Toutputs") .Attr("Tinputs: list(type) >= 0") .Attr("Toutputs: list(type) >= 0") + .Attr("ancestors: list(string) >= 0") .Attr("key: string") .Attr("shape_inference_graph: string = ''") .Attr("shapes: list(shape) >= 0") @@ -899,6 +932,7 @@ TEST(EncapsulateSubgraphsTest, OneFunctionOneOutside) { {"C:o:0", "c:o:0"}, {{"Tinputs", gtl::ArraySlice({DT_FLOAT, DT_FLOAT})}, {"Toutputs", gtl::ArraySlice({DT_FLOAT})}, + {"ancestors", gtl::ArraySlice({})}, {"key", "host_compute_channel_F1_O1"}, {"shape_inference_graph", "_outside_compilation_shape_inference_F1_O1"}, @@ -1044,17 +1078,20 @@ TEST(EncapsulateSubgraphsTest, OneFunctionTwoOutside) { {"D:o:0", "F:o:0"}, {{"Tinputs", gtl::ArraySlice({DT_FLOAT, DT_FLOAT})}, {"Toutputs", gtl::ArraySlice({DT_FLOAT})}, + {"ancestors", + gtl::ArraySlice({"outside_compilation_O1_host_compute"})}, {"key", "host_compute_channel_F1_O2"}, {"shape_inference_graph", "_outside_compilation_shape_inference_F1_O2"}, {"shapes", gtl::ArraySlice({})}, {"_outside_compilation_subgraph", "O2"}}, - {"F"}}, + {"F", "outside_compilation_O1_host_compute"}}, {{"outside_compilation_O1_host_compute"}, "XlaHostCompute", {"C:o:0", "D:o:0"}, {{"Tinputs", gtl::ArraySlice({DT_FLOAT, DT_FLOAT})}, {"Toutputs", gtl::ArraySlice({DT_FLOAT})}, + {"ancestors", gtl::ArraySlice({})}, {"key", "host_compute_channel_F1_O1"}, {"shape_inference_graph", "_outside_compilation_shape_inference_F1_O1"}, @@ -1193,6 +1230,7 @@ TEST(EncapsulateSubgraphsTest, TwoFunctionsTwoOutside) { {"C:o:0", "D:o:0"}, {{"Tinputs", gtl::ArraySlice({DT_FLOAT, DT_FLOAT})}, {"Toutputs", gtl::ArraySlice({DT_FLOAT})}, + {"ancestors", gtl::ArraySlice({})}, {"key", "host_compute_channel_F1_O1"}, {"shape_inference_graph", "_outside_compilation_shape_inference_F1_O1"}, @@ -1215,6 +1253,7 @@ TEST(EncapsulateSubgraphsTest, TwoFunctionsTwoOutside) { {"G:o:0"}, {{"Tinputs", gtl::ArraySlice({DT_FLOAT})}, {"Toutputs", gtl::ArraySlice({DT_FLOAT})}, + {"ancestors", gtl::ArraySlice({})}, {"key", "host_compute_channel_F2_O1"}, {"shape_inference_graph", ""}, {"shapes", @@ -1279,6 +1318,179 @@ TEST(EncapsulateSubgraphsTest, TwoFunctionsTwoOutside) { TF_EXPECT_FUNCTIONDEFLIBRARY_EQ(library_expected, library); } +// Test with two functions to transform, each with one outside_compilation +// cluster, with the dependency between them purely from an outside_compilation +// edge. +TEST(EncapsulateSubgraphsTest, TwoFunctionsTwoOutsideDependencyFromOutside) { + FunctionDefLibrary library; + GraphDef graphdef; + + { + GraphDefBuilder b1(GraphDefBuilder::kFailImmediately); + Node* a = InputShaped(b1.opts().WithName("A")); + Node* b = InputShaped(b1.opts().WithName("B")); + Node* c = Unary(a, b1.opts().WithName("C").WithAttr("_encapsulate", "F1")); + Node* d = + Binary(b, c, b1.opts().WithName("D").WithAttr("_encapsulate", "F1")); + Node* e = Binary(c, d, + b1.opts() + .WithName("E") + .WithControlInputs({b, d}) + .WithAttr("_encapsulate", "F1") + .WithAttr("_outside", "O1")); + Node* f = Binary(c, e, + b1.opts().WithName("F").WithControlInput(e).WithAttr( + "_encapsulate", "F1")); + Node* g = + Binary(a, b, b1.opts().WithName("G").WithAttr("_encapsulate", "F2")); + Node* h = Unary(g, b1.opts() + .WithName("H") + .WithAttr("_encapsulate", "F2") + .WithAttr("_outside", "O1") + .WithControlInput(e)); + Node* i = Unary(h, b1.opts().WithName("I").WithAttr("_encapsulate", "F2")); + Binary(f, i, b1.opts().WithName("J")); + TF_EXPECT_OK(b1.ToGraphDef(&graphdef)); + } + + TF_EXPECT_OK(Encapsulate(&graphdef, &library)); + + FunctionDefLibrary library_expected; + GraphDef graphdef_expected; + + { + GraphDefBuilder shape(GraphDefBuilder::kFailImmediately); + Node* key_constant = + KeyPlaceholderShape(shape.opts().WithName("KnownShape/_0")); + Node* recv = RecvAtHost(ops::NodeOut(key_constant, 0), "F1", "O1", + {DT_FLOAT, DT_FLOAT}, shape.opts()); + Node* e = Binary(ops::NodeOut(recv, 0), ops::NodeOut(recv, 1), + shape.opts() + .WithName("E") + .WithAttr("_encapsulate", "F1") + .WithAttr("_outside", "O1")); + SendFromHost(ops::NodeOut(key_constant, 0), "F1", "O1", {e}, shape.opts()); + TF_EXPECT_OK( + AddGraphDefToFunctionLibrary(shape, "F1_O1", &library_expected)); + } + + { + GraphDefBuilder shape(GraphDefBuilder::kFailImmediately); + Node* key_constant = + KeyPlaceholderShape(shape.opts().WithName("KnownShape/_0")); + Node* recv = RecvAtHost(ops::NodeOut(key_constant, 0), "F2", "O1", + {DT_FLOAT}, shape.opts()); + Node* h = Unary(recv, shape.opts() + .WithName("H") + .WithAttr("_encapsulate", "F2") + .WithAttr("_outside", "O1")); + SendFromHost(ops::NodeOut(key_constant, 0), "F2", "O1", {h}, shape.opts()); + TF_EXPECT_OK( + AddGraphDefToFunctionLibrary(shape, "F2_O1", &library_expected)); + } + + *library_expected.add_function() = FunctionDefHelper::Create( + "F1", {"a_0_arg:float", "b_0_arg:float"}, {"f_0_retval:float"}, {}, + { + {{"C"}, "UnaryTest", {"a_0_arg"}}, + {{"D"}, "BinaryTest", {"b_0_arg", "C:o:0"}}, + {{"F"}, + "BinaryTest", + {"C:o:0", "outside_compilation_O1_host_compute:outputs:0"}, + {}, + {"outside_compilation_O1_host_compute"}}, + {{"outside_compilation_O1_host_compute"}, + "XlaHostCompute", + {"C:o:0", "D:o:0"}, + {{"Tinputs", gtl::ArraySlice({DT_FLOAT, DT_FLOAT})}, + {"Toutputs", gtl::ArraySlice({DT_FLOAT})}, + {"ancestors", gtl::ArraySlice({})}, + {"key", "host_compute_channel_F1_O1"}, + {"shape_inference_graph", + "_outside_compilation_shape_inference_F1_O1"}, + {"shapes", gtl::ArraySlice({})}, + {"_outside_compilation_subgraph", "O1"}}, + {"D"}}, + }, + {{"f_0_retval", "F:o:0"}}); + + *library_expected.add_function() = FunctionDefHelper::Create( + "F2", {"a_0_arg:float", "b_0_arg:float"}, {"i_0_retval:float"}, {}, + { + {{"G"}, "BinaryTest", {"a_0_arg", "b_0_arg"}}, + {{"I"}, + "UnaryTest", + {"outside_compilation_O1_host_compute:outputs:0"}}, + {{"outside_compilation_O1_host_compute"}, + "XlaHostCompute", + {"G:o:0"}, + {{"Tinputs", gtl::ArraySlice({DT_FLOAT})}, + {"Toutputs", gtl::ArraySlice({DT_FLOAT})}, + {"ancestors", gtl::ArraySlice({})}, + {"key", "host_compute_channel_F2_O1"}, + {"shape_inference_graph", + "_outside_compilation_shape_inference_F2_O1"}, + {"shapes", gtl::ArraySlice({})}, + {"_outside_compilation_subgraph", "O1"}}}, + }, + {{"i_0_retval", "I:o:0"}}); + + { + std::unique_ptr lib_def( + new FunctionLibraryDefinition(OpRegistry::Global(), library_expected)); + GraphDefBuilder b2(GraphDefBuilder::kFailImmediately, lib_def.get()); + Node* a = InputShaped(b2.opts().WithName("A")); + Node* b = InputShaped(b2.opts().WithName("B")); + + Node* key_constant1 = + KeyPlaceholder("F1", b2.opts().WithName("F1_key_placeholder")); + Node* recv1 = RecvAtHost(ops::NodeOut(key_constant1, 0), "F1", "O1", + {DT_FLOAT, DT_FLOAT}, b2.opts()); + Node* e = Binary(ops::NodeOut(recv1, 0), ops::NodeOut(recv1, 1), + b2.opts() + .WithName("E") + .WithControlInputs({recv1, b}) + .WithAttr("_encapsulate", "F1") + .WithAttr("_outside", "O1")); + Node* send1 = SendFromHost(ops::NodeOut(key_constant1, 0), "F1", "O1", {e}, + b2.opts().WithControlInput(e)); + Node* s1 = Sequencer( + b2.opts().WithName("F1_sequencer").WithControlInputs({recv1, send1}), + "F1"); + + NodeBuilder node_builder1("F1", "F1", lib_def.get()); + node_builder1.Input(a).Input(b); + Node* call1 = + b2.opts().WithControlInput(s1).FinalizeBuilder(&node_builder1); + + Node* key_constant2 = + KeyPlaceholder("F2", b2.opts().WithName("F2_key_placeholder")); + Node* recv2 = RecvAtHost(ops::NodeOut(key_constant2, 0), "F2", "O1", + {DT_FLOAT}, b2.opts()); + Node* h = Unary(recv2, b2.opts() + .WithName("H") + .WithAttr("_encapsulate", "F2") + .WithAttr("_outside", "O1") + .WithControlInput(e)); + Node* send2 = SendFromHost(ops::NodeOut(key_constant2, 0), "F2", "O1", {h}, + b2.opts()); + + Node* s2 = Sequencer( + b2.opts().WithName("F2_sequencer").WithControlInputs({recv2, send2}), + "F2"); + NodeBuilder node_builder2("F2", "F2", lib_def.get()); + node_builder2.Input(a).Input(b); + Node* call2 = b2.opts() + .WithControlInputs({s2, call1}) + .FinalizeBuilder(&node_builder2); + Binary(call1, call2, b2.opts().WithName("J")); + TF_EXPECT_OK(b2.ToGraphDef(&graphdef_expected)); + } + + TF_EXPECT_GRAPH_EQ(graphdef_expected, graphdef); + TF_EXPECT_FUNCTIONDEFLIBRARY_EQ(library_expected, library); +} + // Test with one outside_compilation cluster that has no inputs from the // compiled subgraph. TEST(EncapsulateSubgraphsTest, OutsideCompilationNoInputs) { @@ -1323,6 +1535,7 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationNoInputs) { {}, {{"Tinputs", gtl::ArraySlice({})}, {"Toutputs", gtl::ArraySlice({DT_FLOAT})}, + {"ancestors", gtl::ArraySlice({})}, {"key", "host_compute_channel_F1_O1"}, {"shape_inference_graph", ""}, {"shapes", @@ -1406,6 +1619,7 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationControlInput) { {}, {{"Tinputs", gtl::ArraySlice({})}, {"Toutputs", gtl::ArraySlice({DT_FLOAT})}, + {"ancestors", gtl::ArraySlice({})}, {"key", "host_compute_channel_F1_O1"}, {"shape_inference_graph", ""}, {"shapes", @@ -1487,6 +1701,7 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationNoOutputs) { {"D:o:0"}, {{"Tinputs", gtl::ArraySlice({DT_FLOAT})}, {"Toutputs", gtl::ArraySlice({})}, + {"ancestors", gtl::ArraySlice({})}, {"key", "host_compute_channel_F1_O1"}, {"shape_inference_graph", ""}, {"shapes", gtl::ArraySlice({})}, @@ -1567,6 +1782,7 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationControlOutput) { {"D:o:0"}, {{"Tinputs", gtl::ArraySlice({DT_FLOAT})}, {"Toutputs", gtl::ArraySlice({})}, + {"ancestors", gtl::ArraySlice({})}, {"key", "host_compute_channel_F1_O1"}, {"shape_inference_graph", ""}, {"shapes", gtl::ArraySlice({})}, @@ -1607,6 +1823,371 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationControlOutput) { TF_EXPECT_FUNCTIONDEFLIBRARY_EQ(library_expected, library); } +// Test with two outside_compilation clusters that interact outside the compiled +// subgraph, where the ancestor has no HostCompute Op. +TEST(EncapsulateSubgraphsTest, + OutsideCompilationClusterDependencyNoSrcCluster) { + FunctionDefLibrary library; + GraphDef graphdef; + + { + GraphDefBuilder b1(GraphDefBuilder::kFailImmediately); + Node* a = Input(b1.opts().WithName("A")); + Node* b = Input(b1.opts().WithName("B")); + Node* c = Unary(a, b1.opts().WithName("C").WithAttr("_encapsulate", "F1")); + Node* d = + Binary(b, c, b1.opts().WithName("D").WithAttr("_encapsulate", "F1")); + Node* e = Unary(a, b1.opts() + .WithName("E") + .WithAttr("_encapsulate", "F1") + .WithAttr("_outside", "O1")); + Node* f = Unary(d, b1.opts().WithName("F").WithAttr("_encapsulate", "F1")); + Node* g = Unary(f, b1.opts() + .WithName("G") + .WithAttr("_encapsulate", "F1") + .WithAttr("_outside", "O2") + .WithControlInput(e)); + Node* h = Unary(g, b1.opts().WithName("H").WithAttr("_encapsulate", "F1")); + Binary(e, h, b1.opts().WithName("I")); + TF_EXPECT_OK(b1.ToGraphDef(&graphdef)); + } + + TF_EXPECT_OK(Encapsulate(&graphdef, &library)); + + FunctionDefLibrary library_expected; + GraphDef graphdef_expected; + + { + GraphDefBuilder shape2(GraphDefBuilder::kFailImmediately); + Node* key_constant = + KeyPlaceholderShape(shape2.opts().WithName("KnownShape/_0")); + Node* recv2 = RecvAtHost(ops::NodeOut(key_constant, 0), "F1", "O2", + {DT_FLOAT}, shape2.opts()); + Node* g = Unary(ops::NodeOut(recv2, 0), shape2.opts() + .WithName("G") + .WithAttr("_encapsulate", "F1") + .WithAttr("_outside", "O2")); + SendFromHost(ops::NodeOut(key_constant, 0), "F1", "O2", {g}, shape2.opts()); + TF_EXPECT_OK( + AddGraphDefToFunctionLibrary(shape2, "F1_O2", &library_expected)); + } + + *library_expected.add_function() = FunctionDefHelper::Create( + "F1", {"a_0_arg:float", "b_0_arg:float"}, {"h_0_retval:float"}, {}, + { + {{"C"}, "UnaryTest", {"a_0_arg"}}, + {{"D"}, "BinaryTest", {"b_0_arg", "C:o:0"}}, + {{"F"}, "UnaryTest", {"D:o:0"}}, + {{"H"}, + "UnaryTest", + {"outside_compilation_O2_host_compute:outputs:0"}}, + {{"outside_compilation_O2_host_compute"}, + "XlaHostCompute", + {"F:o:0"}, + {{"Tinputs", gtl::ArraySlice({DT_FLOAT})}, + {"Toutputs", gtl::ArraySlice({DT_FLOAT})}, + {"ancestors", gtl::ArraySlice({})}, + {"key", "host_compute_channel_F1_O2"}, + {"shape_inference_graph", + "_outside_compilation_shape_inference_F1_O2"}, + {"shapes", gtl::ArraySlice({})}, + {"_outside_compilation_subgraph", "O2"}}}, + }, + {{"h_0_retval", "H:o:0"}}); + + { + std::unique_ptr lib_def( + new FunctionLibraryDefinition(OpRegistry::Global(), library_expected)); + GraphDefBuilder b2(GraphDefBuilder::kFailImmediately, lib_def.get()); + Node* a = Input(b2.opts().WithName("A")); + Node* b = Input(b2.opts().WithName("B")); + + Node* e = Unary(a, b2.opts() + .WithName("E") + .WithAttr("_encapsulate", "F1") + .WithAttr("_outside", "O1")); + Node* key_constant = + KeyPlaceholder("F1", b2.opts().WithName("F1_key_placeholder")); + Node* recv = RecvAtHost(ops::NodeOut(key_constant, 0), "F1", "O2", + {DT_FLOAT}, b2.opts()); + Node* g = Unary(recv, b2.opts() + .WithName("G") + .WithAttr("_encapsulate", "F1") + .WithAttr("_outside", "O2") + .WithControlInput(e)); + Node* send = + SendFromHost(ops::NodeOut(key_constant, 0), "F1", "O2", {g}, b2.opts()); + Node* s1 = Sequencer( + b2.opts().WithName("F1_sequencer").WithControlInputs({recv, send}), + "F1"); + NodeBuilder node_builder1("F1", "F1", lib_def.get()); + node_builder1.Input(a).Input(b).ControlInput(s1); + Node* call1 = b2.opts().FinalizeBuilder(&node_builder1); + + Binary(e, call1, b2.opts().WithName("I")); + TF_EXPECT_OK(b2.ToGraphDef(&graphdef_expected)); + } + + TF_EXPECT_GRAPH_EQ(graphdef_expected, graphdef); + TF_EXPECT_FUNCTIONDEFLIBRARY_EQ(library_expected, library); +} + +// Test with two outside_compilation clusters that interact outside the compiled +// subgraph, where the successor has no HostCompute Op. +TEST(EncapsulateSubgraphsTest, + OutsideCompilationClusterDependencyNoDstCluster) { + FunctionDefLibrary library; + GraphDef graphdef; + + { + GraphDefBuilder b1(GraphDefBuilder::kFailImmediately); + Node* a = Input(b1.opts().WithName("A")); + Node* b = Input(b1.opts().WithName("B")); + Node* c = Unary(a, b1.opts().WithName("C").WithAttr("_encapsulate", "F1")); + Node* d = + Binary(b, c, b1.opts().WithName("D").WithAttr("_encapsulate", "F1")); + Node* e = Unary(d, b1.opts() + .WithName("E") + .WithAttr("_encapsulate", "F1") + .WithAttr("_outside", "O1")); + Node* f = Unary(e, b1.opts().WithName("F").WithAttr("_encapsulate", "F1")); + /*Node* g =*/Unary(a, b1.opts() + .WithName("G") + .WithAttr("_encapsulate", "F1") + .WithAttr("_outside", "O2") + .WithControlInput(e)); + Node* h = Unary(f, b1.opts().WithName("H").WithAttr("_encapsulate", "F1")); + Binary(e, h, b1.opts().WithName("I")); + TF_EXPECT_OK(b1.ToGraphDef(&graphdef)); + } + + TF_EXPECT_OK(Encapsulate(&graphdef, &library)); + + FunctionDefLibrary library_expected; + GraphDef graphdef_expected; + + { + GraphDefBuilder shape1(GraphDefBuilder::kFailImmediately); + Node* key_constant = + KeyPlaceholderShape(shape1.opts().WithName("KnownShape/_0")); + Node* recv2 = RecvAtHost(ops::NodeOut(key_constant, 0), "F1", "O1", + {DT_FLOAT}, shape1.opts()); + Node* e = Unary(ops::NodeOut(recv2, 0), shape1.opts() + .WithName("E") + .WithAttr("_encapsulate", "F1") + .WithAttr("_outside", "O1")); + SendFromHost(ops::NodeOut(key_constant, 0), "F1", "O1", {e}, shape1.opts()); + TF_EXPECT_OK( + AddGraphDefToFunctionLibrary(shape1, "F1_O1", &library_expected)); + } + + *library_expected.add_function() = FunctionDefHelper::Create( + "F1", {"a_0_arg:float", "b_0_arg:float"}, {"h_0_retval:float"}, {}, + { + {{"C"}, "UnaryTest", {"a_0_arg"}}, + {{"D"}, "BinaryTest", {"b_0_arg", "C:o:0"}}, + {{"F"}, + "UnaryTest", + {"outside_compilation_O1_host_compute:outputs:0"}}, + {{"H"}, "UnaryTest", {"F:o:0"}}, + {{"outside_compilation_O1_host_compute"}, + "XlaHostCompute", + {"D:o:0"}, + {{"Tinputs", gtl::ArraySlice({DT_FLOAT})}, + {"Toutputs", gtl::ArraySlice({DT_FLOAT})}, + {"ancestors", gtl::ArraySlice({})}, + {"key", "host_compute_channel_F1_O1"}, + {"shape_inference_graph", + "_outside_compilation_shape_inference_F1_O1"}, + {"shapes", gtl::ArraySlice({})}, + {"_outside_compilation_subgraph", "O1"}}}, + }, + {{"h_0_retval", "H:o:0"}}); + + { + std::unique_ptr lib_def( + new FunctionLibraryDefinition(OpRegistry::Global(), library_expected)); + GraphDefBuilder b2(GraphDefBuilder::kFailImmediately, lib_def.get()); + Node* a = Input(b2.opts().WithName("A")); + Node* b = Input(b2.opts().WithName("B")); + + Node* key_constant = + KeyPlaceholder("F1", b2.opts().WithName("F1_key_placeholder")); + Node* recv = RecvAtHost(ops::NodeOut(key_constant, 0), "F1", "O1", + {DT_FLOAT}, b2.opts()); + Node* e = Unary(recv, b2.opts() + .WithName("E") + .WithAttr("_encapsulate", "F1") + .WithAttr("_outside", "O1")); + Node* send = + SendFromHost(ops::NodeOut(key_constant, 0), "F1", "O1", {e}, b2.opts()); + /*Node* g =*/Unary(a, b2.opts() + .WithName("G") + .WithAttr("_encapsulate", "F1") + .WithAttr("_outside", "O2") + .WithControlInput(e)); + Node* s1 = Sequencer( + b2.opts().WithName("F1_sequencer").WithControlInputs({recv, send}), + "F1"); + NodeBuilder node_builder1("F1", "F1", lib_def.get()); + node_builder1.Input(a).Input(b).ControlInput(s1); + Node* call1 = b2.opts().FinalizeBuilder(&node_builder1); + + Binary(e, call1, b2.opts().WithName("I")); + TF_EXPECT_OK(b2.ToGraphDef(&graphdef_expected)); + } + + TF_EXPECT_GRAPH_EQ(graphdef_expected, graphdef); + TF_EXPECT_FUNCTIONDEFLIBRARY_EQ(library_expected, library); +} + +// Test with two outside_compilation clusters that interact outside the compiled +// subgraph. +TEST(EncapsulateSubgraphsTest, OutsideCompilationClusterDependency) { + FunctionDefLibrary library; + GraphDef graphdef; + + { + GraphDefBuilder b1(GraphDefBuilder::kFailImmediately); + Node* a = Input(b1.opts().WithName("A")); + Node* b = Input(b1.opts().WithName("B")); + Node* c = Unary(a, b1.opts().WithName("C").WithAttr("_encapsulate", "F1")); + Node* d = + Binary(b, c, b1.opts().WithName("D").WithAttr("_encapsulate", "F1")); + Node* e = Unary(d, b1.opts() + .WithName("E") + .WithAttr("_encapsulate", "F1") + .WithAttr("_outside", "O1")); + Node* f = Unary(e, b1.opts().WithName("F").WithAttr("_encapsulate", "F1")); + Node* g = Unary(d, b1.opts() + .WithName("G") + .WithAttr("_encapsulate", "F1") + .WithAttr("_outside", "O2") + .WithControlInput(e)); + Node* h = Unary(f, b1.opts().WithName("H").WithAttr("_encapsulate", "F1")); + /*Node* i =*/Binary(d, e, + b1.opts() + .WithName("I") + .WithAttr("_encapsulate", "F1") + .WithAttr("_outside", "O3") + .WithControlInput(g)); + Binary(e, h, b1.opts().WithName("J")); + TF_EXPECT_OK(b1.ToGraphDef(&graphdef)); + } + + TF_EXPECT_OK(Encapsulate(&graphdef, &library)); + + FunctionDefLibrary library_expected; + GraphDef graphdef_expected; + + { + GraphDefBuilder shape1(GraphDefBuilder::kFailImmediately); + Node* key_constant = + KeyPlaceholderShape(shape1.opts().WithName("KnownShape/_0")); + Node* recv2 = RecvAtHost(ops::NodeOut(key_constant, 0), "F1", "O1", + {DT_FLOAT}, shape1.opts()); + Node* e = Unary(ops::NodeOut(recv2, 0), shape1.opts() + .WithName("E") + .WithAttr("_encapsulate", "F1") + .WithAttr("_outside", "O1")); + SendFromHost(ops::NodeOut(key_constant, 0), "F1", "O1", {e}, shape1.opts()); + TF_EXPECT_OK( + AddGraphDefToFunctionLibrary(shape1, "F1_O1", &library_expected)); + } + + *library_expected.add_function() = FunctionDefHelper::Create( + "F1", {"a_0_arg:float", "b_0_arg:float"}, {"h_0_retval:float"}, {}, + {{{"C"}, "UnaryTest", {"a_0_arg"}}, + {{"D"}, "BinaryTest", {"b_0_arg", "C:o:0"}}, + {{"F"}, "UnaryTest", {"outside_compilation_O1_host_compute:outputs:0"}}, + {{"H"}, "UnaryTest", {"F:o:0"}}, + {{"outside_compilation_O1_host_compute"}, + "XlaHostCompute", + {"D:o:0"}, + {{"Tinputs", gtl::ArraySlice({DT_FLOAT})}, + {"Toutputs", gtl::ArraySlice({DT_FLOAT})}, + {"ancestors", gtl::ArraySlice({})}, + {"key", "host_compute_channel_F1_O1"}, + {"shape_inference_graph", + "_outside_compilation_shape_inference_F1_O1"}, + {"shapes", gtl::ArraySlice({})}, + {"_outside_compilation_subgraph", "O1"}}}, + {{"outside_compilation_O2_host_compute"}, + "XlaHostCompute", + {"D:o:0"}, + {{"Tinputs", gtl::ArraySlice({DT_FLOAT})}, + {"Toutputs", gtl::ArraySlice({})}, + {"ancestors", + gtl::ArraySlice({"outside_compilation_O1_host_compute"})}, + {"key", "host_compute_channel_F1_O2"}, + {"shape_inference_graph", ""}, + {"shapes", gtl::ArraySlice({})}, + {"_outside_compilation_subgraph", "O2"}}, + {"outside_compilation_O1_host_compute"}}, + {{"outside_compilation_O3_host_compute"}, + "XlaHostCompute", + {"D:o:0"}, + {{"Tinputs", gtl::ArraySlice({DT_FLOAT})}, + {"Toutputs", gtl::ArraySlice({})}, + {"ancestors", + gtl::ArraySlice({"outside_compilation_O1_host_compute", + "outside_compilation_O2_host_compute"})}, + {"key", "host_compute_channel_F1_O3"}, + {"shape_inference_graph", ""}, + {"shapes", gtl::ArraySlice({})}, + {"_outside_compilation_subgraph", "O3"}}, + {"outside_compilation_O1_host_compute", + "outside_compilation_O2_host_compute"}}}, + {{"h_0_retval", "H:o:0"}}); + + { + std::unique_ptr lib_def( + new FunctionLibraryDefinition(OpRegistry::Global(), library_expected)); + GraphDefBuilder b2(GraphDefBuilder::kFailImmediately, lib_def.get()); + Node* a = Input(b2.opts().WithName("A")); + Node* b = Input(b2.opts().WithName("B")); + + Node* key_constant = + KeyPlaceholder("F1", b2.opts().WithName("F1_key_placeholder")); + Node* recv1 = RecvAtHost(ops::NodeOut(key_constant, 0), "F1", "O1", + {DT_FLOAT}, b2.opts()); + Node* e = Unary(recv1, b2.opts() + .WithName("E") + .WithAttr("_encapsulate", "F1") + .WithAttr("_outside", "O1")); + Node* send = + SendFromHost(ops::NodeOut(key_constant, 0), "F1", "O1", {e}, b2.opts()); + Node* recv2 = RecvAtHost(ops::NodeOut(key_constant, 0), "F1", "O2", + {DT_FLOAT}, b2.opts()); + Node* g = Unary(recv2, b2.opts() + .WithName("G") + .WithAttr("_encapsulate", "F1") + .WithAttr("_outside", "O2") + .WithControlInput(e)); + Node* recv3 = RecvAtHost(ops::NodeOut(key_constant, 0), "F1", "O3", + {DT_FLOAT}, b2.opts()); + /*Node* i =*/Binary(recv3, e, + b2.opts() + .WithName("I") + .WithAttr("_encapsulate", "F1") + .WithAttr("_outside", "O3") + .WithControlInput(g)); + Node* s1 = Sequencer(b2.opts() + .WithName("F1_sequencer") + .WithControlInputs({recv1, send, recv2, recv3}), + "F1"); + NodeBuilder node_builder1("F1", "F1", lib_def.get()); + node_builder1.Input(a).Input(b).ControlInput(s1); + Node* call1 = b2.opts().FinalizeBuilder(&node_builder1); + + Binary(e, call1, b2.opts().WithName("J")); + TF_EXPECT_OK(b2.ToGraphDef(&graphdef_expected)); + } + + TF_EXPECT_GRAPH_EQ(graphdef_expected, graphdef); + TF_EXPECT_FUNCTIONDEFLIBRARY_EQ(library_expected, library); +} + // Test with one outside_compilation cluster that has no outputs from the // compiled subgraph. TEST(EncapsulateSubgraphsTest, OutsideCompilationNoInputsOrOutputs) { @@ -1731,6 +2312,7 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationShapeInference) { {"c:o:0"}, {{"Tinputs", gtl::ArraySlice({DT_FLOAT})}, {"Toutputs", gtl::ArraySlice({DT_FLOAT})}, + {"ancestors", gtl::ArraySlice({})}, {"key", "host_compute_channel_F1_O1"}, {"shape_inference_graph", "_outside_compilation_shape_inference_F1_O1"}, diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc index 86263d847a..c0e9967684 100644 --- a/tensorflow/compiler/tf2xla/xla_compiler.cc +++ b/tensorflow/compiler/tf2xla/xla_compiler.cc @@ -813,4 +813,29 @@ Status XlaCompiler::SetHostToDeviceMetadata( return Status::OK(); } +Status XlaCompiler::GetHostComputeControlDependency( + const string& host_compute_name, xla::ComputationDataHandle* handle) { + const auto iter = host_compute_control_output_.find(host_compute_name); + if (iter == host_compute_control_output_.end()) { + return errors::InvalidArgument( + "No registered control handle for host compute Op '", host_compute_name, + "'"); + } else { + *handle = iter->second; + } + return Status::OK(); +} + +Status XlaCompiler::SetHostComputeControlDependency( + const string& host_compute_name, const xla::ComputationDataHandle& handle) { + if (host_compute_control_output_.find(host_compute_name) != + host_compute_control_output_.end()) { + return errors::InvalidArgument( + "Duplicate control handles registered for for host compute Op ", + host_compute_name); + } + host_compute_control_output_[host_compute_name] = handle; + return Status::OK(); +} + } // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/xla_compiler.h b/tensorflow/compiler/tf2xla/xla_compiler.h index a6747bbe72..8f564f35ec 100644 --- a/tensorflow/compiler/tf2xla/xla_compiler.h +++ b/tensorflow/compiler/tf2xla/xla_compiler.h @@ -325,6 +325,23 @@ class XlaCompiler { gtl::ArraySlice types, gtl::ArraySlice shapes); + // In order to avoid deadlocks from dependencies in host computations, it can + // be necessary to enforce a partial order on the execution of HostCompute + // Ops. In particular it may be necessary to constrain the SendToHost for one + // HostCompute to run before blocking on the RecvAtHost for another + // HostCompute. The compiler maintains a mapping from 'host_compute_name' to + // handle, where the handle is an 'output' of the HostCompute Op corresponding + // to 'host_compute_name'. Another HostCompute Op that needs to be sequenced + // later can add the handle as an 'input' to enforce the constraints. + // 'host_compute_name' can be any string the client wishes to use to identify + // a given HostCompute Op as long as the names are unique within the + // compilation. + Status GetHostComputeControlDependency(const string& host_compute_name, + xla::ComputationDataHandle* handle); + Status SetHostComputeControlDependency( + const string& host_compute_name, + const xla::ComputationDataHandle& handle); + const Options& options() const { return options_; } xla::Client* client() const { return options_.client; } FunctionLibraryRuntime* flib_runtime() const { return flib_runtime_; } @@ -391,6 +408,9 @@ class XlaCompiler { std::unordered_map host_compute_sends_; std::unordered_map host_compute_recvs_; + std::unordered_map + host_compute_control_output_; + TF_DISALLOW_COPY_AND_ASSIGN(XlaCompiler); }; -- GitLab From d82d04f15992e224743f29aa75134ed04aa064a7 Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Fri, 20 Apr 2018 13:58:51 -0700 Subject: [PATCH 759/791] Automated g4 rollback of changelist 193694958 PiperOrigin-RevId: 193718607 --- .../core/distributed_runtime/master_session.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tensorflow/core/distributed_runtime/master_session.cc b/tensorflow/core/distributed_runtime/master_session.cc index 1c67b42e76..ebe350d313 100644 --- a/tensorflow/core/distributed_runtime/master_session.cc +++ b/tensorflow/core/distributed_runtime/master_session.cc @@ -89,10 +89,6 @@ class MasterSession::ReffedClientGraph : public core::RefCounted { ~ReffedClientGraph() override { if (should_deregister_) { DeregisterPartitions(); - } else { - for (Part& part : partitions_) { - worker_cache_->ReleaseWorker(part.name, part.worker); - } } } @@ -1178,8 +1174,14 @@ Status MasterSession::Create(GraphDef* graph_def, TF_RETURN_IF_ERROR(GraphExecutionState::MakeForBaseGraph( graph_def, execution_options, &execution_state_)); } - should_delete_worker_sessions_ = true; - return CreateWorkerSessions(options); + // TODO(b/36574172): Remove these conditions when ClusterSpec + // propagation is supported in all servers. + if (options.cluster_def != nullptr || + session_opts_.config.isolate_session_state()) { + should_delete_worker_sessions_ = true; + return CreateWorkerSessions(options); + } + return Status::OK(); } Status MasterSession::CreateWorkerSessions( -- GitLab From 9fc5bacba49eb31c7d536963879ccc62ecfbaf76 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Apr 2018 14:25:57 -0700 Subject: [PATCH 760/791] Pin rbe-debian8-tf container tp a newer base image - Also improve how numpy is installed (not compiling from source) for containers based on other distros than Ubuntu14.04 PiperOrigin-RevId: 193722848 --- tensorflow/tools/ci_build/Dockerfile.rbe.cpu | 2 +- .../tools/ci_build/install/install_pip_packages.sh | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tensorflow/tools/ci_build/Dockerfile.rbe.cpu b/tensorflow/tools/ci_build/Dockerfile.rbe.cpu index 6f0798b1af..3bc52b9ed6 100644 --- a/tensorflow/tools/ci_build/Dockerfile.rbe.cpu +++ b/tensorflow/tools/ci_build/Dockerfile.rbe.cpu @@ -1,4 +1,4 @@ -FROM launcher.gcr.io/google/rbe-debian8:r322167 +FROM launcher.gcr.io/google/rbe-debian8:r327695 LABEL maintainer="Yu Yi " # Copy install scripts diff --git a/tensorflow/tools/ci_build/install/install_pip_packages.sh b/tensorflow/tools/ci_build/install/install_pip_packages.sh index 9644277fab..5aaf544afd 100755 --- a/tensorflow/tools/ci_build/install/install_pip_packages.sh +++ b/tensorflow/tools/ci_build/install/install_pip_packages.sh @@ -65,8 +65,13 @@ rm -rf /usr/lib/python3/dist-packages/six* # numpy needs to be installed from source to fix segfaults. See: # https://github.com/tensorflow/tensorflow/issues/6968 # This workaround isn't needed for Ubuntu 16.04 or later. -pip2 install --no-binary=:all: --upgrade numpy==1.12.0 -pip3 install --no-binary=:all: --upgrade numpy==1.12.0 +if $(cat /etc/*-release | grep -q 14.04); then + pip2 install --no-binary=:all: --upgrade numpy==1.12.0 + pip3 install --no-binary=:all: --upgrade numpy==1.12.0 +else + pip2 install --upgrade numpy==1.12.0 + pip3 install --upgrade numpy==1.12.0 +fi pip2 install scipy==0.18.1 pip3 install scipy==0.18.1 -- GitLab From 9f312f32091534bfc115212d2ec7c838180df663 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Apr 2018 14:30:48 -0700 Subject: [PATCH 761/791] Updating Generate Random Tensor to generate tensors whose values are small and do not cause overflow for arithmetic operations. PiperOrigin-RevId: 193723661 --- tensorflow/core/grappler/optimizers/BUILD | 1 - tensorflow/core/grappler/utils/BUILD | 1 + tensorflow/core/grappler/utils/grappler_test.h | 4 +++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD index 3ab8d8f584..42c3580d40 100644 --- a/tensorflow/core/grappler/optimizers/BUILD +++ b/tensorflow/core/grappler/optimizers/BUILD @@ -112,7 +112,6 @@ tf_cc_test( name = "constant_folding_test", srcs = ["constant_folding_test.cc"], shard_count = 5, - tags = ["noasan"], deps = [ ":constant_folding", "//tensorflow/cc:cc_ops", diff --git a/tensorflow/core/grappler/utils/BUILD b/tensorflow/core/grappler/utils/BUILD index b473f32c45..44ef4a965b 100644 --- a/tensorflow/core/grappler/utils/BUILD +++ b/tensorflow/core/grappler/utils/BUILD @@ -128,6 +128,7 @@ cc_library( "//tensorflow/core:direct_session", "//tensorflow/core:framework", "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", "//tensorflow/core:protos_all_cc", "//tensorflow/core:test", "//tensorflow/core/grappler:grappler_item", diff --git a/tensorflow/core/grappler/utils/grappler_test.h b/tensorflow/core/grappler/utils/grappler_test.h index e1394b9c35..c2ba5ee7e8 100644 --- a/tensorflow/core/grappler/utils/grappler_test.h +++ b/tensorflow/core/grappler/utils/grappler_test.h @@ -23,6 +23,7 @@ limitations under the License. #include "tensorflow/core/framework/types.h" #include "tensorflow/core/grappler/grappler_item.h" #include "tensorflow/core/grappler/utils.h" +#include "tensorflow/core/lib/random/random.h" #include "tensorflow/core/platform/test.h" #include "tensorflow/core/public/session_options.h" @@ -62,7 +63,8 @@ class GrapplerTest : public ::testing::Test { Tensor GenerateRandomTensor(const TensorShape& shape) const { typedef typename EnumToDataType::Type T; Tensor tensor(DTYPE, shape); - tensor.flat() = tensor.flat().random(); + for (auto i = 0; i < tensor.NumElements(); i++) + tensor.flat()(i) = i + random::New64() % 10; return tensor; } -- GitLab From bc78f9b060cece8e29a89f7dbcdedcadbc61891d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Apr 2018 14:32:07 -0700 Subject: [PATCH 762/791] internal END_PUBLIC BEGIN_PUBLIC Automated g4 rollback of changelist 193600682 PiperOrigin-RevId: 193723856 --- .../layers/python/layers/rev_block_lib.py | 77 ++----------- .../python/layers/rev_block_lib_test.py | 102 ------------------ 2 files changed, 11 insertions(+), 168 deletions(-) diff --git a/tensorflow/contrib/layers/python/layers/rev_block_lib.py b/tensorflow/contrib/layers/python/layers/rev_block_lib.py index 9f904cc302..02d294c68f 100644 --- a/tensorflow/contrib/layers/python/layers/rev_block_lib.py +++ b/tensorflow/contrib/layers/python/layers/rev_block_lib.py @@ -45,7 +45,6 @@ from tensorflow.python.ops import math_ops from tensorflow.python.ops import variable_scope from tensorflow.python.platform import tf_logging as logging from tensorflow.python.util import nest -from tensorflow.python.util import tf_inspect __all__ = ["rev_block", "RevBlock", "recompute_grad"] @@ -430,13 +429,12 @@ def enable_with_args(dec): @enable_with_args -def recompute_grad(fn, use_data_dep=_USE_DEFAULT, tupleize_grads=False, - tensor_arg_names=None): +def recompute_grad(fn, use_data_dep=_USE_DEFAULT, tupleize_grads=False): """Decorator that recomputes the function on the backwards pass. Args: - fn: the subgraph-producing function to wrap and recompute when computing - gradients. Provide `tensor_arg_names` if not all arguments are `Tensor`s. + fn: a function that takes Tensors (all as positional arguments) and returns + a tuple of Tensors. use_data_dep: `bool`, if `True` will use a dummy data dependency to force the recompute to happen. If `False` will use a control dependency. By default will be `True` if in an XLA context and `False` otherwise. XLA @@ -445,25 +443,17 @@ def recompute_grad(fn, use_data_dep=_USE_DEFAULT, tupleize_grads=False, that all gradients are produced before any are consumed by downstream ops. If `use_data_dep` is also `True`, will use a data dependency instead of a control dependency. - tensor_arg_names: `list`, names of the `Tensor` arguments to `fn`. If - `None`, assumes all arguments are `Tensor`s. Returns: A wrapped fn that is identical to fn when called, but its activations will be discarded and recomputed on the backwards pass (i.e. on a call to tf.gradients). """ - if tensor_arg_names: - if not isinstance(tensor_arg_names, (list, tuple)): - raise TypeError("tensor_arg_names must be a list") @functools.wraps(fn) - def wrapped(*args, **kwargs): - tensor_only_fn, tensor_args = _make_tensor_only(fn, args, kwargs, - tensor_arg_names) + def wrapped(*args): return _recompute_grad( - tensor_only_fn, tensor_args, use_data_dep=use_data_dep, - tupleize_grads=tupleize_grads) + fn, args, use_data_dep=use_data_dep, tupleize_grads=tupleize_grads) return wrapped @@ -473,59 +463,11 @@ def _is_on_tpu(): return control_flow_util.GetContainingXLAContext(ctxt) is not None -def _make_tensor_only(fn, args, kwargs, tensor_arg_names): - """Return fn such that it only takes Tensor args for tensor_arg_names.""" - argspec = tf_inspect.getargspec(fn) - if argspec.varargs is not None or argspec.keywords is not None: - raise ValueError("Function decorated with recompute_grad must not use " - "*args or **kwargs.") - fn_arg_names = list(argspec.args) - - # name_to_arg is a dict of argument name to argument value, including both - # positional and keyword arguments passed. - name_to_arg = {} - # Populate positional arguments. - for name, arg in zip(fn_arg_names[:len(args)], args): - name_to_arg[name] = arg - # Populate keyword arguments. - name_to_arg.update(kwargs) - - # Separate the Tensor arguments from the non-Tensor arguments. - # The default is that all arguments are Tensor arguments. - tensor_arg_names = tensor_arg_names or fn_arg_names - for name in tensor_arg_names: - if name not in name_to_arg: - raise ValueError("Must provide Tensor argument %s" % name) - tensor_args = [name_to_arg[name] for name in tensor_arg_names] - non_tensor_kwargs = dict([(name, arg) for name, arg in name_to_arg.items() - if name not in tensor_arg_names]) - - # Check that Tensor arguments are in fact Tensors and that non-Tensor - # arguments are not. - for name, arg in zip(tensor_arg_names, tensor_args): - if not isinstance(arg, framework_ops.Tensor): - raise TypeError("Fn argument %s must be a Tensor." % name) - for name, arg in non_tensor_kwargs.items(): - if isinstance(arg, framework_ops.Tensor): - raise TypeError("Fn argument %s must not be a Tensor." % name) - - # Construct a Tensor-only wrapper function that will pass the non-Tensor - # arguments as well when called. - def tensor_only_fn(*tensors): - all_kwargs = dict(zip(tensor_arg_names, tensors)) - all_kwargs.update(non_tensor_kwargs) - return fn(**all_kwargs) - - return tensor_only_fn, tensor_args - - -def _recompute_grad(fn, args, use_data_dep=_USE_DEFAULT, - tupleize_grads=False): +def _recompute_grad(fn, args, use_data_dep=_USE_DEFAULT, tupleize_grads=False): """See recompute_grad.""" for arg in args: if not isinstance(arg, framework_ops.Tensor): raise ValueError("All inputs to function must be Tensors") - use_data_dep_ = use_data_dep if use_data_dep_ == _USE_DEFAULT: use_data_dep_ = _is_on_tpu() @@ -559,11 +501,14 @@ def _recompute_grad(fn, args, use_data_dep=_USE_DEFAULT, grad_vars = grads[len(inputs):] return grad_inputs, grad_vars - # TODO(rsepassi): Replace with tf.custom_gradient @_fn_with_custom_grad(grad_fn) def fn_with_recompute(*args): cached_vs.append(variable_scope.get_variable_scope()) - cached_arg_scope.append(contrib_framework_ops.current_arg_scope()) + # TODO(rsepassi): Rm conditional in TF 1.4 + if hasattr(contrib_framework_ops, "current_arg_scope"): + cached_arg_scope.append(contrib_framework_ops.current_arg_scope()) + else: + cached_arg_scope.append({}) return fn(*args) return fn_with_recompute(*args) diff --git a/tensorflow/contrib/layers/python/layers/rev_block_lib_test.py b/tensorflow/contrib/layers/python/layers/rev_block_lib_test.py index 66ccc696f9..392a490be1 100644 --- a/tensorflow/contrib/layers/python/layers/rev_block_lib_test.py +++ b/tensorflow/contrib/layers/python/layers/rev_block_lib_test.py @@ -318,108 +318,6 @@ class RecomputeTest(test.TestCase): self.assertEqual(1, len(grads)) self.assertTrue(grads[0] is not None) - def testWithNontensorArgs(self): - @rev_block_lib.recompute_grad(tupleize_grads=True, - tensor_arg_names=["inputs"]) - def layer_with_recompute(inputs, plus=None): - var = variable_scope.get_variable("var", ()) - self.assertFalse(plus) # called with False below - if plus: - return var + inputs - else: - return var * inputs - - inputs = array_ops.ones((), dtypes.float32) - outputs = layer_with_recompute(inputs, plus=False) - loss = math_ops.square(outputs) - grads = gradients_impl.gradients(loss, variables.trainable_variables()) - self.assertEqual(1, len(grads)) - self.assertTrue(grads[0] is not None) - - -class MakeTensorOnlyTest(test.TestCase): - - def testMakeTensorOnly(self): - def fn(a, b, c, d=1, e=None, f=7): - return (a, b, c, d, e, f) - - t1 = array_ops.ones(()) - t2 = array_ops.ones(()) - t3 = array_ops.ones(()) - args = [1, t1, 3, t2] - kwargs = {"e": t3} - tensor_only_fn, tensor_args = rev_block_lib._make_tensor_only( - fn, args, kwargs, ["b", "d", "e"]) - self.assertAllEqual(tensor_args, [t1, t2, t3]) - out = tensor_only_fn(*tensor_args) - self.assertAllEqual(out, (1, t1, 3, t2, t3, 7)) - - def testMakeTensorOnlyPositionalArgsOnly(self): - def fn(a, b, c): - return (a, b, c) - - t1 = array_ops.ones(()) - t2 = array_ops.ones(()) - args = [t1, 3, t2] - tensor_only_fn, tensor_args = rev_block_lib._make_tensor_only( - fn, args, {}, ["a", "c"]) - self.assertAllEqual(tensor_args, [t1, t2]) - out = tensor_only_fn(*tensor_args) - self.assertAllEqual(out, (t1, 3, t2)) - - def testMakeTensorOnlyKwargsArgsOnly(self): - def fn(a=1, b=2, c=3): - return (a, b, c) - - t1 = array_ops.ones(()) - t2 = array_ops.ones(()) - args = [t1] - kwargs = {"c": t2} - tensor_only_fn, tensor_args = rev_block_lib._make_tensor_only( - fn, args, kwargs, ["a", "c"]) - self.assertAllEqual(tensor_args, [t1, t2]) - out = tensor_only_fn(*tensor_args) - self.assertAllEqual(out, (t1, 2, t2)) - - def testErrorOnMissingTensorArg(self): - def fn(a, b): - return (a, b) - - with self.assertRaisesWithPredicateMatch( - ValueError, "provide Tensor argument"): - rev_block_lib._make_tensor_only(fn, [], {"b": 2}, ["a"]) - - def testErrorOnSignatureSplats(self): - def fn1(a, *args): - return (a, args) - - err_msg = r"must not use \*args or \*\*kwargs" - with self.assertRaisesWithPredicateMatch(ValueError, err_msg): - rev_block_lib._make_tensor_only(fn1, [1, 2], {}, ["a"]) - - def fn2(a, **kwargs): - return (a, kwargs) - - with self.assertRaisesWithPredicateMatch(ValueError, err_msg): - rev_block_lib._make_tensor_only(fn2, [], {"a": 1, "b": 2}, ["a"]) - - def testErrorOnNonTensorForTensor(self): - def fn(a, b): - return (a, b) - - with self.assertRaisesWithPredicateMatch(TypeError, "must be a Tensor"): - rev_block_lib._make_tensor_only(fn, [2, 3], {}, ["a"]) - - def testErrorOnTensorForNonTensor(self): - def fn(a, b): - return (a, b) - - with self.assertRaisesWithPredicateMatch( - TypeError, "must not be a Tensor"): - t1 = array_ops.ones(()) - t2 = array_ops.ones(()) - rev_block_lib._make_tensor_only(fn, [t1, t2], {}, ["a"]) - class FnWithCustomGradTest(test.TestCase): -- GitLab From b133f8c70622e52f19631fd93d4b87ee21c52ac6 Mon Sep 17 00:00:00 2001 From: Akshay Modi Date: Fri, 20 Apr 2018 14:58:56 -0700 Subject: [PATCH 763/791] Move the guts of TFE_Execute into EagerExecute PiperOrigin-RevId: 193728072 --- tensorflow/c/eager/BUILD | 1 - tensorflow/c/eager/c_api.cc | 531 +----------------- tensorflow/core/common_runtime/eager/BUILD | 21 +- .../core/common_runtime/eager/execute.cc | 489 ++++++++++++++++ .../core/common_runtime/eager/execute.h | 7 + 5 files changed, 508 insertions(+), 541 deletions(-) diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD index d66386acbd..fae922ea3b 100644 --- a/tensorflow/c/eager/BUILD +++ b/tensorflow/c/eager/BUILD @@ -31,7 +31,6 @@ tf_cuda_library( "//tensorflow/core/common_runtime/eager:context", "//tensorflow/core/common_runtime/eager:eager_executor", "//tensorflow/core/common_runtime/eager:execute", - "//tensorflow/core/common_runtime/eager:execute_node", "//tensorflow/core/common_runtime/eager:kernel_and_device", "//tensorflow/core/common_runtime/eager:tensor_handle", "//tensorflow/core/common_runtime/eager:copy_to_device_node", diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc index b7a3097208..975bde7c7f 100644 --- a/tensorflow/c/eager/c_api.cc +++ b/tensorflow/c/eager/c_api.cc @@ -34,7 +34,6 @@ limitations under the License. #include "tensorflow/core/common_runtime/device_set.h" #include "tensorflow/core/common_runtime/eager/copy_to_device_node.h" #include "tensorflow/core/common_runtime/eager/execute.h" -#include "tensorflow/core/common_runtime/eager/execute_node.h" #include "tensorflow/core/common_runtime/function.h" #include "tensorflow/core/common_runtime/rendezvous_mgr.h" #include "tensorflow/core/framework/node_def_util.h" @@ -219,9 +218,6 @@ TF_Tensor* TFE_TensorHandleResolve(TFE_TensorHandle* h, TF_Status* status) { } return retval; } -} // extern "C" - -extern "C" { TFE_Op* TFE_NewOp(TFE_Context* ctx, const char* op_or_function_name, TF_Status* status) { @@ -423,531 +419,18 @@ void TFE_OpSetAttrFunctionList(TFE_Op* op, const char* attr_name, attr_name, tensorflow::gtl::ArraySlice( funcs.get(), num_values)); } -} // extern "C" - -namespace { - -// Initializes the step stats if needed. -void MaybeInitializeStepStats(tensorflow::StepStats* step_stats, - tensorflow::EagerContext* ctx) { - // Lazily initialize the RunMetadata with information about all devices if - // this is the first call. - while (step_stats->dev_stats_size() < ctx->devices()->size()) { - int device_idx = step_stats->dev_stats_size(); - auto* dev_stats = step_stats->add_dev_stats(); - dev_stats->set_device(ctx->devices()->at(device_idx)->name()); - } -} - -int StepStatsDeviceIndex(tensorflow::StepStats* step_stats, - tensorflow::EagerContext* ctx, - tensorflow::Device* device) { - // Find the current device's index. - if (device == nullptr) { - device = ctx->HostCPU(); - } - for (int i = 0; i < ctx->devices()->size(); ++i) { - if (ctx->devices()->at(i) == device || - ctx->devices()->at(i)->name() == device->name()) { - return i; - } - } - // TODO(apassos) do not fall back to host CPU if device is unknown. - return 0; -} - -tensorflow::Status ValidateInputTypeAndPlacement( - tensorflow::EagerContext* ctx, tensorflow::Device* op_device, - tensorflow::EagerOperation* op, const tensorflow::OpKernel* kernel, - tensorflow::RunMetadata* run_metadata) { - tensorflow::Device* host_device = ctx->HostCPU(); - const tensorflow::MemoryTypeVector& memtypes = kernel->input_memory_types(); - if (memtypes.size() != op->Inputs().size()) { - return tensorflow::errors::InvalidArgument( - "expected ", memtypes.size(), " inputs, got ", op->Inputs().size()); - } - for (int i = 0; i < op->Inputs().size(); ++i) { - const tensorflow::Device* expected_device = - memtypes[i] == tensorflow::HOST_MEMORY ? host_device : op_device; - tensorflow::TensorHandle* handle = op->Inputs()[i]; - tensorflow::Device* handle_device = nullptr; - TF_RETURN_IF_ERROR(handle->Device(&handle_device)); - const tensorflow::Device* actual_device = - handle_device == nullptr ? host_device : handle_device; - if (expected_device != actual_device) { - switch (ctx->GetDevicePlacementPolicy()) { - case tensorflow::DEVICE_PLACEMENT_SILENT_FOR_INT32: - // TODO(xpan): See if we could bubble python related error up - // to python level. - if (handle->dtype == tensorflow::DT_INT32) { - // Note: enabling silent copies of int32 tensors to match behavior - // of graph mode. - break; - } - TF_FALLTHROUGH_INTENDED; - case tensorflow::DEVICE_PLACEMENT_EXPLICIT: - return tensorflow::errors::InvalidArgument( - "Tensors on conflicting devices:" - " cannot compute ", - op->Name(), " as input #", i, " was expected to be on ", - expected_device->name(), " but is actually on ", - actual_device->name(), " (operation running on ", - op_device->name(), ")", - " Tensors can be copied explicitly using .gpu() or .cpu() " - "methods," - " or transparently copied by using tf.enable_eager_execution(" - "device_policy=tfe.DEVICE_PLACEMENT_SILENT). Copying tensors " - "between devices" - " may slow down your model"); - case tensorflow::DEVICE_PLACEMENT_WARN: - LOG(WARNING) << "before computing " << op->Name() << " input #" << i - << " was expected to be on " << expected_device->name() - << " but is actually on " << actual_device->name() - << " (operation running on " << op_device->name() - << "). This triggers a copy which can be a performance " - "bottleneck."; - break; - case tensorflow::DEVICE_PLACEMENT_SILENT: // Do nothing. - break; - } - // We are only here if the policy is warn or silent copies, so we should - // trigger a copy. - auto pre_time = tensorflow::Env::Default()->NowMicros(); - tensorflow::TensorHandle* copied_tensor = nullptr; - tensorflow::Status status = tensorflow::EagerCopyToDevice( - handle, ctx, expected_device->name().c_str(), &copied_tensor); - if (run_metadata != nullptr) { - auto* step_stats = run_metadata->mutable_step_stats(); - MaybeInitializeStepStats(step_stats, ctx); - // Record the sending on the source device for now. - int device_idx = StepStatsDeviceIndex(step_stats, ctx, handle_device); - auto* dev_stats = step_stats->mutable_dev_stats(device_idx); - auto* node_stats = dev_stats->add_node_stats(); - node_stats->set_node_name("_Send"); - node_stats->set_all_start_micros(pre_time); - node_stats->set_op_end_rel_micros( - tensorflow::Env::Default()->NowMicros() - pre_time); - } - if (!status.ok()) { - if (copied_tensor != nullptr) copied_tensor->Unref(); - return tensorflow::errors::Internal( - "Failed copying input tensor from ", actual_device->name(), " to ", - expected_device->name(), " in order to run ", op->Name(), ": ", - status.error_message()); - } - handle->Unref(); - handle = copied_tensor; - (*op->MutableInputs())[i] = copied_tensor; - } - if (handle->dtype != kernel->input_type(i)) { - return tensorflow::errors::InvalidArgument( - "cannot compute ", op->Name(), " as input #", i, - " was expected to be a ", - tensorflow::DataTypeString(kernel->input_type(i)), - " tensor but is a ", tensorflow::DataTypeString(handle->dtype), - " tensor"); - } - } - return tensorflow::Status::OK(); -} - -tensorflow::Device* SelectDevice(const tensorflow::NodeDef& ndef, - tensorflow::EagerContext* ctx, - TF_Status* status) { - tensorflow::DeviceSet ds; - for (tensorflow::Device* d : *ctx->devices()) { - ds.AddDevice(d); - } - tensorflow::DeviceTypeVector final_devices; - status->status = tensorflow::SupportedDeviceTypesForNode( - ds.PrioritizedDeviceTypeList(), ndef, &final_devices); - if (!status->status.ok()) { - return nullptr; - } - if (final_devices.empty()) { - status->status = tensorflow::errors::Internal( - "Could not find valid device for node ", ndef.DebugString()); - return nullptr; - } - for (tensorflow::Device* d : *ctx->devices()) { - if (d->device_type() == final_devices[0].type_string()) { - return d; - } - } - status->status = tensorflow::errors::Unknown( - "Could not find a device for node ", ndef.DebugString()); - return nullptr; -} - -#ifdef TENSORFLOW_EAGER_USE_XLA -// Synthesizes and returns a wrapper function over `op`, which must be a -// primitive op (e.g. matmul). -// -// The wrapper function conforms to the function signature expected by -// _XlaLaunchOp, with input params ordered by . For example, if the op has input params , they will be reordered to as the input params to the synthesized function. -// -// It populates `const_input_types`, `arg_input_types` and -// `op_input_to_func_input` based on the reordering results, that the caller can -// use them to build an _XlaLaunchOp. On error, it returns NULL, and sets -// `status` accordingly. -const tensorflow::FunctionDef* OpToFunction( - TFE_Op* op, std::vector* const_input_types, - std::vector* arg_input_types, - tensorflow::gtl::FlatMap* op_input_to_func_input, - TF_Status* status) { - DCHECK(!op->operation.is_function()); - - tensorflow::FunctionDef fdef; - - // Get the OpDef of the op we are trying to encapsulate. - TFE_Context* ctx = op->operation.ctx; - const tensorflow::OpRegistrationData* op_data; - { - status->status = - ctx->context.FindFunctionOpData(op->operation.Name(), &op_data); - if (!status->status.ok()) { - return nullptr; - } - } - const tensorflow::OpDef& op_def = op_data->op_def; - - tensorflow::OpDef* signature = fdef.mutable_signature(); - - // Handle constant inputs. - const std::unordered_set const_inputs( - *tensorflow::XlaOpRegistry::CompileTimeConstantInputs( - op->operation.Name())); - - // First add place holders for the input args, so that we can refer to them by - // position in the next loop. Also tally up the resource inputs. - int num_resource_inputs = 0; - for (int i = 0; i < op_def.input_arg_size(); ++i) { - if (op_def.input_arg(i).type() == tensorflow::DT_RESOURCE) { - ++num_resource_inputs; - } - signature->add_input_arg(); - } - - // Now we map the input params from `op_def` to `signature`, where the param - // ordering for `signature` is: . - int const_index = 0; - int arg_index = const_inputs.size(); - int resource_index = op_def.input_arg_size() - num_resource_inputs; - for (int i = 0; i < op_def.input_arg_size(); ++i) { - const tensorflow::OpDef::ArgDef& op_input_arg = op_def.input_arg(i); - tensorflow::OpDef::ArgDef* func_input_arg = nullptr; - if (const_inputs.find(op_input_arg.name()) != const_inputs.end()) { - VLOG(1) << "For const input, mapping op input " << i << " to func input " - << const_index; - (*op_input_to_func_input)[i] = const_index; - func_input_arg = signature->mutable_input_arg(const_index++); - const_input_types->push_back( - static_cast(op->operation.Inputs()[i]->dtype)); - } else if (op_input_arg.type() == tensorflow::DT_RESOURCE) { - VLOG(1) << "For resource input, mapping op input " << i - << " to func input " << resource_index; - (*op_input_to_func_input)[i] = resource_index; - func_input_arg = signature->mutable_input_arg(resource_index++); - } else { - VLOG(1) << "For arg input, mapping op input " << i << " to func input " - << arg_index; - (*op_input_to_func_input)[i] = arg_index; - func_input_arg = signature->mutable_input_arg(arg_index++); - arg_input_types->push_back( - static_cast(op->operation.Inputs()[i]->dtype)); - } - - func_input_arg->set_name(op_input_arg.name()); - func_input_arg->set_type(op->operation.Inputs()[i]->dtype); - } - VLOG(1) << "Added OpDef Inputs: " << fdef.DebugString(); - - // Resources args are at the end of the function input params, and we should - // have iterated over all of them. - DCHECK_EQ(signature->input_arg_size(), resource_index); - - // Make the synthesized function's name unique. - signature->set_name(tensorflow::strings::StrCat( - op_def.name(), func_id_generator.fetch_add(1))); - - // Add the node def and set its input names to match op_def's names. - const tensorflow::NodeDef& ndef = - op->operation.MutableAttrs()->BuildNodeDef(); - DCHECK_EQ(signature->input_arg_size(), ndef.input_size()); - *fdef.add_node_def() = ndef; - for (int i = 0; i < op_def.input_arg_size(); ++i) { - fdef.mutable_node_def(0)->set_input(i, op_def.input_arg(i).name()); - } - VLOG(1) << "Added NodeDef: " << fdef.DebugString(); - - // Fix the output names and set output types. - for (int i = 0; i < op_def.output_arg_size(); ++i) { - tensorflow::OpDef::ArgDef* arg = signature->add_output_arg(); - const tensorflow::OpDef::ArgDef& op_def_arg = op_def.output_arg(i); - const string& out_tensor_name = tensorflow::strings::StrCat( - ndef.name(), ":", op_def_arg.name(), ":", 0); - arg->set_name(op_def_arg.name()); - (*fdef.mutable_ret())[op_def_arg.name()] = out_tensor_name; - const string& type_attr = op_def_arg.type_attr(); - if (!type_attr.empty()) { - auto i = ndef.attr().find(type_attr); - if (i == ndef.attr().end()) { - status->status = tensorflow::errors::InvalidArgument( - tensorflow::strings::StrCat("Could not find attr ", type_attr, - " in NodeDef ", ndef.DebugString())); - return nullptr; - } - arg->set_type(i->second.type()); - } - } - VLOG(1) << "Fixed Output names and all types: " << fdef.DebugString(); - - status->status = ctx->context.AddFunctionDef(fdef); - if (!status->status.ok()) return nullptr; - const auto ret = ctx->context.FindFunctionDef(signature->name()); - DCHECK(ret != nullptr); - return ret; -} - -// Builds an _XLALaunchOp as a wrapper over 'op', so that 'op' can be executed -// via XLA. -std::unique_ptr BuildXlaLaunch(TFE_Op* op, TF_Status* status) { - VLOG(1) << "Creating _XlaLaunchOp for TFE_Op " << op->operation.Name(); - auto launch_op = std::unique_ptr( - TFE_NewOp(op->operation.ctx, "_XlaLaunch", status)); - if (TF_GetCode(status) != TF_OK) return nullptr; - if (op->operation.device) { - TFE_OpSetDevice(launch_op.get(), op->operation.device->name().c_str(), - status); - if (TF_GetCode(status) != TF_OK) return nullptr; - } - - const tensorflow::FunctionDef* fdef; - { fdef = op->operation.ctx->FindFunctionDef(op->operation.Name()); } - std::vector const_input_types; - std::vector arg_input_types; - tensorflow::gtl::FlatMap op_input_to_func_input; - if (fdef == nullptr) { - // See if this is a primitive op, and if so create a function for it, so - // that _XlaLaunchOp can access it. - fdef = OpToFunction(op, &const_input_types, &arg_input_types, - &op_input_to_func_input, status); - if (!status->status.ok()) return nullptr; - } else { - // TODO(hongm): XlaOpRegistry::CompileTimeConstantInputs() does not work for - // functions, so we need to find another way to handle constant inputs. - for (int i = const_input_types.size(); - i < fdef->signature().input_arg_size(); ++i) { - VLOG(1) << "Adding Targs from input arg " << i; - const tensorflow::OpDef::ArgDef& arg = fdef->signature().input_arg(i); - arg_input_types.push_back(static_cast(arg.type())); - } - } - DCHECK(fdef != nullptr); - - // Copy inputs and their devices. - // Since input param reordering may have occurred between `op` and `launch_op` - // via `op_input_to_func_input`, adjust the actual inputs accordingly. - *launch_op->operation.MutableInputs() = op->operation.Inputs(); - for (tensorflow::TensorHandle* h : launch_op->operation.Inputs()) { - h->Ref(); - } - if (!op_input_to_func_input.empty()) { - DCHECK_EQ(op->operation.Inputs().size(), op_input_to_func_input.size()); - for (int i = 0; i < op_input_to_func_input.size(); ++i) { - VLOG(1) << "mapping op input " << i << " to func input " - << op_input_to_func_input[i]; - - (*launch_op->operation.MuableInputs())[op_input_to_func_input[i]] = - op->operation.Inputs()[i]; - } - } - launch_op->operation.MutableAttrs()->NumInputs(op->operation.Inputs().size()); - - TFE_OpSetAttrTypeList(launch_op.get(), "Tconstants", const_input_types.data(), - const_input_types.size()); - - // Set Targs and Nresources attrs. - TFE_OpSetAttrTypeList(launch_op.get(), "Targs", arg_input_types.data(), - arg_input_types.size()); - const int num_resource_inputs = fdef->signature().input_arg_size() - - const_input_types.size() - - arg_input_types.size(); - TFE_OpSetAttrInt(launch_op.get(), "Nresources", num_resource_inputs); - - // Set Tresults attr. - std::vector tresults; - for (const tensorflow::OpDef::ArgDef& arg : fdef->signature().output_arg()) { - tresults.push_back(static_cast(arg.type())); - } - TFE_OpSetAttrTypeList(launch_op.get(), "Tresults", tresults.data(), - tresults.size()); - - // Set function attr. - tensorflow::AttrValue attr_value; - tensorflow::NameAttrList* func = attr_value.mutable_func(); - func->set_name(fdef->signature().name()); - launch_op->attrs.Set("function", attr_value); - - return launch_op; -} -#endif // TENSORFLOW_EAGER_USE_XLA -} // namespace - -extern "C" { - -void TFE_Execute(TFE_Op* tfe_op, TFE_TensorHandle** retvals, int* num_retvals, +void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals, TF_Status* status) { - tensorflow::EagerOperation* op = &tfe_op->operation; - tensorflow::EagerContext* ctx = op->EagerContext(); - status->status = ctx->GetStatus(); + tensorflow::gtl::InlinedVector handle_retvals( + *num_retvals); + status->status = + tensorflow::EagerExecute(&op->operation, &handle_retvals, num_retvals); if (!status->status.ok()) { return; } -#ifdef TENSORFLOW_EAGER_USE_XLA - std::unique_ptr xla_launch_op; - if (op->UseXla() && op->Name() != "_XlaLaunch") { - xla_launch_op = BuildXlaLaunch(op, status); - if (!status->status.ok()) { - return; - } - op = xla_launch_op.get(); - } -#endif // TENSORFLOW_EAGER_USE_XLA - // Ensure all resource-touching ops run in the device the resource is, - // regardless of anything else that has been specified. This is identical to - // the graph mode behavior. - for (int i = 0; i < op->Inputs().size(); ++i) { - tensorflow::Device* input_op_device = nullptr; - status->status = op->Inputs()[i]->OpDevice(&input_op_device); - if (!status->status.ok()) return; - VLOG(2) << "for op " << op->Name() << " input " << i << " " - << tensorflow::DataTypeString(op->Inputs()[i]->dtype) << " " - << (input_op_device == nullptr ? "cpu" : input_op_device->name()) - << " " << (op->Device() == nullptr ? "cpu" : op->Device()->name()); - if (op->Inputs()[i]->dtype == tensorflow::DT_RESOURCE && - (input_op_device != op->Device() || input_op_device == nullptr)) { - tensorflow::Device* d = - input_op_device == nullptr ? ctx->HostCPU() : input_op_device; - VLOG(1) << "Changing device of operation " << op->Name() << " to " - << d->name() << " because input #" << i - << " is a resource in this device."; - op->SetDevice(d); - } - } - tensorflow::Device* device = op->Device(); - - tensorflow::Fprint128 cache_key = op->MutableAttrs()->CacheKey( - device == nullptr ? "unspecified" : device->name()); - tensorflow::KernelAndDevice* kernel = ctx->GetCachedKernel(cache_key); - if (kernel == nullptr) { - const tensorflow::NodeDef& ndef = op->MutableAttrs()->BuildNodeDef(); - if (device == nullptr) { - device = SelectDevice(ndef, ctx, status); - if (!status->status.ok()) { - return; - } - } - CHECK(device != nullptr); - if (ctx->LogDevicePlacement()) { - LOG(INFO) << "Executing op " << ndef.op() << " in device " - << device->name(); - } - kernel = new tensorflow::KernelAndDevice(ctx->GetRendezvous()); - // Knowledge of the implementation of Init (and in-turn - // FunctionLibraryRuntime::CreateKernel) tells us that ctx->func_lib_def - // will be accessed, so grab on to the lock. - // See WARNING comment in Execute (before kernel->Run) - would be nice to - // rework to avoid this subtlety. - tensorflow::tf_shared_lock l(*ctx->FunctionsMu()); - status->status = - tensorflow::KernelAndDevice::Init(ndef, ctx->func_lib(device), kernel); - if (!status->status.ok()) { - delete kernel; - return; - } - // Update output_dtypes inside `kernel`. - const tensorflow::OpDef* op_def = nullptr; - const tensorflow::FunctionDef* function_def = - ctx->FuncLibDef()->Find(ndef.op()); - if (function_def != nullptr) { - op_def = &(function_def->signature()); - } - if (op_def == nullptr) { - status->status = OpDefForOp(ndef.op().c_str(), &op_def); - if (!status->status.ok()) { - return; - } - } - tensorflow::DataTypeVector input_dtypes; - status->status = InOutTypesForNode(ndef, *op_def, &input_dtypes, - kernel->mutable_output_dtypes()); - if (!status->status.ok()) { - return; - } - ctx->AddKernelToCache(cache_key, kernel); - } - const tensorflow::DataTypeVector& output_dtypes = kernel->output_dtypes(); - const int output_dtypes_size = output_dtypes.size(); - if (output_dtypes_size > *num_retvals) { - TF_SetStatus(status, TF_INVALID_ARGUMENT, - tensorflow::strings::StrCat("Expecting ", output_dtypes.size(), - " outputs, but *num_retvals is ", - *num_retvals) - .c_str()); - return; - } - *num_retvals = output_dtypes_size; - if (device == nullptr) { - // TODO(apassos) debug how the assignment below might return a different - // device from the one requested above. - device = kernel->device(); - } - status->status = ValidateInputTypeAndPlacement( - ctx, device, op, kernel->kernel(), - ctx->ShouldStoreMetadata() ? ctx->RunMetadataProto() : nullptr); - if (!status->status.ok()) return; - std::unique_ptr maybe_stats; - if (ctx->ShouldStoreMetadata()) { - maybe_stats.reset(new tensorflow::NodeExecStats); - maybe_stats->set_node_name(op->Name()); - maybe_stats->set_all_start_micros(tensorflow::Env::Default()->NowMicros()); - maybe_stats->set_op_start_rel_micros(0); - maybe_stats->set_scheduled_micros(tensorflow::Env::Default()->NowMicros()); - // TODO(apassos) track referenced tensors - } - if (ctx->Async()) { - // Note that for async mode, execution order will make sure that all - // input handles are ready before executing them. - // TODO(agarwal): Consider executing "cheap" kernels inline for performance. - tensorflow::gtl::InlinedVector handle_retvals( - *num_retvals); - tensorflow::uint64 id = ctx->NextId(); - for (int i = 0; i < *num_retvals; ++i) { - tensorflow::TensorHandle* h = - new tensorflow::TensorHandle(id, output_dtypes[i], ctx); - retvals[i] = new TFE_TensorHandle(h); - handle_retvals[i] = h; - } - tensorflow::EagerNode* node = new tensorflow::ExecuteNode( - id, ctx, op->Device(), op->Inputs(), kernel, maybe_stats.release(), - output_dtypes, handle_retvals); - ctx->ExecutorAdd(node); - } else { - // Execute checks if retvals[i] is nullptr or not to figure if it needs to - // allocate it. - tensorflow::gtl::InlinedVector handle_retvals( - *num_retvals); - status->status = tensorflow::EagerExecute( - ctx, op->Device(), op->Inputs(), kernel, maybe_stats.get(), - handle_retvals.data(), *num_retvals); - for (int i = 0; i < *num_retvals; ++i) { - retvals[i] = new TFE_TensorHandle(handle_retvals[i]); - } + for (int i = 0; i < *num_retvals; ++i) { + retvals[i] = new TFE_TensorHandle(handle_retvals[i]); } } diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD index 00ac4a4e47..13d6b021b5 100644 --- a/tensorflow/core/common_runtime/eager/BUILD +++ b/tensorflow/core/common_runtime/eager/BUILD @@ -154,26 +154,15 @@ tf_cc_test( cc_library( name = "execute", srcs = ["execute.cc"], - hdrs = ["execute.h"], - deps = [ - ":context", - ":copy_to_device_node", - ":kernel_and_device", - ":tensor_handle", - "//tensorflow/core:core_cpu_lib", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:protos_all_cc", + hdrs = [ + "execute.h", + "execute_node.h", ], -) - -cc_library( - name = "execute_node", - hdrs = ["execute_node.h"], deps = [ ":context", + ":copy_to_device_node", ":eager_executor", - ":execute", + ":eager_operation", ":kernel_and_device", ":tensor_handle", "//tensorflow/core:core_cpu_lib", diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc index 98e8471102..a514f81e14 100644 --- a/tensorflow/core/common_runtime/eager/execute.cc +++ b/tensorflow/core/common_runtime/eager/execute.cc @@ -18,8 +18,10 @@ limitations under the License. #include #include "tensorflow/core/common_runtime/device.h" +#include "tensorflow/core/common_runtime/device_set.h" #include "tensorflow/core/common_runtime/eager/context.h" #include "tensorflow/core/common_runtime/eager/copy_to_device_node.h" +#include "tensorflow/core/common_runtime/eager/execute_node.h" #include "tensorflow/core/common_runtime/eager/kernel_and_device.h" #include "tensorflow/core/common_runtime/eager/tensor_handle.h" #include "tensorflow/core/framework/step_stats.pb.h" @@ -32,6 +34,493 @@ limitations under the License. namespace tensorflow { +namespace { + +// Initializes the step stats if needed. +void MaybeInitializeStepStats(StepStats* step_stats, EagerContext* ctx) { + // Lazily initialize the RunMetadata with information about all devices if + // this is the first call. + while (step_stats->dev_stats_size() < ctx->devices()->size()) { + int device_idx = step_stats->dev_stats_size(); + auto* dev_stats = step_stats->add_dev_stats(); + dev_stats->set_device(ctx->devices()->at(device_idx)->name()); + } +} + +int StepStatsDeviceIndex(StepStats* step_stats, EagerContext* ctx, + Device* device) { + // Find the current device's index. + if (device == nullptr) { + device = ctx->HostCPU(); + } + for (int i = 0; i < ctx->devices()->size(); ++i) { + if (ctx->devices()->at(i) == device || + ctx->devices()->at(i)->name() == device->name()) { + return i; + } + } + // TODO(apassos) do not fall back to host CPU if device is unknown. + return 0; +} + +Status ValidateInputTypeAndPlacement(EagerContext* ctx, Device* op_device, + EagerOperation* op, const OpKernel* kernel, + RunMetadata* run_metadata) { + Device* host_device = ctx->HostCPU(); + const MemoryTypeVector& memtypes = kernel->input_memory_types(); + if (memtypes.size() != op->Inputs().size()) { + return errors::InvalidArgument("expected ", memtypes.size(), + " inputs, got ", op->Inputs().size()); + } + for (int i = 0; i < op->Inputs().size(); ++i) { + const Device* expected_device = + memtypes[i] == HOST_MEMORY ? host_device : op_device; + TensorHandle* handle = op->Inputs()[i]; + Device* handle_device = nullptr; + TF_RETURN_IF_ERROR(handle->Device(&handle_device)); + const Device* actual_device = + handle_device == nullptr ? host_device : handle_device; + if (expected_device != actual_device) { + switch (ctx->GetDevicePlacementPolicy()) { + case DEVICE_PLACEMENT_SILENT_FOR_INT32: + // TODO(xpan): See if we could bubble python related error up + // to python level. + if (handle->dtype == DT_INT32) { + // Note: enabling silent copies of int32 tensors to match behavior + // of graph mode. + break; + } + TF_FALLTHROUGH_INTENDED; + case DEVICE_PLACEMENT_EXPLICIT: + return errors::InvalidArgument( + "Tensors on conflicting devices:" + " cannot compute ", + op->Name(), " as input #", i, " was expected to be on ", + expected_device->name(), " but is actually on ", + actual_device->name(), " (operation running on ", + op_device->name(), ")", + " Tensors can be copied explicitly using .gpu() or .cpu() " + "methods," + " or transparently copied by using tf.enable_eager_execution(" + "device_policy=tfe.DEVICE_PLACEMENT_SILENT). Copying tensors " + "between devices" + " may slow down your model"); + case DEVICE_PLACEMENT_WARN: + LOG(WARNING) << "before computing " << op->Name() << " input #" << i + << " was expected to be on " << expected_device->name() + << " but is actually on " << actual_device->name() + << " (operation running on " << op_device->name() + << "). This triggers a copy which can be a performance " + "bottleneck."; + break; + case DEVICE_PLACEMENT_SILENT: // Do nothing. + break; + } + // We are only here if the policy is warn or silent copies, so we should + // trigger a copy. + auto pre_time = Env::Default()->NowMicros(); + TensorHandle* copied_tensor = nullptr; + Status status = EagerCopyToDevice( + handle, ctx, expected_device->name().c_str(), &copied_tensor); + if (run_metadata != nullptr) { + auto* step_stats = run_metadata->mutable_step_stats(); + MaybeInitializeStepStats(step_stats, ctx); + // Record the sending on the source device for now. + int device_idx = StepStatsDeviceIndex(step_stats, ctx, handle_device); + auto* dev_stats = step_stats->mutable_dev_stats(device_idx); + auto* node_stats = dev_stats->add_node_stats(); + node_stats->set_node_name("_Send"); + node_stats->set_all_start_micros(pre_time); + node_stats->set_op_end_rel_micros(Env::Default()->NowMicros() - + pre_time); + } + if (!status.ok()) { + if (copied_tensor != nullptr) copied_tensor->Unref(); + return errors::Internal("Failed copying input tensor from ", + actual_device->name(), " to ", + expected_device->name(), " in order to run ", + op->Name(), ": ", status.error_message()); + } + handle->Unref(); + handle = copied_tensor; + (*op->MutableInputs())[i] = copied_tensor; + } + if (handle->dtype != kernel->input_type(i)) { + return errors::InvalidArgument( + "cannot compute ", op->Name(), " as input #", i, + " was expected to be a ", DataTypeString(kernel->input_type(i)), + " tensor but is a ", DataTypeString(handle->dtype), " tensor"); + } + } + return Status::OK(); +} + +Status SelectDevice(const NodeDef& ndef, EagerContext* ctx, Device** device) { + DeviceSet ds; + for (Device* d : *ctx->devices()) { + ds.AddDevice(d); + } + DeviceTypeVector final_devices; + auto status = SupportedDeviceTypesForNode(ds.PrioritizedDeviceTypeList(), + ndef, &final_devices); + if (!status.ok()) return status; + if (final_devices.empty()) { + return errors::Internal("Could not find valid device for node ", + ndef.DebugString()); + } + for (Device* d : *ctx->devices()) { + if (d->device_type() == final_devices[0].type_string()) { + *device = d; + return Status::OK(); + } + } + return errors::Unknown("Could not find a device for node ", + ndef.DebugString()); +} + +#ifdef TENSORFLOW_EAGER_USE_XLA +// Synthesizes and returns a wrapper function over `op`, which must be a +// primitive op (e.g. matmul). +// +// The wrapper function conforms to the function signature expected by +// _XlaLaunchOp, with input params ordered by . For example, if the op has input params , they will be reordered to as the input params to the synthesized function. +// +// It populates `const_input_types`, `arg_input_types` and +// `op_input_to_func_input` based on the reordering results, that the caller can +// use them to build an _XlaLaunchOp. On error, it returns NULL, and sets +// `status` accordingly. +const FunctionDef* OpToFunction(TFE_Op* op, + std::vector* const_input_types, + std::vector* arg_input_types, + gtl::FlatMap* op_input_to_func_input, + TF_Status* status) { + DCHECK(!op->operation.is_function()); + + FunctionDef fdef; + + // Get the OpDef of the op we are trying to encapsulate. + TFE_Context* ctx = op->operation.ctx; + const OpRegistrationData* op_data; + { + status = ctx->context.FindFunctionOpData(op->operation.Name(), &op_data); + if (!status.ok()) { + return nullptr; + } + } + const OpDef& op_def = op_data->op_def; + + OpDef* signature = fdef.mutable_signature(); + + // Handle constant inputs. + const std::unordered_set const_inputs( + *XlaOpRegistry::CompileTimeConstantInputs(op->operation.Name())); + + // First add place holders for the input args, so that we can refer to them by + // position in the next loop. Also tally up the resource inputs. + int num_resource_inputs = 0; + for (int i = 0; i < op_def.input_arg_size(); ++i) { + if (op_def.input_arg(i).type() == DT_RESOURCE) { + ++num_resource_inputs; + } + signature->add_input_arg(); + } + + // Now we map the input params from `op_def` to `signature`, where the param + // ordering for `signature` is: . + int const_index = 0; + int arg_index = const_inputs.size(); + int resource_index = op_def.input_arg_size() - num_resource_inputs; + for (int i = 0; i < op_def.input_arg_size(); ++i) { + const OpDef::ArgDef& op_input_arg = op_def.input_arg(i); + OpDef::ArgDef* func_input_arg = nullptr; + if (const_inputs.find(op_input_arg.name()) != const_inputs.end()) { + VLOG(1) << "For const input, mapping op input " << i << " to func input " + << const_index; + (*op_input_to_func_input)[i] = const_index; + func_input_arg = signature->mutable_input_arg(const_index++); + const_input_types->push_back( + static_cast(op->operation.Inputs()[i]->dtype)); + } else if (op_input_arg.type() == DT_RESOURCE) { + VLOG(1) << "For resource input, mapping op input " << i + << " to func input " << resource_index; + (*op_input_to_func_input)[i] = resource_index; + func_input_arg = signature->mutable_input_arg(resource_index++); + } else { + VLOG(1) << "For arg input, mapping op input " << i << " to func input " + << arg_index; + (*op_input_to_func_input)[i] = arg_index; + func_input_arg = signature->mutable_input_arg(arg_index++); + arg_input_types->push_back( + static_cast(op->operation.Inputs()[i]->dtype)); + } + + func_input_arg->set_name(op_input_arg.name()); + func_input_arg->set_type(op->operation.Inputs()[i]->dtype); + } + VLOG(1) << "Added OpDef Inputs: " << fdef.DebugString(); + + // Resources args are at the end of the function input params, and we should + // have iterated over all of them. + DCHECK_EQ(signature->input_arg_size(), resource_index); + + // Make the synthesized function's name unique. + signature->set_name( + strings::StrCat(op_def.name(), func_id_generator.fetch_add(1))); + + // Add the node def and set its input names to match op_def's names. + const NodeDef& ndef = op->operation.MutableAttrs()->BuildNodeDef(); + DCHECK_EQ(signature->input_arg_size(), ndef.input_size()); + *fdef.add_node_def() = ndef; + for (int i = 0; i < op_def.input_arg_size(); ++i) { + fdef.mutable_node_def(0)->set_input(i, op_def.input_arg(i).name()); + } + VLOG(1) << "Added NodeDef: " << fdef.DebugString(); + + // Fix the output names and set output types. + for (int i = 0; i < op_def.output_arg_size(); ++i) { + OpDef::ArgDef* arg = signature->add_output_arg(); + const OpDef::ArgDef& op_def_arg = op_def.output_arg(i); + const string& out_tensor_name = + strings::StrCat(ndef.name(), ":", op_def_arg.name(), ":", 0); + arg->set_name(op_def_arg.name()); + (*fdef.mutable_ret())[op_def_arg.name()] = out_tensor_name; + const string& type_attr = op_def_arg.type_attr(); + if (!type_attr.empty()) { + auto i = ndef.attr().find(type_attr); + if (i == ndef.attr().end()) { + status = errors::InvalidArgument( + strings::StrCat("Could not find attr ", type_attr, " in NodeDef ", + ndef.DebugString())); + return nullptr; + } + arg->set_type(i->second.type()); + } + } + VLOG(1) << "Fixed Output names and all types: " << fdef.DebugString(); + + status = ctx->context.AddFunctionDef(fdef); + if (!status.ok()) return nullptr; + const auto ret = ctx->context.FindFunctionDef(signature->name()); + DCHECK(ret != nullptr); + return ret; +} + +// Builds an _XLALaunchOp as a wrapper over 'op', so that 'op' can be executed +// via XLA. +std::unique_ptr BuildXlaLaunch(TFE_Op* op, TF_Status* status) { + VLOG(1) << "Creating _XlaLaunchOp for TFE_Op " << op->operation.Name(); + auto launch_op = std::unique_ptr( + TFE_NewOp(op->operation.ctx, "_XlaLaunch", status)); + if (TF_GetCode(status) != TF_OK) return nullptr; + if (op->operation.device) { + TFE_OpSetDevice(launch_op.get(), op->operation.device->name().c_str(), + status); + if (TF_GetCode(status) != TF_OK) return nullptr; + } + + const FunctionDef* fdef; + { fdef = op->operation.ctx->FindFunctionDef(op->operation.Name()); } + std::vector const_input_types; + std::vector arg_input_types; + gtl::FlatMap op_input_to_func_input; + if (fdef == nullptr) { + // See if this is a primitive op, and if so create a function for it, so + // that _XlaLaunchOp can access it. + fdef = OpToFunction(op, &const_input_types, &arg_input_types, + &op_input_to_func_input, status); + if (!status.ok()) return nullptr; + } else { + // TODO(hongm): XlaOpRegistry::CompileTimeConstantInputs() does not work for + // functions, so we need to find another way to handle constant inputs. + for (int i = const_input_types.size(); + i < fdef->signature().input_arg_size(); ++i) { + VLOG(1) << "Adding Targs from input arg " << i; + const OpDef::ArgDef& arg = fdef->signature().input_arg(i); + arg_input_types.push_back(static_cast(arg.type())); + } + } + DCHECK(fdef != nullptr); + + // Copy inputs and their devices. + // Since input param reordering may have occurred between `op` and `launch_op` + // via `op_input_to_func_input`, adjust the actual inputs accordingly. + *launch_op->operation.MutableInputs() = op->operation.Inputs(); + for (TensorHandle* h : launch_op->operation.Inputs()) { + h->Ref(); + } + if (!op_input_to_func_input.empty()) { + DCHECK_EQ(op->operation.Inputs().size(), op_input_to_func_input.size()); + for (int i = 0; i < op_input_to_func_input.size(); ++i) { + VLOG(1) << "mapping op input " << i << " to func input " + << op_input_to_func_input[i]; + + (*launch_op->operation.MuableInputs())[op_input_to_func_input[i]] = + op->operation.Inputs()[i]; + } + } + launch_op->operation.MutableAttrs()->NumInputs(op->operation.Inputs().size()); + + TFE_OpSetAttrTypeList(launch_op.get(), "Tconstants", const_input_types.data(), + const_input_types.size()); + + // Set Targs and Nresources attrs. + TFE_OpSetAttrTypeList(launch_op.get(), "Targs", arg_input_types.data(), + arg_input_types.size()); + const int num_resource_inputs = fdef->signature().input_arg_size() - + const_input_types.size() - + arg_input_types.size(); + TFE_OpSetAttrInt(launch_op.get(), "Nresources", num_resource_inputs); + + // Set Tresults attr. + std::vector tresults; + for (const OpDef::ArgDef& arg : fdef->signature().output_arg()) { + tresults.push_back(static_cast(arg.type())); + } + TFE_OpSetAttrTypeList(launch_op.get(), "Tresults", tresults.data(), + tresults.size()); + + // Set function attr. + AttrValue attr_value; + NameAttrList* func = attr_value.mutable_func(); + func->set_name(fdef->signature().name()); + launch_op->attrs.Set("function", attr_value); + + return launch_op; +} +#endif // TENSORFLOW_EAGER_USE_XLA + +} // namespace + +Status EagerExecute(EagerOperation* op, + gtl::InlinedVector* retvals, + int* num_retvals) { + EagerContext* ctx = op->EagerContext(); + auto status = ctx->GetStatus(); + if (!status.ok()) return status; +#ifdef TENSORFLOW_EAGER_USE_XLA + std::unique_ptr xla_launch_op; + if (op->UseXla() && op->Name() != "_XlaLaunch") { + xla_launch_op = BuildXlaLaunch(op, status); + if (!status.ok()) return status; + op = xla_launch_op.get(); + } +#endif // TENSORFLOW_EAGER_USE_XLA + // Ensure all resource-touching ops run in the device the resource is, + // regardless of anything else that has been specified. This is identical to + // the graph mode behavior. + for (int i = 0; i < op->Inputs().size(); ++i) { + Device* input_op_device = nullptr; + status = op->Inputs()[i]->OpDevice(&input_op_device); + if (!status.ok()) return status; + VLOG(2) << "for op " << op->Name() << " input " << i << " " + << DataTypeString(op->Inputs()[i]->dtype) << " " + << (input_op_device == nullptr ? "cpu" : input_op_device->name()) + << " " << (op->Device() == nullptr ? "cpu" : op->Device()->name()); + if (op->Inputs()[i]->dtype == DT_RESOURCE && + (input_op_device != op->Device() || input_op_device == nullptr)) { + Device* d = input_op_device == nullptr ? ctx->HostCPU() : input_op_device; + VLOG(1) << "Changing device of operation " << op->Name() << " to " + << d->name() << " because input #" << i + << " is a resource in this device."; + op->SetDevice(d); + } + } + Device* device = op->Device(); + + Fprint128 cache_key = op->MutableAttrs()->CacheKey( + device == nullptr ? "unspecified" : device->name()); + KernelAndDevice* kernel = ctx->GetCachedKernel(cache_key); + if (kernel == nullptr) { + const NodeDef& ndef = op->MutableAttrs()->BuildNodeDef(); + if (device == nullptr) { + status = SelectDevice(ndef, ctx, &device); + if (!status.ok()) return status; + } + CHECK(device != nullptr); + if (ctx->LogDevicePlacement()) { + LOG(INFO) << "Executing op " << ndef.op() << " in device " + << device->name(); + } + kernel = new KernelAndDevice(ctx->GetRendezvous()); + // Knowledge of the implementation of Init (and in-turn + // FunctionLibraryRuntime::CreateKernel) tells us that ctx->func_lib_def + // will be accessed, so grab on to the lock. + // See WARNING comment in Execute (before kernel->Run) - would be nice to + // rework to avoid this subtlety. + tf_shared_lock l(*ctx->FunctionsMu()); + status = KernelAndDevice::Init(ndef, ctx->func_lib(device), kernel); + if (!status.ok()) { + delete kernel; + return status; + } + // Update output_dtypes inside `kernel`. + const OpDef* op_def = nullptr; + const FunctionDef* function_def = ctx->FuncLibDef()->Find(ndef.op()); + if (function_def != nullptr) { + op_def = &(function_def->signature()); + } + if (op_def == nullptr) { + status = OpDefForOp(ndef.op().c_str(), &op_def); + if (!status.ok()) return status; + } + DataTypeVector input_dtypes; + status = InOutTypesForNode(ndef, *op_def, &input_dtypes, + kernel->mutable_output_dtypes()); + if (!status.ok()) return status; + ctx->AddKernelToCache(cache_key, kernel); + } + const DataTypeVector& output_dtypes = kernel->output_dtypes(); + const int output_dtypes_size = static_cast(output_dtypes.size()); + if (output_dtypes_size > *num_retvals) { + return errors::InvalidArgument("Expecting ", output_dtypes.size(), + " outputs, but *num_retvals is ", + *num_retvals); + } + *num_retvals = output_dtypes_size; + if (device == nullptr) { + // TODO(apassos) debug how the assignment below might return a different + // device from the one requested above. + device = kernel->device(); + } + status = ValidateInputTypeAndPlacement( + ctx, device, op, kernel->kernel(), + ctx->ShouldStoreMetadata() ? ctx->RunMetadataProto() : nullptr); + if (!status.ok()) return status; + std::unique_ptr maybe_stats; + if (ctx->ShouldStoreMetadata()) { + maybe_stats.reset(new NodeExecStats); + maybe_stats->set_node_name(op->Name()); + maybe_stats->set_all_start_micros(Env::Default()->NowMicros()); + maybe_stats->set_op_start_rel_micros(0); + maybe_stats->set_scheduled_micros(Env::Default()->NowMicros()); + // TODO(apassos) track referenced tensors + } + retvals->resize(*num_retvals); + if (ctx->Async()) { + // Note that for async mode, execution order will make sure that all + // input handles are ready before executing them. + // TODO(agarwal): Consider executing "cheap" kernels inline for performance. + tensorflow::uint64 id = ctx->NextId(); + for (int i = 0; i < *num_retvals; ++i) { + (*retvals)[i] = new TensorHandle(id, output_dtypes[i], ctx); + } + EagerNode* node = + new ExecuteNode(id, ctx, op->Device(), op->Inputs(), kernel, + maybe_stats.release(), output_dtypes, *retvals); + ctx->ExecutorAdd(node); + } else { + // Execute checks if retvals[i] is nullptr or not to figure if it needs to + // allocate it. + status = EagerExecute(ctx, op->Device(), op->Inputs(), kernel, + maybe_stats.get(), retvals->data(), *num_retvals); + } + + return status; +} + Status EagerExecute(EagerContext* ctx, Device* device, const gtl::InlinedVector& op_inputs, KernelAndDevice* kernel, NodeExecStats* maybe_stats, diff --git a/tensorflow/core/common_runtime/eager/execute.h b/tensorflow/core/common_runtime/eager/execute.h index 0f6ad031e1..7c8d7e164d 100644 --- a/tensorflow/core/common_runtime/eager/execute.h +++ b/tensorflow/core/common_runtime/eager/execute.h @@ -17,6 +17,7 @@ limitations under the License. #include "tensorflow/core/common_runtime/device.h" #include "tensorflow/core/common_runtime/eager/context.h" +#include "tensorflow/core/common_runtime/eager/eager_operation.h" #include "tensorflow/core/common_runtime/eager/kernel_and_device.h" #include "tensorflow/core/common_runtime/eager/tensor_handle.h" #include "tensorflow/core/framework/step_stats.pb.h" @@ -25,6 +26,12 @@ limitations under the License. namespace tensorflow { +// Utility function that executes a fully constructed EagerOperation. +Status EagerExecute( + EagerOperation* op, + tensorflow::gtl::InlinedVector* retvals, + int* num_retvals); + // Low-level utility to execute the kernel specified by kernel on device device, // with the inputs op_inputs, in the context ctx. Status EagerExecute(EagerContext* ctx, Device* device, -- GitLab From 60a0e2f5261cf72da4e4d8e65b56b695d611b984 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Apr 2018 15:19:59 -0700 Subject: [PATCH 764/791] Do not force default layout when there is no need to. Allow the inner computations to negotiate a root and parameter layouts different from default. PiperOrigin-RevId: 193731341 --- tensorflow/compiler/xla/service/BUILD | 3 + .../xla/service/computation_layout.cc | 7 +- .../compiler/xla/service/computation_layout.h | 5 +- .../compiler/xla/service/hlo_instruction.h | 8 + .../compiler/xla/service/layout_assignment.cc | 328 +++++++++++++----- .../compiler/xla/service/layout_assignment.h | 65 +++- tensorflow/compiler/xla/service/service.cc | 5 +- .../compiler/xla/service/tuple_simplifier.cc | 25 +- 8 files changed, 325 insertions(+), 121 deletions(-) diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index 9555d91817..bc577c173d 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -1953,10 +1953,12 @@ cc_library( deps = [ ":computation_layout", ":hlo", + ":hlo_dce", ":hlo_graph_dumper", ":hlo_pass", ":logical_buffer", ":tuple_points_to_analysis", + ":tuple_simplifier", "//tensorflow/compiler/xla:shape_layout", "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:status_macros", @@ -2433,6 +2435,7 @@ cc_library( "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:status_macros", "//tensorflow/compiler/xla:util", + "//tensorflow/compiler/xla:xla_data_proto", "//tensorflow/core:lib", ], ) diff --git a/tensorflow/compiler/xla/service/computation_layout.cc b/tensorflow/compiler/xla/service/computation_layout.cc index d2d4f14fce..cb61f3da39 100644 --- a/tensorflow/compiler/xla/service/computation_layout.cc +++ b/tensorflow/compiler/xla/service/computation_layout.cc @@ -23,12 +23,15 @@ limitations under the License. namespace xla { -ComputationLayout::ComputationLayout(const ProgramShape& program_shape) +ComputationLayout::ComputationLayout(const ProgramShape& program_shape, + bool ignore_layouts) : result_layout_(program_shape.result()) { for (auto& shape : program_shape.parameters()) { parameter_layouts_.emplace_back(shape); } - SetToDefaultLayout(); + if (ignore_layouts) { + SetToDefaultLayout(); + } } void ComputationLayout::SetToDefaultLayout() { diff --git a/tensorflow/compiler/xla/service/computation_layout.h b/tensorflow/compiler/xla/service/computation_layout.h index 80e102411c..53c3a3f7b7 100644 --- a/tensorflow/compiler/xla/service/computation_layout.h +++ b/tensorflow/compiler/xla/service/computation_layout.h @@ -34,8 +34,9 @@ class ComputationLayout { public: // Constructs a ComputationLayout from a ProgramShape. The layouts of the // parameters and results are set to the default layout. Layouts in the - // ProgramShape are ignored. - explicit ComputationLayout(const ProgramShape& program_shape); + // ProgramShape are ignored if ignore_layouts is true. + explicit ComputationLayout(const ProgramShape& program_shape, + bool ignore_layouts = true); // Returns the layout of a particular parameter. const ShapeLayout& parameter_layout(int64 param_no) const { diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h index a5e9aecb9e..f3da3fc256 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.h +++ b/tensorflow/compiler/xla/service/hlo_instruction.h @@ -956,6 +956,14 @@ class HloInstruction { void clear_sharding() { sharding_ = nullptr; } // Return true if this operator has a sharding assigned. bool has_sharding() const { return sharding_ != nullptr; } + // Checks whether the instruction has compatible sharding with the other + // instruction. + bool has_compatible_sharding(const HloInstruction* other) const { + if (!has_sharding()) { + return !other->has_sharding(); + } + return other->has_sharding() ? sharding() == other->sharding() : false; + } // When creating a new instruction which either replaces, or shifts up (kCopy // insertion case), another instruction, we need to make sure the certain diff --git a/tensorflow/compiler/xla/service/layout_assignment.cc b/tensorflow/compiler/xla/service/layout_assignment.cc index 2494569db5..7067b6f86a 100644 --- a/tensorflow/compiler/xla/service/layout_assignment.cc +++ b/tensorflow/compiler/xla/service/layout_assignment.cc @@ -31,10 +31,12 @@ limitations under the License. #include "tensorflow/compiler/xla/ptr_util.h" #include "tensorflow/compiler/xla/service/computation_layout.h" #include "tensorflow/compiler/xla/service/hlo_computation.h" +#include "tensorflow/compiler/xla/service/hlo_dce.h" #include "tensorflow/compiler/xla/service/hlo_graph_dumper.h" #include "tensorflow/compiler/xla/service/hlo_instruction.h" #include "tensorflow/compiler/xla/service/hlo_opcode.h" #include "tensorflow/compiler/xla/service/logical_buffer.h" +#include "tensorflow/compiler/xla/service/tuple_simplifier.h" #include "tensorflow/compiler/xla/shape_layout.h" #include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/status_macros.h" @@ -400,9 +402,9 @@ string LayoutConstraints::ToString() const { } Status LayoutAssignment::AddMandatoryConstraints( - const ComputationLayout& computation_layout, - const ChannelLayoutConstraints* channel_constraints, - HloComputation* computation, LayoutConstraints* constraints) { + const ComputationLayout* computation_layout, + ChannelLayoutConstraints* channel_constraints, HloComputation* computation, + LayoutConstraints* constraints) { VLOG(3) << "Adding mandatory layout constraints to computation " << computation->name(); @@ -424,11 +426,16 @@ Status LayoutAssignment::AddMandatoryConstraints( TF_RETURN_IF_ERROR(constraints->SetOperandLayout( instruction->outfeed_shape(), instruction, 0)); } else if (instruction->opcode() == HloOpcode::kParameter) { - // Parameter layouts must match the respective layout in - // ComputationLayout. - shape_with_layout = - &computation_layout.parameter_layout(instruction->parameter_number()) - .shape(); + if (computation_layout != nullptr) { + const ShapeLayout& parameter_layout = + computation_layout->parameter_layout( + instruction->parameter_number()); + if (parameter_layout.LayoutIsSet()) { + // Parameter layouts must match the respective layout in + // ComputationLayout, if there is one. + shape_with_layout = ¶meter_layout.shape(); + } + } } if (shape_with_layout != nullptr) { TF_RETURN_IF_ERROR( @@ -493,9 +500,8 @@ Status LayoutAssignment::AddMandatoryConstraints( HloComputation* body = instruction->while_body(); HloComputation* condition = instruction->while_condition(); const HloInstruction* init = instruction->operand(0); - const ComputationLayout& body_layout = - FindOrDie(computation_layouts_, body); - const ComputationLayout& condition_layout = + ComputationLayout& body_layout = FindOrDie(computation_layouts_, body); + ComputationLayout& condition_layout = FindOrDie(computation_layouts_, condition); // Check a few invariants irrespective of layout. @@ -508,26 +514,19 @@ Status LayoutAssignment::AddMandatoryConstraints( condition_layout.parameter_shape(0))); DCHECK(ShapeUtil::Compatible(body_layout.result_shape(), init->shape())); - // Return error if earlier layout assignment of the embedded computations - // has produced conflicting layouts. - if (!ShapeUtil::Equal(body_layout.result_shape(), - body_layout.parameter_shape(0))) { - return InternalError( - "Parameter and result of body computation %s of while instruction " - "%s have different layouts: %s vs %s", - body->name().c_str(), instruction->name().c_str(), - ShapeUtil::HumanString(body_layout.result_shape()).c_str(), - ShapeUtil::HumanString(body_layout.parameter_shape(0)).c_str()); + if (body_layout.result_layout() != body_layout.parameter_layout(0)) { + VLOG(2) << "Reset %while body parameter layout: body=" << body->name() + << " while=" << instruction->name() + << " shape=" << body_layout.result_layout().ToString(); + *body_layout.mutable_parameter_layout(0) = body_layout.result_layout(); } - if (!ShapeUtil::Equal(body->root_instruction()->shape(), - condition->parameter_instruction(0)->shape())) { - return InternalError( - "Parameter of condition computation %s of while instruction " - "%s does not match body computation %s result: %s vs %s", - condition->name().c_str(), instruction->name().c_str(), - body->name().c_str(), - ShapeUtil::HumanString(condition_layout.parameter_shape(0)).c_str(), - ShapeUtil::HumanString(body_layout.result_shape()).c_str()); + if (condition_layout.parameter_layout(0) != + body_layout.parameter_layout(0)) { + VLOG(2) << "Reset %while condition parameter layout: cond=" + << condition->name() << " while=" << instruction->name() + << " shape=" << body_layout.parameter_layout(0).ToString(); + *condition_layout.mutable_parameter_layout(0) = + body_layout.parameter_layout(0); } // Constrain the output and the operand of the while instruction to match @@ -557,7 +556,20 @@ Status LayoutAssignment::AddMandatoryConstraints( true_computation_layout.parameter_shape(0))); DCHECK(ShapeUtil::Compatible( false_operand->shape(), false_computation_layout.parameter_shape(0))); - + if (true_computation_layout.result_layout() != + false_computation_layout.result_layout()) { + // We assign layouts in DFS fashion, so the true and false computations + // might have negotiated a different layout. But for the conditional + // instruction POV the layout must match, so we run again on the false + // computation, this time with proper computation layout. + VLOG(2) << "Reset %conditional false computation result layout: " + "false_computation=" + << false_computation->name() + << " conditional=" << instruction->name() << " shape=" + << true_computation_layout.result_layout().ToString(); + *false_computation_layout.mutable_result_layout() = + true_computation_layout.result_layout(); + } TF_RETURN_IF_ERROR(constraints->SetInstructionLayout( true_computation_layout.result_shape(), instruction)); TF_RETURN_IF_ERROR(constraints->SetOperandLayout( @@ -593,10 +605,14 @@ Status LayoutAssignment::AddMandatoryConstraints( } } } - - // Finally set the result layout to match ComputationLayout. - return constraints->SetResultLayout( - computation_layout.result_layout().shape()); + // Finally set the result layout to match ComputationLayout, if there is one. + if (computation_layout != nullptr) { + const ShapeLayout& result_layout = computation_layout->result_layout(); + if (result_layout.LayoutIsSet()) { + TF_RETURN_IF_ERROR(constraints->SetResultLayout(result_layout.shape())); + } + } + return Status::OK(); } namespace { @@ -760,6 +776,7 @@ StatusOr LayoutAssignment::CreateCopyWithNewLayout( HloInstruction* copy = instruction->parent()->AddInstruction(HloInstruction::CreateUnary( instruction->shape(), HloOpcode::kCopy, instruction)); + RegisterAddedCopy(copy); SetupCopiedInstruction(*instruction, copy, {}); LayoutUtil::ClearLayout(copy->mutable_shape()); TF_RETURN_IF_ERROR(LayoutUtil::CopyLayoutBetweenShapes( @@ -783,13 +800,19 @@ Status LayoutAssignment::CopyOperandIfLayoutsDiffer( TF_RET_CHECK(LayoutUtil::HasLayout(operand->shape())); if (ShapeUtil::Equal(operand_layout.shape(), operand->shape())) { + VLOG(5) << "Operand " << operand->ToString() << " layout matches in " + << instruction->ToString(); // Operand layout already matches our constraint. Nothing to do. return Status::OK(); } + VLOG(4) << "Operand " << operand->ToString() << " layout does not match " + << operand_layout.ToString() << " in " << instruction->ToString(); TF_ASSIGN_OR_RETURN(HloInstruction * operand_copy, CreateCopyWithNewLayout(operand_layout.shape(), operand)); + VLOG(4) << "New copy of " << operand->ToString() << " is " + << operand_copy->ToString(); return instruction->ReplaceOperandWith(operand_no, operand_copy); } @@ -896,15 +919,16 @@ Status LayoutAssignment::CheckLayouts(HloModule* module) { } } } - - // Finally verify the result layout matches the layout of the entry + // Finally verify the result layout, if set, matches the layout of the entry // computation root. - TF_RET_CHECK(ShapeUtil::Equal( - module->entry_computation()->root_instruction()->shape(), + const ShapeLayout& result_layout = FindOrDie(computation_layouts_, module->entry_computation()) - .result_layout() - .shape())); - + .result_layout(); + if (result_layout.LayoutIsSet()) { + TF_RET_CHECK(ShapeUtil::Equal( + module->entry_computation()->root_instruction()->shape(), + result_layout.shape())); + } return Status::OK(); } @@ -913,18 +937,13 @@ LayoutAssignment::LayoutAssignment( ChannelLayoutConstraints* channel_constraints) : entry_computation_layout_(entry_computation_layout), channel_layout_constraints_(channel_constraints) { - VLOG(1) << "entry computation layout given to layout assignment: " + VLOG(1) << "Entry computation layout given to layout assignment: " << entry_computation_layout_->ToString(); // Layouts of all parameter instructions must be set. for (const ShapeLayout& parameter_layout : entry_computation_layout_->parameter_layouts()) { CHECK(parameter_layout.LayoutIsSet()); } - // If the result layout is not set, then choose the default. - // TODO(b/29118294): Choose a better layout in this case. - if (!entry_computation_layout_->result_layout().LayoutIsSet()) { - entry_computation_layout_->mutable_result_layout()->SetToDefaultLayout(); - } } std::unique_ptr LayoutAssignment::ChooseOperandLayoutFromOutputLayout( @@ -1484,16 +1503,60 @@ Status LayoutAssignment::AssignLayouts(const LayoutConstraints& constraints, return Status::OK(); } +Status LayoutAssignment::CalculateComputationLayout( + HloComputation* computation) { + ComputationLayout computation_layout(computation->ComputeProgramShape(), + /*ignore_layouts=*/false); + InsertOrDie(&computation_layouts_, computation, computation_layout); + VLOG(2) << " Calculated ComputationLayout = " + << computation_layout.ToString(); + return Status::OK(); +} + +Status LayoutAssignment::ClearComputationLayouts(HloComputation* computation) { + // Clear existing layouts of the instructions. All layouts must be assigned + // by the LayoutAssignment pass, except for those on infeeds, parameters, + // and the computation result. The latter two are specified in + // computation_layout, so we only need to keep the existing layouts for + // infeeds. Clearing the layouts here avoids hiding potential bugs in the + // layout assignment pass that may accidently use the existing layout. + for (HloInstruction* instruction : computation->instructions()) { + if (instruction->opcode() == HloOpcode::kBitcast) { + // bitcasts are inherently layout sensitive and so a bitcast instruction + // present in the IR before layout assignment is a bug. + return InternalError( + "Unexpected bitcast operation seen during layout assignment: %s.", + instruction->ToString().c_str()); + } + if (instruction->opcode() != HloOpcode::kInfeed) { + LayoutUtil::ClearLayout(instruction->mutable_shape()); + } + } + return Status::OK(); +} + Status LayoutAssignment::RunOnComputation( - const ComputationLayout& computation_layout, + ComputationLayout* computation_layout, const TuplePointsToAnalysis& points_to_analysis, HloComputation* computation, ChannelLayoutConstraints* channel_constraints) { - DCHECK(computation_layout.LayoutIsSet()); - InsertOrDie(&computation_layouts_, computation, computation_layout); VLOG(2) << "LayoutAssignment::RunOnComputation(" << computation->name() << ")"; - VLOG(2) << " ComputationLayout = " << computation_layout.ToString(); + TF_RETURN_IF_ERROR(ClearComputationLayouts(computation)); + if (computation_layout != nullptr) { + auto it = computation_layouts_.find(computation); + if (it == computation_layouts_.end()) { + VLOG(2) << " New ComputationLayout = " << computation_layout->ToString(); + computation_layouts_.emplace(computation, *computation_layout); + } else { + TF_RET_CHECK(computation_layout == &it->second || + computation_layout == entry_computation_layout_); + VLOG(2) << " Existing ComputationLayout = " + << computation_layout->ToString(); + } + } else { + VLOG(2) << " No ComputationLayout specified (will be calculated)"; + } // Construct LayoutConstraints with all layout constraints of the computation. LayoutConstraints constraints(points_to_analysis, computation); @@ -1536,12 +1599,19 @@ Status LayoutAssignment::RunOnComputation( CHECK_LT(constraints.unconstrained_buffer_ids().size(), unconstrained_count); } - // All logical buffers should have constraints at this point. All that // remains is assign the constraints to the buffers and infer layouts for // aliased buffers. TF_RETURN_IF_ERROR(AssignLayouts(constraints, computation)); + // If the computation layout wasn't specified, now it is the time to compute + // it according to the parameters and root instruction layouts. + // This allows the first pass through this API to record the best flowing + // layout to parameters and root instruction. + if (computation_layout == nullptr) { + TF_RETURN_IF_ERROR(CalculateComputationLayout(computation)); + } + // Record the layouts assigned for any communication ops in // channel_constraints so that they are constrained for future modules. for (HloInstruction* instruction : computation->instructions()) { @@ -1556,6 +1626,34 @@ Status LayoutAssignment::RunOnComputation( return Status::OK(); } +Status LayoutAssignment::PropagateComputationLayouts( + HloComputation* computation, ComputationLayout* computation_layout) { + ComputationLayout computed_computation_layout( + computation->ComputeProgramShape(), + /*ignore_layouts=*/false); + for (int64 i = 0; i < computed_computation_layout.parameter_count(); ++i) { + ShapeLayout* param_layout = computation_layout->mutable_parameter_layout(i); + if (!param_layout->LayoutIsSet()) { + VLOG(4) << "Assigning layout to parameter " << i << " of computation " + << computation->name() << ": " + << computed_computation_layout.parameter_layout(i).ToString(); + *param_layout = computed_computation_layout.parameter_layout(i); + } else { + TF_RET_CHECK(computed_computation_layout.parameter_layout(i) == + *param_layout); + } + } + ShapeLayout* result_layout = computation_layout->mutable_result_layout(); + if (!result_layout->LayoutIsSet()) { + VLOG(4) << "Assigning result layout of computation " << computation->name() + << ": " << computed_computation_layout.result_layout().ToString(); + *result_layout = computed_computation_layout.result_layout(); + } else { + TF_RET_CHECK(computed_computation_layout.result_layout() == *result_layout); + } + return Status::OK(); +} + StatusOr LayoutAssignment::Run(HloModule* module) { VLOG(2) << "Running layout assignment on module " << module->name(); XLA_VLOG_LINES(3, module->ToString()); @@ -1564,52 +1662,45 @@ StatusOr LayoutAssignment::Run(HloModule* module) { "before layout assignment", module->config().debug_options()); } - - TF_ASSIGN_OR_RETURN(auto points_to_analysis, - TuplePointsToAnalysis::Run(module)); - - // Assign layouts to computations in an order such that a callee computation - // is handled before its caller computation. This ensures that the layout of - // all callers of a computation will agree. - std::list computation_post_order = - module->MakeComputationPostOrder(); - for (auto* computation : module->MakeComputationPostOrder()) { - if (computation->IsFusionComputation()) { - continue; - } - // Clear existing layouts of the instructions. All layouts must be assigned - // by the LayoutAssignment pass, except for those on infeeds, parameters, - // and the computation result. The latter two are specified in - // computation_layout, so we only need to keep the existing layouts for - // infeeds. Clearing the layouts here avoids hiding potential bugs in the - // layout assignment pass that may accidently use the existing layout. - for (HloInstruction* instruction : computation->instructions()) { - if (instruction->opcode() == HloOpcode::kBitcast) { - // bitcasts are inherently layout sensitive and so a bitcast instruction - // present in the IR before layout assignment is a bug. - return InternalError( - "Unexpected bitcast operation seen during layout assignment: %s.", - instruction->ToString().c_str()); + TF_RETURN_IF_ERROR(Init()); + + // We do two passes. The first one we pass a nullptr ComputationLayout to + // the RunOnComputation() calls (for non entry computations), and we register + // the ComputationLayout which are naturally flowing in DFS fashion to the + // parameters and root instruction. + // Walking in DFS mode though, means that we can end up with incorrect layouts + // when seen from an outer instruction, which has across-computation + // constraints to impose. + // For example, the kWhile instruction needs to enforce the same layouts for + // the parameters and root of the bosy, as well as the condition parameters. + // Similarly, the kConditional instruction needs to enforce the same layouts + // for the root of the true and false computations. + // So in the first pass, while allowing the layouts to flow to parameters and + // root, we also fix up the eventually inconsistent ComputationLayout, which + // will be then made mandatory by the second pass. + for (int64 i = 0; i < 2; ++i) { + TF_RETURN_IF_ERROR(ClearPreviousPassSideEffects(module)); + TF_ASSIGN_OR_RETURN(auto points_to_analysis, + TuplePointsToAnalysis::Run(module)); + for (auto* computation : module->MakeComputationPostOrder()) { + if (computation->IsFusionComputation()) { + continue; } - if (instruction->opcode() != HloOpcode::kInfeed) { - LayoutUtil::ClearLayout(instruction->mutable_shape()); + if (computation == module->entry_computation()) { + TF_RETURN_IF_ERROR(RunOnComputation( + entry_computation_layout_, *points_to_analysis, + module->entry_computation(), channel_layout_constraints_)); + } else { + ComputationLayout* computation_layout = + (i == 0) ? nullptr : &FindOrDie(computation_layouts_, computation); + TF_RETURN_IF_ERROR(RunOnComputation(computation_layout, + *points_to_analysis, computation, + channel_layout_constraints_)); } } - if (computation == module->entry_computation()) { - TF_RETURN_IF_ERROR(RunOnComputation( - *entry_computation_layout_, *points_to_analysis, - module->entry_computation(), channel_layout_constraints_)); - } else { - ComputationLayout computation_layout(computation->ComputeProgramShape()); - // Setting all embedded computations to the default layout is potentially - // suboptimal. - computation_layout.SetToDefaultLayout(); - TF_RETURN_IF_ERROR(RunOnComputation(computation_layout, - *points_to_analysis, computation, - channel_layout_constraints_)); - } } - + TF_RETURN_IF_ERROR(PropagateComputationLayouts(module->entry_computation(), + entry_computation_layout_)); TF_RETURN_IF_ERROR(CheckLayouts(module)); VLOG(3) << "After layout assignment:"; @@ -1619,9 +1710,54 @@ StatusOr LayoutAssignment::Run(HloModule* module) { "after layout assignment", module->config().debug_options()); } - // All layouts are reset then reassigned by this pass. return true; } +Status LayoutAssignment::Init() { + computation_layouts_.clear(); + return Status::OK(); +} + +Status LayoutAssignment::ClearPreviousPassSideEffects(HloModule* module) { + // Clear all the copies which have been added, and all the related + // instructions (like GTE and tuples). + int64 removed_copies = 0; + for (HloComputation* computation : module->computations()) { + for (HloInstruction* instruction : + computation->MakeInstructionPostOrder()) { + if (instruction->opcode() == HloOpcode::kCopy && + added_copies_.count(instruction) > 0) { + VLOG(5) << "Removing added copy: " << instruction->ToString(); + TF_RETURN_IF_ERROR( + instruction->ReplaceAllUsesWith(instruction->mutable_operand(0))); + TF_RETURN_IF_ERROR(computation->RemoveInstruction(instruction)); + ++removed_copies; + } + } + } + added_copies_.clear(); + if (removed_copies > 0) { + TupleSimplifier tuple_simplifier; + HloDCE dce; + TF_RETURN_IF_ERROR(tuple_simplifier.Run(module).status()); + TF_RETURN_IF_ERROR(dce.Run(module).status()); + } + return Status::OK(); +} + +Status LayoutAssignment::AddCopyForOperand(HloInstruction* instruction, + int64 operand_number) { + HloInstruction* operand = instruction->mutable_operand(operand_number); + if (operand->opcode() != HloOpcode::kCopy || operand->user_count() > 1) { + HloInstruction* copy = + instruction->parent()->AddInstruction(HloInstruction::CreateUnary( + operand->shape(), HloOpcode::kCopy, operand)); + SetupCopiedInstruction(*operand, copy, {}); + LayoutUtil::ClearLayout(copy->mutable_shape()); + TF_RETURN_IF_ERROR(instruction->ReplaceOperandWith(operand_number, copy)); + } + return Status::OK(); +} + } // namespace xla diff --git a/tensorflow/compiler/xla/service/layout_assignment.h b/tensorflow/compiler/xla/service/layout_assignment.h index ae4986d6ad..8b4e07995a 100644 --- a/tensorflow/compiler/xla/service/layout_assignment.h +++ b/tensorflow/compiler/xla/service/layout_assignment.h @@ -39,6 +39,7 @@ limitations under the License. #include "tensorflow/compiler/xla/xla_data.pb.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/gtl/flatmap.h" +#include "tensorflow/core/lib/gtl/flatset.h" #include "tensorflow/core/platform/types.h" namespace xla { @@ -362,12 +363,15 @@ class LayoutAssignment : public HloPassInterface { int64 operand_no); private: + // Initializes the layout assignment object for a new Run() call. + Status Init(); + // Adds constraints which must be satisfied for correctness on all // backends. Called once prior to propagating constraints. - Status AddMandatoryConstraints( - const ComputationLayout& computation_layout, - const ChannelLayoutConstraints* channel_constraints, - HloComputation* computation, LayoutConstraints* constraints); + Status AddMandatoryConstraints(const ComputationLayout* computation_layout, + ChannelLayoutConstraints* channel_constraints, + HloComputation* computation, + LayoutConstraints* constraints); // This method can be overridden to add backend-specific constraints to the // layout of the instructions of a computation. This method is called after @@ -378,10 +382,12 @@ class LayoutAssignment : public HloPassInterface { } // Construct contraints and assign layouts to all instructions in the - // computation satisfying the given ComputationLayout. Layouts constraints are - // added, then propagated until all LogicalBuffers in the computation are - // constrained. - Status RunOnComputation(const ComputationLayout& computation_layout, + // computation satisfying the given ComputationLayout, if not nullptr. + // Otherwise the ComputationLayout will be calculated by propagating the + // computation instruction contraints. + // Layouts constraints are added, then propagated until all LogicalBuffers in + // the computation are constrained. + Status RunOnComputation(ComputationLayout* computation_layout, const TuplePointsToAnalysis& points_to_analysis, HloComputation* computation, ChannelLayoutConstraints* channel_constraints); @@ -402,6 +408,25 @@ class LayoutAssignment : public HloPassInterface { // necessary conditions. Status CheckLayouts(HloModule* module); + // Computes the ComputationLayout of the given computation based of the + // layouts assigned to parameters and root instruction, and inserts it to the + // computation_layouts_ map. + Status CalculateComputationLayout(HloComputation* computation); + + // Clears all the layouts which can be cleared within a computation. + Status ClearComputationLayouts(HloComputation* computation); + + // Clears the side effects of a previous pass, like added copy instructions. + Status ClearPreviousPassSideEffects(HloModule* module); + + // Propagates the layouts computed by the layout assignment pass on the given + // computation, to the computation layout passed in to this API. + // This API propagates missing layout, and also checks that the caller + // specified have been respected, by comparing those with the parameters and + // root computation instruction. + Status PropagateComputationLayouts(HloComputation* computation, + ComputationLayout* computation_layout); + ComputationLayout* entry_computation_layout_; protected: @@ -418,21 +443,37 @@ class LayoutAssignment : public HloPassInterface { // Creates and returns a copy of the given instruction with a different // layout. Tuple-shaped instructions will be deep-copied, and the last Tuple // instruction producing the copy is returned. - static StatusOr CreateCopyWithNewLayout( + StatusOr CreateCopyWithNewLayout( const Shape& shape_with_layout, HloInstruction* instruction); // Creates a copy of the given operand if the operand's layout does not match // the given layout. This copy replaces the use in the given instruction. // Tuple operands will be deep-copied. - static Status CopyOperandIfLayoutsDiffer(const ShapeLayout& operand_layout, - HloInstruction* instruction, - int64 operand_no); + Status CopyOperandIfLayoutsDiffer(const ShapeLayout& operand_layout, + HloInstruction* instruction, + int64 operand_no); + + // Registers a copy instruction added by the layout assignment pass. + void RegisterAddedCopy(HloInstruction* copy) { + CHECK_EQ(copy->opcode(), HloOpcode::kCopy); + added_copies_.insert(copy); + } + + // Adds a copy for the operand of an instruction, unless such operand is + // already a copy, and has a single user (which is forcibly the instruction + // itself). + Status AddCopyForOperand(HloInstruction* instruction, int64 operand_number); // Map containing the layouts of all computations assigned so // far. Computations are handled in a topological sort where computations are // handled before their caller instructions so the layouts of caller // instructions can be set to match the computation. std::map computation_layouts_; + + // Every copy added to the module by the layout assignment pass is registered + // here. + tensorflow::gtl::FlatSet added_copies_; + ChannelLayoutConstraints* channel_layout_constraints_; }; diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc index 39f3aefdf8..a73118c68a 100644 --- a/tensorflow/compiler/xla/service/service.cc +++ b/tensorflow/compiler/xla/service/service.cc @@ -308,7 +308,10 @@ StatusOr> Service::CreateModuleConfig( computation_layout->mutable_result_layout()->CopyLayoutFromShape( shape_with_output_layout)); } else { - computation_layout->mutable_result_layout()->Clear(); + // TODO(b/78356948): We are forcing the default layout here. We should fix + // clients which expect a default layout, to be explicit about it, by + // passing the proper ExecutionOptions with shape_with_output_layout set. + computation_layout->mutable_result_layout()->SetToDefaultLayout(); } config->set_replica_count(options_.number_of_replicas()); diff --git a/tensorflow/compiler/xla/service/tuple_simplifier.cc b/tensorflow/compiler/xla/service/tuple_simplifier.cc index 113c2e2bd9..d668855084 100644 --- a/tensorflow/compiler/xla/service/tuple_simplifier.cc +++ b/tensorflow/compiler/xla/service/tuple_simplifier.cc @@ -69,6 +69,7 @@ StatusOr TupleSimplifier::Run(HloModule* module) { // Tuple // HloInstruction* top_tuple = nullptr; + HloInstruction* first_gte = nullptr; bool can_simplify = true; for (int64 operand_number = 0; operand_number < instruction->operand_count(); ++operand_number) { @@ -78,11 +79,17 @@ StatusOr TupleSimplifier::Run(HloModule* module) { can_simplify = false; break; } - + if (first_gte == nullptr) { + first_gte = operand; + } else if (!first_gte->has_compatible_sharding(operand)) { + can_simplify = false; + break; + } if (top_tuple == nullptr) { top_tuple = operand->mutable_operand(0); if (!ShapeUtil::Compatible(top_tuple->shape(), - instruction->shape())) { + instruction->shape()) || + !instruction->has_compatible_sharding(top_tuple)) { can_simplify = false; break; } @@ -108,15 +115,17 @@ StatusOr TupleSimplifier::Run(HloModule* module) { // | // GTE if (instruction->operand(0)->opcode() == HloOpcode::kTuple) { - changed = true; HloInstruction* element_source = instruction->mutable_operand(0)->mutable_operand( instruction->tuple_index()); - TF_RETURN_IF_ERROR(instruction->ReplaceAllUsesWith(element_source)); - for (HloInstruction* user : element_source->users()) { - if (user->opcode() == HloOpcode::kTuple || - user->opcode() == HloOpcode::kGetTupleElement) { - worklist.push(user); + if (instruction->has_compatible_sharding(element_source)) { + changed = true; + TF_RETURN_IF_ERROR(instruction->ReplaceAllUsesWith(element_source)); + for (HloInstruction* user : element_source->users()) { + if (user->opcode() == HloOpcode::kTuple || + user->opcode() == HloOpcode::kGetTupleElement) { + worklist.push(user); + } } } } -- GitLab From 6af31f6260161bab02db83d7e9e1d7ba7fd14b2c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Apr 2018 15:20:37 -0700 Subject: [PATCH 765/791] [XLA] Redesign: add comparator and printer for the XlaOp. This is to prepare the migration of tf2xla. There were some codes used ComputationDataHandle::handle() for comparison/printing. Now implement XlaOp's comparator and printer. PiperOrigin-RevId: 193731437 --- .../compiler/xla/client/xla_client/xla_builder.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.h b/tensorflow/compiler/xla/client/xla_client/xla_builder.h index 5977ee4f4b..4955f1515d 100644 --- a/tensorflow/compiler/xla/client/xla_client/xla_builder.h +++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.h @@ -57,11 +57,27 @@ class XlaOp { StatusOr GetShape() const; + const XlaBuilder* builder() const { return builder_; } + + bool operator==(const XlaOp& rhs) const { + return handle_ == rhs.handle_ && builder_ == rhs.builder_; + } + + bool operator!=(const XlaOp& rhs) const { + return handle_ != rhs.handle_ || builder_ != rhs.builder_; + } + + friend std::ostream& operator<<(std::ostream& out, const XlaOp& op) { + out << op.handle(); + return out; + } + private: XlaOp(int64 handle, XlaBuilder* builder) : handle_(handle), builder_(builder) {} int64 handle() const { return handle_; } + friend class XlaBuilder; int64 handle_; -- GitLab From cadbb0b70b9441388a04533433245ac85f2887a9 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Apr 2018 15:32:32 -0700 Subject: [PATCH 766/791] [XLA] Redesign: implement DumpToDirectory for the HloSession. This is to prepare the migration of tf2xla. PiperOrigin-RevId: 193733029 --- tensorflow/compiler/xla/service/BUILD | 1 + tensorflow/compiler/xla/service/executable.cc | 20 +++++++++++++++++++ tensorflow/compiler/xla/service/executable.h | 5 +++++ 3 files changed, 26 insertions(+) diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index bc577c173d..afb344e5ae 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -755,6 +755,7 @@ cc_library( ":hlo", ":hlo_execution_profile", ":hlo_graph_dumper", + ":hlo_proto", ":pool", ":session_proto", ":shaped_buffer", diff --git a/tensorflow/compiler/xla/service/executable.cc b/tensorflow/compiler/xla/service/executable.cc index b097ef79cc..8218b5f7c8 100644 --- a/tensorflow/compiler/xla/service/executable.cc +++ b/tensorflow/compiler/xla/service/executable.cc @@ -163,4 +163,24 @@ Status Executable::DumpSessionModule() { result); } +/* static */ Status Executable::DumpToDirectory(const string& directory_path, + string filename, + const HloSession& hlo_session) { + tensorflow::Env* env = tensorflow::Env::Default(); + if (!env->IsDirectory(directory_path).ok()) { + // NB! CreateDir does not work reliably with multiple XLA threads -- two + // threads can race to observe the absence of the dump directory and + // simultaneously try to create it, causing the "losing" thread to get a + // "directory already exists" error. + TF_RETURN_IF_ERROR(env->RecursivelyCreateDir(directory_path)); + } + filename = SanitizeFileName(std::move(filename)); + string file_path = tensorflow::io::JoinPath(directory_path, filename); + string result; + TF_RET_CHECK( + tensorflow::SerializeToStringDeterministic(hlo_session, &result)); + return tensorflow::WriteStringToFile(tensorflow::Env::Default(), file_path, + result); +} + } // namespace xla diff --git a/tensorflow/compiler/xla/service/executable.h b/tensorflow/compiler/xla/service/executable.h index 9c725f21d8..bdbe119120 100644 --- a/tensorflow/compiler/xla/service/executable.h +++ b/tensorflow/compiler/xla/service/executable.h @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow/compiler/xla/legacy_flags/debug_options_flags.h" #include "tensorflow/compiler/xla/service/computation_layout.h" #include "tensorflow/compiler/xla/service/device_memory_allocator.h" +#include "tensorflow/compiler/xla/service/hlo.pb.h" #include "tensorflow/compiler/xla/service/hlo_execution_profile.h" #include "tensorflow/compiler/xla/service/hlo_graph_dumper.h" #include "tensorflow/compiler/xla/service/hlo_module.h" @@ -155,6 +156,10 @@ class Executable { static Status DumpToDirectory(const string& directory_path, string filename, const SessionModule& session_module); + // Dump hlo_session to directory_path/filename. + static Status DumpToDirectory(const string& directory_path, string filename, + const HloSession& hlo_session); + protected: mutable tensorflow::mutex mutex_; -- GitLab From b2f786867dca85b6b848f09f2c1d40dd123fc0fc Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Fri, 20 Apr 2018 15:38:06 -0700 Subject: [PATCH 767/791] Always use the local worker name in CreateWorkerSession when not doing ClusterSpec propagation. Previously, the master would send a job name and task index in an otherwise-empty ServerDef, and the worker would unquestioningly use those to build its worker name. However, this would lead to errors if the worker had a local name like "/job:worker/replica:1/task:0", because the ServerDef doesn't support non-zero replica IDs, and so the local worker would end up an inconsistent view of what its worker name should be. In particular `WorkerSession::worker_name` would disagree with the device names added during graph partitioning by the master, which would lead to runtime failures ("InvalidArgumentError: Invalid rendezvous key"). PiperOrigin-RevId: 193733855 --- tensorflow/core/distributed_runtime/BUILD | 1 + .../distributed_runtime/master_session.cc | 28 +++++++++--------- .../core/distributed_runtime/session_mgr.cc | 6 ++-- .../distributed_runtime/session_mgr_test.cc | 29 +++++++++++++++++++ 4 files changed, 49 insertions(+), 15 deletions(-) diff --git a/tensorflow/core/distributed_runtime/BUILD b/tensorflow/core/distributed_runtime/BUILD index d564727da5..343dd5d456 100644 --- a/tensorflow/core/distributed_runtime/BUILD +++ b/tensorflow/core/distributed_runtime/BUILD @@ -145,6 +145,7 @@ tf_cc_test( deps = [ ":session_mgr", ":worker_env", + "//tensorflow/core:protos_all_cc", "//tensorflow/core:test", "//tensorflow/core:test_main", "//tensorflow/core/distributed_runtime/rpc:rpc_rendezvous_mgr", diff --git a/tensorflow/core/distributed_runtime/master_session.cc b/tensorflow/core/distributed_runtime/master_session.cc index ebe350d313..e3022f38a2 100644 --- a/tensorflow/core/distributed_runtime/master_session.cc +++ b/tensorflow/core/distributed_runtime/master_session.cc @@ -1219,17 +1219,6 @@ Status MasterSession::CreateWorkerSessions( workers[i].name = &worker_names[i]; workers[i].worker = worker_cache->CreateWorker(worker_names[i]); workers[i].request.set_session_handle(handle_); - if (options.cluster_def) { - *workers[i].request.mutable_server_def()->mutable_cluster() = - *options.cluster_def; - workers[i].request.mutable_server_def()->set_protocol(*options.protocol); - // Session state is always isolated when ClusterSpec propagation - // is in use. - workers[i].request.set_isolate_session_state(true); - } else { - workers[i].request.set_isolate_session_state( - session_opts_.config.isolate_session_state()); - } DeviceNameUtils::ParsedName name; if (!DeviceNameUtils::ParseFullName(worker_names[i], &name)) { @@ -1243,8 +1232,21 @@ Status MasterSession::CreateWorkerSessions( return status; } - workers[i].request.mutable_server_def()->set_job_name(name.job); - workers[i].request.mutable_server_def()->set_task_index(name.task); + if (options.cluster_def) { + *workers[i].request.mutable_server_def()->mutable_cluster() = + *options.cluster_def; + workers[i].request.mutable_server_def()->set_protocol(*options.protocol); + workers[i].request.mutable_server_def()->set_job_name(name.job); + workers[i].request.mutable_server_def()->set_task_index(name.task); + // Session state is always isolated when ClusterSpec propagation + // is in use. + workers[i].request.set_isolate_session_state(true); + } else { + // NOTE(mrry): Do not set any component of the ServerDef, + // because the worker will use its local configuration. + workers[i].request.set_isolate_session_state( + session_opts_.config.isolate_session_state()); + } } for (size_t i = 0; i < worker_names.size(); ++i) { diff --git a/tensorflow/core/distributed_runtime/session_mgr.cc b/tensorflow/core/distributed_runtime/session_mgr.cc index 357e9f8930..7ef4206c78 100644 --- a/tensorflow/core/distributed_runtime/session_mgr.cc +++ b/tensorflow/core/distributed_runtime/session_mgr.cc @@ -43,6 +43,7 @@ SessionMgr::SessionMgr( new GraphMgr(worker_env, worker_env->device_mgr)))), worker_cache_factory_(std::move(worker_cache_factory)) {} +/* static */ string SessionMgr::WorkerNameFromServerDef(const ServerDef& server_def) { return strings::StrCat("/job:", server_def.job_name(), "/replica:0/task:", server_def.task_index()); @@ -56,13 +57,14 @@ Status SessionMgr::CreateSession(const string& session, return errors::InvalidArgument("Session must be non-empty."); } - const string worker_name = WorkerNameFromServerDef(server_def); - WorkerCacheInterface* worker_cache = nullptr; + string worker_name; if (server_def.cluster().job().empty()) { worker_cache = new WorkerCacheWrapper(default_worker_cache_.get()); + worker_name = legacy_session_->worker_name; } else { TF_RETURN_IF_ERROR(worker_cache_factory_(server_def, &worker_cache)); + worker_name = WorkerNameFromServerDef(server_def); } if (worker_cache != nullptr & default_worker_cache_.get() != nullptr) { diff --git a/tensorflow/core/distributed_runtime/session_mgr_test.cc b/tensorflow/core/distributed_runtime/session_mgr_test.cc index 0da333833a..99192119a6 100644 --- a/tensorflow/core/distributed_runtime/session_mgr_test.cc +++ b/tensorflow/core/distributed_runtime/session_mgr_test.cc @@ -19,6 +19,7 @@ limitations under the License. #include "tensorflow/core/distributed_runtime/worker_env.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/platform/test.h" +#include "tensorflow/core/protobuf/cluster.pb.h" namespace tensorflow { @@ -77,6 +78,34 @@ TEST_F(SessionMgrTest, CreateSessionSimple) { TF_EXPECT_OK(mgr_.DeleteSession(session_handle)); } +TEST_F(SessionMgrTest, CreateSessionClusterDefWorkerName) { + ServerDef server_def; + server_def.set_job_name("worker"); + server_def.set_task_index(3); + auto job = server_def.mutable_cluster()->add_job(); + job->set_name("worker"); + job->mutable_tasks()->insert({3, "localhost:3333"}); + + string session_handle = "test_session_handle"; + TF_EXPECT_OK(mgr_.CreateSession(session_handle, server_def, true)); + std::shared_ptr session; + TF_EXPECT_OK(mgr_.WorkerSessionForSession(session_handle, &session)); + EXPECT_NE(nullptr, session) << "Session for " << session_handle << "was null"; + EXPECT_EQ("/job:worker/replica:0/task:3", session->worker_name); + TF_EXPECT_OK(mgr_.DeleteSession(session_handle)); +} + +TEST_F(SessionMgrTest, CreateSessionDefaultWorkerName) { + ServerDef server_def; + string session_handle = "test_session_handle"; + TF_EXPECT_OK(mgr_.CreateSession(session_handle, server_def, true)); + std::shared_ptr session; + TF_EXPECT_OK(mgr_.WorkerSessionForSession(session_handle, &session)); + EXPECT_NE(nullptr, session) << "Session for " << session_handle << "was null"; + EXPECT_EQ("/job:mnist/replica:0/task:0", session->worker_name); + TF_EXPECT_OK(mgr_.DeleteSession(session_handle)); +} + TEST_F(SessionMgrTest, CreateSessionIsolateSessionState) { ServerDef server_def; server_def.set_job_name("worker"); -- GitLab From c015a45646029f8c116028505f2da9e023b5c2b7 Mon Sep 17 00:00:00 2001 From: Brennan Saeta Date: Fri, 20 Apr 2018 15:51:16 -0700 Subject: [PATCH 768/791] Support legacy clusters PiperOrigin-RevId: 193735742 --- .../cluster_resolver/python/training/tpu_cluster_resolver.py | 2 +- .../python/training/tpu_cluster_resolver_test.py | 3 +-- tensorflow/contrib/tpu/python/tpu/tpu_config.py | 5 +++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py index 5a2771229d..1403483d28 100644 --- a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py +++ b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py @@ -245,7 +245,7 @@ class TPUClusterResolver(ClusterResolver): else: if not self._tpu.startswith(compat.as_bytes('grpc://')): # Case 3. - return server_lib.ClusterSpec({}) + return None # Case 2. cluster_spec = {self._job_name: [self._tpu[len( compat.as_bytes('grpc://')):]]} diff --git a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py index dff7a03b68..5b3f9be5a1 100644 --- a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py +++ b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py @@ -356,8 +356,7 @@ class TPUClusterResolverTest(test.TestCase): tpu_cluster_resolver = TPUClusterResolver(tpu='/bns/foo/bar') self.assertEqual( compat.as_bytes('/bns/foo/bar'), tpu_cluster_resolver.master()) - self.assertEqual( - server_lib.ClusterSpec({}), tpu_cluster_resolver.cluster_spec()) + self.assertEqual(None, tpu_cluster_resolver.cluster_spec()) def testGkeEnvironment(self): os.environ['KUBE_GOOGLE_CLOUD_TPU_ENDPOINTS'] = 'grpc://10.120.27.5:8470' diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_config.py b/tensorflow/contrib/tpu/python/tpu/tpu_config.py index cc1a7fd801..6d7331e3c7 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_config.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_config.py @@ -210,8 +210,9 @@ class RunConfig(run_config_lib.RunConfig): raise ValueError( 'You cannot provide a ClusterResolver and ' 'session_config.cluster_def.') - self._session_config.cluster_def.CopyFrom( - self._cluster_spec.as_cluster_def()) + if self._cluster_spec: + self._session_config.cluster_def.CopyFrom( + self._cluster_spec.as_cluster_def()) @property def evaluation_master(self): -- GitLab From a0071844d0af47f22ab512363b56383acf762dff Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Apr 2018 16:05:47 -0700 Subject: [PATCH 769/791] Remove protected data members from GraphOptimizerStage. PiperOrigin-RevId: 193737654 --- .../optimizers/arithmetic_optimizer.cc | 54 +++++++++---------- .../optimizers/graph_optimizer_stage.h | 5 +- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc index 232132e1e8..ed199c1ac8 100644 --- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc @@ -294,8 +294,8 @@ class ArithmeticOptimizerStage : public GraphOptimizerStage { for (int i = src->input_size() - 1; i >= 0; --i) { if (IsControlInput(src->input(i))) { *target_node->add_input() = src->input(i); - ctx_.node_map->AddOutput(NodeName(src->input(i)), - target_node->name()); + ctx().node_map->AddOutput(NodeName(src->input(i)), + target_node->name()); } else { break; } @@ -442,7 +442,7 @@ class ArithmeticNodesGroupOptimizerStage : public ArithmeticOptimizerStage { // TODO(ezhulenev): move to GraphOptimizerStage? bool DrivesControlDependency(const NodeDef& node) const { int position; - for (const NodeDef* output : ctx_.node_map->GetOutputs(node.name())) { + for (const NodeDef* output : ctx().node_map->GetOutputs(node.name())) { for (int i = 0; i < output->input_size(); ++i) { auto input = output->input(i); string name = ParseNodeName(input, &position); @@ -476,8 +476,8 @@ class ArithmeticNodesGroupOptimizerStage : public ArithmeticOptimizerStage { } bool IsInPreserveSet(const NodeDef& node) const { - return ctx_.nodes_to_preserve->find(node.name()) != - ctx_.nodes_to_preserve->end(); + return ctx().nodes_to_preserve->find(node.name()) != + ctx().nodes_to_preserve->end(); } bool IsAlreadyOptimized(const NodeDef& node) const { @@ -546,7 +546,7 @@ class AddOpsRewriteStage : public ArithmeticNodesGroupOptimizerStage { // with a single output data consumer (presumably if we reach this node from // previously absorbed or a root node, it means that this node is not used // as an input to any other op, outside of the group) - if (NumNonControlDataOutputs(node, *ctx_.node_map) != 1) { + if (NumNonControlDataOutputs(node, *ctx().node_map) != 1) { return false; } // All input shapes must be broadcastable to the node shape @@ -685,7 +685,7 @@ class AddOpsRewriteStage : public ArithmeticNodesGroupOptimizerStage { (*node->mutable_attr())["N"].set_i(inputs.size()); for (const auto& inputAndShape : inputs) { - ctx_.node_map->AddOutput(inputAndShape.input, node_name); + ctx().node_map->AddOutput(inputAndShape.input, node_name); node->add_input(inputAndShape.input); } @@ -707,8 +707,8 @@ class AddOpsRewriteStage : public ArithmeticNodesGroupOptimizerStage { node->set_device(root_node.device()); (*node->mutable_attr())["T"].set_type(dtype); - ctx_.node_map->AddOutput(left.input, node_name); - ctx_.node_map->AddOutput(right.input, node_name); + ctx().node_map->AddOutput(left.input, node_name); + ctx().node_map->AddOutput(right.input, node_name); node->add_input(left.input); node->add_input(right.input); @@ -784,20 +784,20 @@ class HoistCommonFactorOutOfAggregation : public ArithmeticOptimizerStage { new_outer_node->set_input(1, new_add_node->name()); } - ctx_.node_map->AddOutput(common_factor, new_outer_node->name()); - ctx_.node_map->AddOutput(new_add_node->name(), new_outer_node->name()); + ctx().node_map->AddOutput(common_factor, new_outer_node->name()); + ctx().node_map->AddOutput(new_add_node->name(), new_outer_node->name()); // Hoist non-shared factors up into the new AddN node. for (int i = 0; i < unique_factors.size(); ++i) { const string& unique_factor_i = unique_factors[i]; new_add_node->set_input(i, unique_factor_i); - ctx_.node_map->AddOutput(unique_factor_i, new_add_node->name()); + ctx().node_map->AddOutput(unique_factor_i, new_add_node->name()); } // Add control deps on add node for (const string& ctrl_dep : ctrl_deps) { *new_add_node->add_input() = ctrl_dep; - ctx_.node_map->AddOutput(NodeName(ctrl_dep), new_add_node->name()); + ctx().node_map->AddOutput(NodeName(ctrl_dep), new_add_node->name()); } // optimize new inner aggregation node @@ -931,8 +931,8 @@ class HoistCommonFactorOutOfAggregation : public ArithmeticOptimizerStage { // if graph rewrite happens in multiple passes without graph pruning between // them, it's possible that rewritten node already exists in a graph return rewritten_nodes_.find(node->name()) != rewritten_nodes_.end() || - ctx_.node_map->NodeExists(OuterNodeName(node, false)) || - ctx_.node_map->NodeExists(OuterNodeName(node, true)); + ctx().node_map->NodeExists(OuterNodeName(node, false)) || + ctx().node_map->NodeExists(OuterNodeName(node, true)); } // keep names of the nodes that were optimized by this stage @@ -996,7 +996,7 @@ class MinimizeBroadcasts : public ArithmeticNodesGroupOptimizerStage { } // Optimized nodes updated in place, and that would break the graph, if the // node has multiple output consumers - if (NumNonControlOutputs(node, *ctx_.node_map) != 1) { + if (NumNonControlOutputs(node, *ctx().node_map) != 1) { return false; } // All input shapes must be broadcastable to the node shape @@ -1120,13 +1120,13 @@ class MinimizeBroadcasts : public ArithmeticNodesGroupOptimizerStage { node->set_input(0, input_0); node->set_input(1, input_1); // Invalidate node properties (shape) - ctx_.graph_properties->ClearOutputProperties(node->name()); - ctx_.graph_properties->ClearInputProperties(node->name()); + ctx().graph_properties->ClearOutputProperties(node->name()); + ctx().graph_properties->ClearInputProperties(node->name()); // Update the node map - ctx_.node_map->RemoveOutput(NodeName(old_input_0), node->name()); - ctx_.node_map->RemoveOutput(NodeName(old_input_1), node->name()); - ctx_.node_map->AddOutput(NodeName(input_0), node->name()); - ctx_.node_map->AddOutput(NodeName(input_1), node->name()); + ctx().node_map->RemoveOutput(NodeName(old_input_0), node->name()); + ctx().node_map->RemoveOutput(NodeName(old_input_1), node->name()); + ctx().node_map->AddOutput(NodeName(input_0), node->name()); + ctx().node_map->AddOutput(NodeName(input_1), node->name()); // Add updated node to optimization queue AddToOptimizationQueue(node); } @@ -1257,8 +1257,8 @@ class RemoveRedundantBitcastStage : public ArithmeticOptimizerStage { // Bitcast(Bitcast(x, type1), type2) => Bitcast(x, type2) bitcast->set_input(0, operand->input(0)); SetSourceDataType(GetSourceDataType(*operand), bitcast); - ctx_.node_map->UpdateInput(bitcast->name(), bitcast->input(0), - operand->input(0)); + ctx().node_map->UpdateInput(bitcast->name(), bitcast->input(0), + operand->input(0)); AddToOptimizationQueue(bitcast); *simplified_node_name = bitcast->name(); } @@ -1313,14 +1313,14 @@ class RemoveNegationStage : public ArithmeticOptimizerStage { node->mutable_input()->SwapElements(0, 1); node->set_input(1, x->input(0)); node->add_input(AsControlDependency(x->name())); - ctx_.node_map->AddOutput(NodeName(x->input(0)), node_name); + ctx().node_map->AddOutput(NodeName(x->input(0)), node_name); updated = true; } else if (IsNeg(*y)) { // a + (-b) = a - b node->set_op("Sub"); node->set_input(1, y->input(0)); node->add_input(AsControlDependency(y->name())); - ctx_.node_map->AddOutput(NodeName(y->input(0)), node_name); + ctx().node_map->AddOutput(NodeName(y->input(0)), node_name); updated = true; } } else if (IsSub(*node)) { @@ -1329,7 +1329,7 @@ class RemoveNegationStage : public ArithmeticOptimizerStage { node->set_op("Add"); node->set_input(1, y->input(0)); node->add_input(AsControlDependency(y->name())); - ctx_.node_map->AddOutput(NodeName(y->input(0)), node_name); + ctx().node_map->AddOutput(NodeName(y->input(0)), node_name); updated = true; } } diff --git a/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h b/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h index ed398525f3..089cad36e9 100644 --- a/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h +++ b/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h @@ -182,7 +182,10 @@ class GraphOptimizerStage { return ::tensorflow::grappler::AddEmptyNode(ctx_, name); } - protected: // Data members + protected: + const GraphOptimizerContext& ctx() const { return ctx_; } + + private: // Data members const string optimizer_name_; const string stage_name_; const GraphOptimizerContext ctx_; -- GitLab From 3fa8795c511931b55a9703956bdf564fde817c2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Branchaud-Charron?= Date: Fri, 20 Apr 2018 19:10:41 -0400 Subject: [PATCH 770/791] Fix casting in Keras estimator (#18104) --- .../python/keras/_impl/keras/estimator.py | 22 +++++++++++++---- .../keras/_impl/keras/estimator_test.py | 24 +++++++++++++++---- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/tensorflow/python/keras/_impl/keras/estimator.py b/tensorflow/python/keras/_impl/keras/estimator.py index b922a6c683..c3c3fceb45 100644 --- a/tensorflow/python/keras/_impl/keras/estimator.py +++ b/tensorflow/python/keras/_impl/keras/estimator.py @@ -29,12 +29,14 @@ from tensorflow.python.estimator import run_config as run_config_lib from tensorflow.python.framework import ops from tensorflow.python.framework import random_seed from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib +from tensorflow.python.framework import tensor_util from tensorflow.python.keras._impl.keras import backend as K from tensorflow.python.keras._impl.keras import models from tensorflow.python.keras._impl.keras import optimizers from tensorflow.python.keras._impl.keras.engine.base_layer import Layer from tensorflow.python.keras._impl.keras.engine.network import Network from tensorflow.python.keras._impl.keras.utils.generic_utils import CustomObjectScope +from tensorflow.python.ops import check_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import metrics as metrics_module from tensorflow.python.ops import variables as variables_module @@ -55,6 +57,17 @@ def _cast_tensor_to_floatx(x): return math_ops.cast(x, K.floatx()) +def _convert_tensor(x): + """Create or cast tensor if needed.""" + if not tensor_util.is_tensor(x): + # x is a numpy array + x = sparse_tensor_lib.convert_to_tensor_or_sparse_tensor(x) + if check_ops.is_numeric_tensor(x): + # is_numeric_tensor returns False if provided with a numpy array + x = _cast_tensor_to_floatx(x) + return x + + def _any_variable_initalized(): """Check if any variable has been initialized in the Keras model. @@ -86,7 +99,7 @@ def _create_ordered_io(keras_model, estimator_io, is_input=True): if isinstance(estimator_io, (list, tuple)): # Case currently not supported by most built-in input_fn, # but it's good to have for sanity - return [_cast_tensor_to_floatx(x) for x in estimator_io] + return [_convert_tensor(x) for x in estimator_io] elif isinstance(estimator_io, dict): if is_input: if keras_model._is_graph_network: @@ -108,12 +121,12 @@ def _create_ordered_io(keras_model, estimator_io, is_input=True): 'It needs to match one ' 'of the following: %s' % ('input' if is_input else 'output', key, ', '.join(keras_io_names))) - tensors = [_cast_tensor_to_floatx(estimator_io[io_name]) + tensors = [_convert_tensor(estimator_io[io_name]) for io_name in keras_io_names] return tensors else: # Plain array. - return _cast_tensor_to_floatx(estimator_io) + return _convert_tensor(estimator_io) def _in_place_subclassed_model_reset(model): @@ -274,8 +287,7 @@ def _clone_and_build_model(mode, is_input=False) else: target_tensors = [ - _cast_tensor_to_floatx( - sparse_tensor_lib.convert_to_tensor_or_sparse_tensor(labels)) + _convert_tensor(labels) ] if keras_model._is_graph_network: diff --git a/tensorflow/python/keras/_impl/keras/estimator_test.py b/tensorflow/python/keras/_impl/keras/estimator_test.py index 653cdc01e2..80fa87d041 100644 --- a/tensorflow/python/keras/_impl/keras/estimator_test.py +++ b/tensorflow/python/keras/_impl/keras/estimator_test.py @@ -30,6 +30,7 @@ from tensorflow.python.estimator.inputs import numpy_io from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.keras._impl import keras +from tensorflow.python.keras._impl.keras import backend as K from tensorflow.python.keras._impl.keras import testing_utils from tensorflow.python.keras._impl.keras.applications import mobilenet from tensorflow.python.keras._impl.keras.optimizers import SGD @@ -142,16 +143,20 @@ def randomize_io_type(array, name): def multi_inputs_multi_outputs_model(): - # test multi-input layer a = keras.layers.Input(shape=(16,), name='input_a') b = keras.layers.Input(shape=(16,), name='input_b') + m = keras.layers.Input(shape=(8,), dtype='bool', name='input_m') dense = keras.layers.Dense(8, name='dense_1') + a_2 = dense(a) + # Apply a mask + s_2 = keras.layers.Lambda(lambda k: + K.switch(k[0], k[1], K.zeros_like(k[1])))([m, a_2]) b_2 = dense(b) - merged = keras.layers.concatenate([a_2, b_2], name='merge') + merged = keras.layers.concatenate([s_2, b_2], name='merge') c = keras.layers.Dense(3, activation='softmax', name='dense_2')(merged) d = keras.layers.Dense(2, activation='softmax', name='dense_3')(merged) - model = keras.models.Model(inputs=[a, b], outputs=[c, d]) + model = keras.models.Model(inputs=[a, b, m], outputs=[c, d]) model.compile( loss='categorical_crossentropy', optimizer='rmsprop', @@ -352,18 +357,27 @@ class TestKerasEstimator(test_util.TensorFlowTestCase): test_samples=50, input_shape=(16,), num_classes=2) + np.random.seed(_RANDOM_SEED) + (input_m_train, _), (input_m_test, _) = testing_utils.get_test_data( + train_samples=_TRAIN_SIZE, + test_samples=50, + input_shape=(8,), + num_classes=2) + c_train = keras.utils.to_categorical(c_train) c_test = keras.utils.to_categorical(c_test) d_train = keras.utils.to_categorical(d_train) d_test = keras.utils.to_categorical(d_test) def train_input_fn(): - input_dict = {'input_a': a_train, 'input_b': b_train} + input_dict = {'input_a': a_train, 'input_b': b_train, + 'input_m': input_m_train > 0} output_dict = {'dense_2': c_train, 'dense_3': d_train} return input_dict, output_dict def eval_input_fn(): - input_dict = {'input_a': a_test, 'input_b': b_test} + input_dict = {'input_a': a_test, 'input_b': b_test, + 'input_m': input_m_test > 0} output_dict = {'dense_2': c_test, 'dense_3': d_test} return input_dict, output_dict -- GitLab From cd095e0c455b3df98841ca70ba24fd41935552e7 Mon Sep 17 00:00:00 2001 From: Asim Shankar Date: Fri, 20 Apr 2018 16:18:29 -0700 Subject: [PATCH 771/791] tf.contrib.data.scan: Support eager execution. PiperOrigin-RevId: 193739234 --- .../contrib/data/python/kernel_tests/BUILD | 1 + .../kernel_tests/scan_dataset_op_test.py | 23 ++++++++++++------- .../contrib/data/python/ops/scan_ops.py | 1 + 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD index 05a4f5028a..9d1e8b20c2 100644 --- a/tensorflow/contrib/data/python/kernel_tests/BUILD +++ b/tensorflow/contrib/data/python/kernel_tests/BUILD @@ -343,6 +343,7 @@ py_test( "//tensorflow/python:dtypes", "//tensorflow/python:errors", "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/eager:context", "//third_party/py/numpy", ], ) diff --git a/tensorflow/contrib/data/python/kernel_tests/scan_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/scan_dataset_op_test.py index e0494736b7..1a97a84b2c 100644 --- a/tensorflow/contrib/data/python/kernel_tests/scan_dataset_op_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/scan_dataset_op_test.py @@ -24,9 +24,11 @@ import numpy as np from tensorflow.contrib.data.python.kernel_tests import dataset_serialization_test_base from tensorflow.contrib.data.python.ops import scan_ops from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.platform import test @@ -57,19 +59,24 @@ class ScanDatasetTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(next_element) + @test_util.run_in_graph_and_eager_modes() def testFibonacci(self): iterator = dataset_ops.Dataset.from_tensors(1).repeat(None).apply( scan_ops.scan([0, 1], lambda a, _: ([a[1], a[0] + a[1]], a[1])) ).make_one_shot_iterator() - next_element = iterator.get_next() - with self.test_session() as sess: - self.assertEqual(1, sess.run(next_element)) - self.assertEqual(1, sess.run(next_element)) - self.assertEqual(2, sess.run(next_element)) - self.assertEqual(3, sess.run(next_element)) - self.assertEqual(5, sess.run(next_element)) - self.assertEqual(8, sess.run(next_element)) + if context.executing_eagerly(): + next_element = iterator.get_next + else: + get_next = iterator.get_next() + next_element = lambda: get_next + + self.assertEqual(1, self.evaluate(next_element())) + self.assertEqual(1, self.evaluate(next_element())) + self.assertEqual(2, self.evaluate(next_element())) + self.assertEqual(3, self.evaluate(next_element())) + self.assertEqual(5, self.evaluate(next_element())) + self.assertEqual(8, self.evaluate(next_element())) def testChangingStateShape(self): # Test the fixed-point shape invariant calculations: start with diff --git a/tensorflow/contrib/data/python/ops/scan_ops.py b/tensorflow/contrib/data/python/ops/scan_ops.py index 1c88366273..711a538697 100644 --- a/tensorflow/contrib/data/python/ops/scan_ops.py +++ b/tensorflow/contrib/data/python/ops/scan_ops.py @@ -144,6 +144,7 @@ class _ScanDataset(dataset_ops.Dataset): weakened_state_shapes) self._scan_func = tf_scan_func + self._scan_func.add_to_graph(ops.get_default_graph()) def _as_variant_tensor(self): input_t = self._input_dataset._as_variant_tensor() # pylint: disable=protected-access -- GitLab From 8d3a41f459b776856ff668bb076d4bc449927e09 Mon Sep 17 00:00:00 2001 From: Yunxing Dai Date: Fri, 20 Apr 2018 16:30:02 -0700 Subject: [PATCH 772/791] [XLA] Remove constant cast in literal util. It's not portable to modify an underlying char array of a c++ string object: (https://stackoverflow.com/questions/5729203/modifying-underlying-char-array-of-a-c-string-object) RELNOTES: n/a PiperOrigin-RevId: 193740595 --- tensorflow/compiler/xla/literal_util.cc | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tensorflow/compiler/xla/literal_util.cc b/tensorflow/compiler/xla/literal_util.cc index c315b4ff30..bb6dd4f909 100644 --- a/tensorflow/compiler/xla/literal_util.cc +++ b/tensorflow/compiler/xla/literal_util.cc @@ -44,8 +44,16 @@ namespace { constexpr bool kLittleEndian = __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__; -// Converts between little and big endian, assuming elements in the array are 16 -// bits long. +// Converts between little and big endian. +// +// Precondition: size % 2 == 0 (elements in the array are 16 bits long) +void ConvertEndianShort(string* bytes) { + CHECK_EQ(bytes->size() / 2, 0); + for (int64 i = 0; i < bytes->size(); i += 2) { + std::swap((*bytes)[i], (*bytes)[i + 1]); + } +} + void ConvertEndianShort(char* bytes, int64 size) { CHECK_EQ(size / 2, 0); for (int64 i = 0; i < size; i += 2) { @@ -1930,16 +1938,14 @@ void Literal::Piece::WriteToProto(LiteralProto* proto) const { *proto->mutable_f16s() = string( reinterpret_cast(data().data()), size_bytes()); if (!kLittleEndian) { - ConvertEndianShort(const_cast(proto->mutable_f16s()->data()), - proto->f16s().size()); + ConvertEndianShort(proto->mutable_f16s()); } break; case BF16: *proto->mutable_bf16s() = string( reinterpret_cast(data().data()), size_bytes()); if (!kLittleEndian) { - ConvertEndianShort(const_cast(proto->mutable_bf16s()->data()), - proto->bf16s().size()); + ConvertEndianShort(proto->mutable_bf16s()); } break; case F32: -- GitLab From 82679654af098df1de27bcdcf6fc6942ccf4f236 Mon Sep 17 00:00:00 2001 From: ADiegoCAlonso Date: Sat, 21 Apr 2018 11:43:51 +0200 Subject: [PATCH 773/791] Add __init__py --- tensorflow/examples/tutorials/estimators/__init__.py | 0 tensorflow/examples/tutorials/input_fn/__init__.py | 0 tensorflow/examples/tutorials/layers/__init__.py | 0 tensorflow/examples/tutorials/monitors/__init__.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tensorflow/examples/tutorials/estimators/__init__.py create mode 100644 tensorflow/examples/tutorials/input_fn/__init__.py create mode 100644 tensorflow/examples/tutorials/layers/__init__.py create mode 100644 tensorflow/examples/tutorials/monitors/__init__.py diff --git a/tensorflow/examples/tutorials/estimators/__init__.py b/tensorflow/examples/tutorials/estimators/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tensorflow/examples/tutorials/input_fn/__init__.py b/tensorflow/examples/tutorials/input_fn/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tensorflow/examples/tutorials/layers/__init__.py b/tensorflow/examples/tutorials/layers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tensorflow/examples/tutorials/monitors/__init__.py b/tensorflow/examples/tutorials/monitors/__init__.py new file mode 100644 index 0000000000..e69de29bb2 -- GitLab From aed22c552905d74de04c98b34aabedd12926790a Mon Sep 17 00:00:00 2001 From: ADiegoCAlonso Date: Sat, 21 Apr 2018 11:56:10 +0200 Subject: [PATCH 774/791] Specify float32 as float type instead of float64 --- tensorflow/examples/tutorials/monitors/iris_monitors.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/examples/tutorials/monitors/iris_monitors.py b/tensorflow/examples/tutorials/monitors/iris_monitors.py index 850d105f7b..a2b7fe6023 100644 --- a/tensorflow/examples/tutorials/monitors/iris_monitors.py +++ b/tensorflow/examples/tutorials/monitors/iris_monitors.py @@ -32,9 +32,9 @@ IRIS_TEST = os.path.join(os.path.dirname(__file__), "iris_test.csv") def main(unused_argv): # Load datasets. training_set = tf.contrib.learn.datasets.base.load_csv_with_header( - filename=IRIS_TRAINING, target_dtype=np.int, features_dtype=np.float) + filename=IRIS_TRAINING, target_dtype=np.int, features_dtype=np.float32) test_set = tf.contrib.learn.datasets.base.load_csv_with_header( - filename=IRIS_TEST, target_dtype=np.int, features_dtype=np.float) + filename=IRIS_TEST, target_dtype=np.int, features_dtype=np.float32) validation_metrics = { "accuracy": @@ -83,7 +83,7 @@ def main(unused_argv): # Classify two new flower samples. new_samples = np.array( - [[6.4, 3.2, 4.5, 1.5], [5.8, 3.1, 5.0, 1.7]], dtype=float) + [[6.4, 3.2, 4.5, 1.5], [5.8, 3.1, 5.0, 1.7]], dtype=np.float32) y = list(classifier.predict(new_samples)) print("Predictions: {}".format(str(y))) -- GitLab From cea18851e2d81ee97ebf8e9f6aeddd55a34e3227 Mon Sep 17 00:00:00 2001 From: foo0x29a Date: Sat, 21 Apr 2018 13:30:52 -0300 Subject: [PATCH 775/791] fix typo --- .../core/grappler/optimizers/custom_graph_optimizer_registry.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.h b/tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.h index 796da91373..3148a5f809 100644 --- a/tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.h +++ b/tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.h @@ -33,7 +33,7 @@ class CustomGraphOptimizerRegistry { static std::vector GetRegisteredOptimizers(); typedef std::function Creator; - // Regsiter graph optimizer which can be called during program initialization. + // Register graph optimizer which can be called during program initialization. // This class is not thread-safe. static void RegisterOptimizerOrDie(const Creator& optimizer_creator, const string& name); -- GitLab From fe4146d884c8805fceaa6d73d0bcc7fbf21df7cd Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 21 Apr 2018 18:42:03 +0000 Subject: [PATCH 776/791] Update .gitignore for cmake generated files After running cmake on Linux with: ``` tensorflow/tools/ci_build/ci_build.sh CMAKE tensorflow/tools/ci_build/builds/cmake.sh ``` the following file is left: ``` ubuntu@ubuntu:~/tensorflow$ git status On branch master Your branch is up-to-date with 'origin/master'. Untracked files: (use "git add ..." to include in what will be committed) api_init_files_list.txt nothing added to commit but untracked files present (use "git add" to track) ubuntu@ubuntu:~/tensorflow$ ``` This fix updates the .gitignore file so that cmake generated files is not added with git inadvertently. Signed-off-by: Yong Tang --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index be75938ec4..828bbe9bd3 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ Podfile.lock /tensorflow/contrib/lite/examples/ios/simple/data/*.txt /tensorflow/contrib/lite/examples/ios/simple/data/*.tflite xcuserdata/** +/api_init_files_list.txt # Android .gradle -- GitLab From 8f558d67450f3ec6aa0d96af9fad84042d6b79df Mon Sep 17 00:00:00 2001 From: AG Ramesh Date: Sat, 21 Apr 2018 15:25:37 -0700 Subject: [PATCH 777/791] Changed calls to the depreacted StringPiece::contains with str_util::StrContains --- tensorflow/core/graph/mkl_layout_pass.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc index 5368774f2d..72a13d4da7 100644 --- a/tensorflow/core/graph/mkl_layout_pass.cc +++ b/tensorflow/core/graph/mkl_layout_pass.cc @@ -547,14 +547,14 @@ class MklLayoutRewritePass : public GraphOptimizationPass { // If Op has been specifically assigned to a non-CPU device, then No. if (!n->assigned_device_name().empty() && - !StringPiece(n->assigned_device_name()).contains(kCPUDeviceSubStr)) { + !str_util::StrContains(n->assigned_device_name(),kCPUDeviceSubStr)) { result = false; reason = "Op has been assigned a runtime device that is not CPU."; } // If user has specifically assigned this op to a non-CPU device, then No. if (!n->def().device().empty() && - !StringPiece(n->def().device()).contains(kCPUDeviceSubStr)) { + !str_util::StrContains(n->def().device(),kCPUDeviceSubStr)) { result = false; reason = "User has assigned a device that is not CPU."; } @@ -2691,14 +2691,14 @@ class MklLayoutRewritePass : public GraphOptimizationPass { // If Op has been specifically assigned to a non-CPU device, then No. if (!n->assigned_device_name().empty() && - !StringPiece(n->assigned_device_name()).contains(kCPUDeviceSubStr)) { + !str_util::StrContains(n->assigned_device_name(),kCPUDeviceSubStr)) { result = false; reason = "Op has been assigned a runtime device that is not CPU."; } // If user has specifically assigned this op to a non-CPU device, then No. if (!n->def().device().empty() && - !StringPiece(n->def().device()).contains(kCPUDeviceSubStr)) { + !str_util::StrContains(n->def().device(),kCPUDeviceSubStr)) { result = false; reason = "User has assigned a device that is not CPU."; } -- GitLab From 5518db48074c3bd125089bccc3edec03c192bf56 Mon Sep 17 00:00:00 2001 From: Bryan Heden Date: Sat, 21 Apr 2018 19:45:42 -0500 Subject: [PATCH 778/791] update $ source spacing When viewing install_linux, the spacing was off for 'Next Steps' section. --- tensorflow/docs_src/install/install_linux.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md index 1a349f5412..02af21bcf2 100644 --- a/tensorflow/docs_src/install/install_linux.md +++ b/tensorflow/docs_src/install/install_linux.md @@ -231,7 +231,7 @@ Note that you must activate the Virtualenv environment each time you use TensorFlow. If the Virtualenv environment is not currently active, invoke one of the following commands: -
 $ source ~/tensorflow/bin/activate      # bash, sh, ksh, or zsh
+
$ source ~/tensorflow/bin/activate      # bash, sh, ksh, or zsh
 $ source ~/tensorflow/bin/activate.csh  # csh or tcsh

 
 When the Virtualenv environment is active, you may run
-- 
GitLab


From bfffd2041106dac5b7bb3efcbb311a20505ac61f Mon Sep 17 00:00:00 2001
From: Yong Tang 
Date: Sun, 22 Apr 2018 14:43:21 +0000
Subject: [PATCH 779/791] Update docs to add note and examples for
 tf.count_nonzero with string

Signed-off-by: Yong Tang 
---
 tensorflow/python/ops/math_ops.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index 31ce83905b..30ac001c25 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -1466,9 +1466,18 @@ def count_nonzero(input_tensor,
   tf.count_nonzero(x, [0, 1])  # 3
   ```
 
+  **NOTE** Strings are compared against zero-length empty string `""`. Any
+  string with a size greater than zero is already considered as nonzero.
+
+  For example:
+  ```python
+  x = tf.constant(["", "a", "  ", "b", ""])
+  tf.count_nonzero(x) # 3, with "a", "  ", and "b" as nonzero strings.
+  ```
+
   Args:
-    input_tensor: The tensor to reduce. Should be of numeric type, `string`,
-      or `bool`.
+    input_tensor: The tensor to reduce. Should be of numeric type, `bool`,
+      or `string`.
     axis: The dimensions to reduce. If `None` (the default),
       reduces all dimensions. Must be in the range
       `[-rank(input_tensor), rank(input_tensor))`.
-- 
GitLab


From 21bd19a8b8b0be8ac4d39b6bc32366ba908f5105 Mon Sep 17 00:00:00 2001
From: Yong Tang 
Date: Sun, 22 Apr 2018 17:49:13 +0000
Subject: [PATCH 780/791] Change from squeeze_dims to axis when calling
 tf.squeeze

The `squeeze_dims` in `tf.squeeze` has been deprecated in favor
of `axis` while many places still use `squeeze_dims`. That
generates lots of warnings.

This fix switches from `squeeze_dims` to `axis` to remove those warnings.

Signed-off-by: Yong Tang 
---
 tensorflow/python/ops/array_grad.py | 2 +-
 tensorflow/python/ops/array_ops.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/array_grad.py b/tensorflow/python/ops/array_grad.py
index 57d2657838..3678bd4c1f 100644
--- a/tensorflow/python/ops/array_grad.py
+++ b/tensorflow/python/ops/array_grad.py
@@ -196,7 +196,7 @@ def _ConcatGradHelper(op, grad, start_value_index, end_value_index, dim_index):
             array_ops.where(
                 math_ops.logical_and(grad.indices >= start,
                                      grad.indices < end)),
-            squeeze_dims=[1])
+            axis=[1])
         new_indices = array_ops.gather(grad.indices, indices_to_select) - start
         new_values = array_ops.gather(grad.values, indices_to_select)
         out_grads.append(ops.IndexedSlices(new_values, new_indices, size))
diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 23202ae28e..bbffff0483 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -1230,7 +1230,7 @@ def boolean_mask(tensor, mask, name="boolean_mask", axis=None):
 
   def _apply_mask_1d(reshaped_tensor, mask, axis=None):
     """Mask tensor along dimension 0 with a 1-D mask."""
-    indices = squeeze(where(mask), squeeze_dims=[1])
+    indices = squeeze(where(mask), axis=[1])
     return gather(reshaped_tensor, indices, axis=axis)
 
   with ops.name_scope(name, values=[tensor, mask]):
-- 
GitLab


From 100b6000d4d04a344a1516578f724e46cdede5e1 Mon Sep 17 00:00:00 2001
From: Yong Tang 
Date: Sun, 22 Apr 2018 17:52:31 +0000
Subject: [PATCH 781/791] Fix warning in image related ops.

Signed-off-by: Yong Tang 
---
 tensorflow/python/ops/image_ops_impl.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index 601010bce9..bd5b2ae83b 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -652,7 +652,7 @@ def pad_to_bounding_box(image, offset_height, offset_width, target_height,
     padded.set_shape(padded_shape)
 
     if not is_batch:
-      padded = array_ops.squeeze(padded, squeeze_dims=[0])
+      padded = array_ops.squeeze(padded, axis=[0])
 
     return padded
 
@@ -732,7 +732,7 @@ def crop_to_bounding_box(image, offset_height, offset_width, target_height,
     cropped.set_shape(cropped_shape)
 
     if not is_batch:
-      cropped = array_ops.squeeze(cropped, squeeze_dims=[0])
+      cropped = array_ops.squeeze(cropped, axis=[0])
 
     return cropped
 
@@ -849,7 +849,7 @@ def resize_image_with_crop_or_pad(image, target_height, target_width):
     resized = control_flow_ops.with_dependencies(assert_ops, resized)
 
     if not is_batch:
-      resized = array_ops.squeeze(resized, squeeze_dims=[0])
+      resized = array_ops.squeeze(resized, axis=[0])
 
     return resized
 
@@ -942,7 +942,7 @@ def resize_images(images,
            for x in [new_width_const, width, new_height_const, height]) and (
                width == new_width_const and height == new_height_const):
       if not is_batch:
-        images = array_ops.squeeze(images, squeeze_dims=[0])
+        images = array_ops.squeeze(images, axis=[0])
       return images
 
     if method == ResizeMethod.BILINEAR:
@@ -965,7 +965,7 @@ def resize_images(images,
     images.set_shape([None, new_height_const, new_width_const, None])
 
     if not is_batch:
-      images = array_ops.squeeze(images, squeeze_dims=[0])
+      images = array_ops.squeeze(images, axis=[0])
     return images
 
 
-- 
GitLab


From 8cdc752227af998da946decc9365d63bcaa7f184 Mon Sep 17 00:00:00 2001
From: Yong Tang 
Date: Sun, 22 Apr 2018 17:53:10 +0000
Subject: [PATCH 782/791] Fix warning in tf.nn ops where squeeze_dims was used
 with tf.squeeze

Signed-off-by: Yong Tang 
---
 tensorflow/python/ops/nn_impl.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py
index d0d5ed07ce..576627e78e 100644
--- a/tensorflow/python/ops/nn_impl.py
+++ b/tensorflow/python/ops/nn_impl.py
@@ -765,9 +765,9 @@ def weighted_moments(x, axes, frequency_weights, name=None, keep_dims=False):
     weighted_variance = math_ops.multiply(weighted_distsq, divisor)
 
     if not keep_dims:
-      weighted_mean = array_ops.squeeze(weighted_mean, squeeze_dims=axes)
+      weighted_mean = array_ops.squeeze(weighted_mean, axis=axes)
       weighted_variance = array_ops.squeeze(
-          weighted_variance, squeeze_dims=axes)
+          weighted_variance, axis=axes)
 
     if needs_cast:
       weighted_mean = math_ops.cast(weighted_mean, dtypes.float16)
-- 
GitLab


From 12fd64f72f59ff5ba114903d4b851f855aaf2458 Mon Sep 17 00:00:00 2001
From: Yong Tang 
Date: Sun, 22 Apr 2018 17:53:58 +0000
Subject: [PATCH 783/791] Fix warnings in reduce_join_op_test.py

Signed-off-by: Yong Tang 
---
 tensorflow/python/kernel_tests/reduce_join_op_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/reduce_join_op_test.py b/tensorflow/python/kernel_tests/reduce_join_op_test.py
index 7f3049b9f8..fb9e5cc2a3 100644
--- a/tensorflow/python/kernel_tests/reduce_join_op_test.py
+++ b/tensorflow/python/kernel_tests/reduce_join_op_test.py
@@ -160,7 +160,7 @@ class ReduceJoinTest(UnicodeTestCase):
             separator=separator)
       if not reduction_indices:
         truth = constant_op.constant(truth)
-      truth_squeezed = array_ops.squeeze(truth, squeeze_dims=reduction_indices)
+      truth_squeezed = array_ops.squeeze(truth, axis=reduction_indices)
       output_array = output.eval()
       output_keep_dims_array = output_keep_dims.eval()
       truth_array = truth.eval()
-- 
GitLab


From 9aa142284166c51dfc202b551b4592f9c9ed54e7 Mon Sep 17 00:00:00 2001
From: Yong Tang 
Date: Sun, 22 Apr 2018 17:54:26 +0000
Subject: [PATCH 784/791] Fix tf.contrib.timeseries warnings related to
 squeeze_dims

Signed-off-by: Yong Tang 
---
 .../timeseries/python/timeseries/state_management_test.py   | 2 +-
 .../python/timeseries/state_space_models/kalman_filter.py   | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_management_test.py b/tensorflow/contrib/timeseries/python/timeseries/state_management_test.py
index d5dce30fda..5f7e3da2db 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/state_management_test.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/state_management_test.py
@@ -78,7 +78,7 @@ class StubTimeSeriesModel(model.TimeSeriesModel):
     batch_end_values = array_ops.squeeze(
         array_ops.slice(values, [0, array_ops.shape(times)[1] - 1, 0],
                         [-1, 1, -1]),
-        squeeze_dims=[1, 2])
+        axis=[1, 2])
     # A pretty odd but easy to think about loss: L1 loss on the batch end
     # values.
     loss = math_ops.reduce_sum(
diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/kalman_filter.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/kalman_filter.py
index 1fcd3e391b..a614386121 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/kalman_filter.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/kalman_filter.py
@@ -170,7 +170,7 @@ class KalmanFilter(object):
         math_ops.matmul(
             transition_matrices,
             prior_state[..., None]),
-        squeeze_dims=[-1])
+        axis=[-1])
     return advanced_state
 
   def predict_state_var(
@@ -254,7 +254,7 @@ class KalmanFilter(object):
             kalman_gain_transposed,
             array_ops.expand_dims(residual, -1),
             adjoint_a=True),
-        squeeze_dims=[-1])
+        axis=[-1])
     gain_obs = math_ops.matmul(
         kalman_gain_transposed, observation_model, adjoint_a=True)
     identity_extradim = linalg_ops.eye(
@@ -332,7 +332,7 @@ class KalmanFilter(object):
             array_ops.expand_dims(state_mean, 1),
             observation_model,
             adjoint_b=True),
-        squeeze_dims=[1])
+        axis=[1])
     observed_var = math_ops.matmul(
         math_ops.matmul(observation_model, state_var),
         observation_model,
-- 
GitLab


From 8257b9096062a87555d72f7c15e16b1d8e748d70 Mon Sep 17 00:00:00 2001
From: Yong Tang 
Date: Sun, 22 Apr 2018 17:55:06 +0000
Subject: [PATCH 785/791] Fix warnings in tf.contrib.tensor_forest

Signed-off-by: Yong Tang 
---
 tensorflow/contrib/tensor_forest/client/eval_metrics.py       | 4 ++--
 .../tensor_forest/hybrid/python/layers/fully_connected.py     | 2 +-
 tensorflow/contrib/tensor_forest/python/tensor_forest.py      | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/tensor_forest/client/eval_metrics.py b/tensorflow/contrib/tensor_forest/client/eval_metrics.py
index 90033015eb..e893e1d1c8 100644
--- a/tensorflow/contrib/tensor_forest/client/eval_metrics.py
+++ b/tensorflow/contrib/tensor_forest/client/eval_metrics.py
@@ -37,7 +37,7 @@ def _top_k_generator(k):
   def _top_k(probabilities, targets):
     targets = math_ops.to_int32(targets)
     if targets.get_shape().ndims > 1:
-      targets = array_ops.squeeze(targets, squeeze_dims=[1])
+      targets = array_ops.squeeze(targets, axis=[1])
     return metric_ops.streaming_mean(nn.in_top_k(probabilities, targets, k))
   return _top_k
 
@@ -57,7 +57,7 @@ def _r2(probabilities, targets, weights=None):
 
 
 def _squeeze_and_onehot(targets, depth):
-  targets = array_ops.squeeze(targets, squeeze_dims=[1])
+  targets = array_ops.squeeze(targets, axis=[1])
   return array_ops.one_hot(math_ops.to_int32(targets), depth)
 
 
diff --git a/tensorflow/contrib/tensor_forest/hybrid/python/layers/fully_connected.py b/tensorflow/contrib/tensor_forest/hybrid/python/layers/fully_connected.py
index ff3ab21eaa..745a5b1caf 100644
--- a/tensorflow/contrib/tensor_forest/hybrid/python/layers/fully_connected.py
+++ b/tensorflow/contrib/tensor_forest/hybrid/python/layers/fully_connected.py
@@ -55,7 +55,7 @@ class ManyToOneLayer(hybrid_layer.HybridLayer):
 
       # There is always one activation per instance by definition, so squeeze
       # away the extra dimension.
-      return array_ops.squeeze(nn_activations, squeeze_dims=[1])
+      return array_ops.squeeze(nn_activations, axis=[1])
 
 
 class FlattenedFullyConnectedLayer(hybrid_layer.HybridLayer):
diff --git a/tensorflow/contrib/tensor_forest/python/tensor_forest.py b/tensorflow/contrib/tensor_forest/python/tensor_forest.py
index b9bcbb170b..7a35a70bbe 100644
--- a/tensorflow/contrib/tensor_forest/python/tensor_forest.py
+++ b/tensorflow/contrib/tensor_forest/python/tensor_forest.py
@@ -445,7 +445,7 @@ class RandomForestGraphs(object):
           mask = math_ops.less(
               r, array_ops.ones_like(r) * self.params.bagging_fraction)
           gather_indices = array_ops.squeeze(
-              array_ops.where(mask), squeeze_dims=[1])
+              array_ops.where(mask), axis=[1])
           # TODO(thomaswc): Calculate out-of-bag data and labels, and store
           # them for use in calculating statistics later.
           tree_data = array_ops.gather(processed_dense_features, gather_indices)
-- 
GitLab


From 685fec394235b409b58d7ef1c4a26655f9fedcfd Mon Sep 17 00:00:00 2001
From: Yong Tang 
Date: Sun, 22 Apr 2018 17:55:35 +0000
Subject: [PATCH 786/791] Fix squeeze_dims warnings in tf.contrib.learn

Signed-off-by: Yong Tang 
---
 tensorflow/contrib/learn/python/learn/estimators/head.py | 4 ++--
 tensorflow/contrib/learn/python/learn/ops/losses_ops.py  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/learn/python/learn/estimators/head.py b/tensorflow/contrib/learn/python/learn/estimators/head.py
index 2b4b6eff39..e28e6854a5 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/head.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/head.py
@@ -777,7 +777,7 @@ class _RegressionHead(_SingleHead):
     key = prediction_key.PredictionKey.SCORES
     with ops.name_scope(None, "predictions", (logits,)):
       if self.logits_dimension == 1:
-        logits = array_ops.squeeze(logits, squeeze_dims=(1,), name=key)
+        logits = array_ops.squeeze(logits, axis=(1,), name=key)
       return {key: self._link_fn(logits)}
 
   def _metrics(self, eval_loss, predictions, labels, weights):
@@ -974,7 +974,7 @@ def _softmax_cross_entropy_loss(labels, logits, weights=None):
     is_squeezed_labels = False
     # TODO(ptucker): This will break for dynamic shapes.
     if len(labels.get_shape()) == 2:
-      labels = array_ops.squeeze(labels, squeeze_dims=(1,))
+      labels = array_ops.squeeze(labels, axis=(1,))
       is_squeezed_labels = True
 
     loss = nn.sparse_softmax_cross_entropy_with_logits(
diff --git a/tensorflow/contrib/learn/python/learn/ops/losses_ops.py b/tensorflow/contrib/learn/python/learn/ops/losses_ops.py
index 92976d1539..9f2cadb017 100644
--- a/tensorflow/contrib/learn/python/learn/ops/losses_ops.py
+++ b/tensorflow/contrib/learn/python/learn/ops/losses_ops.py
@@ -40,7 +40,7 @@ def mean_squared_error_regressor(tensor_in, labels, weights, biases, name=None):
                       [tensor_in, labels]):
     predictions = nn.xw_plus_b(tensor_in, weights, biases)
     if len(labels.get_shape()) == 1 and len(predictions.get_shape()) == 2:
-      predictions = array_ops_.squeeze(predictions, squeeze_dims=[1])
+      predictions = array_ops_.squeeze(predictions, axis=[1])
     return predictions, losses.mean_squared_error(labels, predictions)
 
 
-- 
GitLab


From 5c19fc7810f13712127b8527b040f8f656474fe5 Mon Sep 17 00:00:00 2001
From: Yong Tang 
Date: Sun, 22 Apr 2018 17:56:09 +0000
Subject: [PATCH 787/791] Fix tf.contrib.layers warnings where squeeze_dims
 were used with tf.squeeze

Signed-off-by: Yong Tang 
---
 tensorflow/contrib/layers/python/layers/target_column.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/layers/python/layers/target_column.py b/tensorflow/contrib/layers/python/layers/target_column.py
index 3e639a180e..69bb6be814 100644
--- a/tensorflow/contrib/layers/python/layers/target_column.py
+++ b/tensorflow/contrib/layers/python/layers/target_column.py
@@ -270,7 +270,7 @@ class _RegressionTargetColumn(_TargetColumn):
 
   def logits_to_predictions(self, logits, proba=False):
     if self.num_label_columns == 1:
-      return array_ops.squeeze(logits, squeeze_dims=[1])
+      return array_ops.squeeze(logits, axis=[1])
     return logits
 
   def get_eval_ops(self, features, logits, labels, metrics=None):
@@ -418,7 +418,7 @@ def _softmax_cross_entropy_loss(logits, target):
                      "Instead got %s." % target.dtype)
   # sparse_softmax_cross_entropy_with_logits requires [batch_size] target.
   if len(target.get_shape()) == 2:
-    target = array_ops.squeeze(target, squeeze_dims=[1])
+    target = array_ops.squeeze(target, axis=[1])
   loss_vec = nn.sparse_softmax_cross_entropy_with_logits(
       labels=target, logits=logits)
   return loss_vec
-- 
GitLab


From 50a8df144d24ce60866bff96645f04e84a31f8b4 Mon Sep 17 00:00:00 2001
From: Yong Tang 
Date: Sun, 22 Apr 2018 17:57:06 +0000
Subject: [PATCH 788/791] Fix warnings in tf.contrib.factorization

Signed-off-by: Yong Tang 
---
 tensorflow/contrib/factorization/python/ops/gmm_ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/factorization/python/ops/gmm_ops.py b/tensorflow/contrib/factorization/python/ops/gmm_ops.py
index ccdd679d6a..e076631bc1 100644
--- a/tensorflow/contrib/factorization/python/ops/gmm_ops.py
+++ b/tensorflow/contrib/factorization/python/ops/gmm_ops.py
@@ -397,7 +397,7 @@ class GmmAlgorithm(object):
     # Compute the effective number of data points assigned to component k.
     with ops.control_dependencies(self._w):
       points_in_k = array_ops.squeeze(
-          math_ops.add_n(self._points_in_k), squeeze_dims=[0])
+          math_ops.add_n(self._points_in_k), axis=[0])
       # Update alpha.
       if 'w' in self._params:
         final_points_in_k = points_in_k / num_batches
-- 
GitLab


From 82eacbd4ac29db754b86a0be0cdfcc65b467c6af Mon Sep 17 00:00:00 2001
From: Yong Tang 
Date: Sun, 22 Apr 2018 17:57:31 +0000
Subject: [PATCH 789/791] Fix warnings in tf.contrib.distributions with
 squeeze_dims

Signed-off-by: Yong Tang 
---
 .../python/ops/bijectors/cholesky_outer_product.py              | 2 +-
 tensorflow/contrib/distributions/python/ops/shape.py            | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/cholesky_outer_product.py b/tensorflow/contrib/distributions/python/ops/bijectors/cholesky_outer_product.py
index caae2adcfa..ecdb8967f4 100644
--- a/tensorflow/contrib/distributions/python/ops/bijectors/cholesky_outer_product.py
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/cholesky_outer_product.py
@@ -170,7 +170,7 @@ class CholeskyOuterProduct(bijector.Bijector):
     sum_weighted_log_diag = array_ops.squeeze(
         math_ops.matmul(math_ops.log(diag),
                         exponents[..., array_ops.newaxis]),
-        squeeze_dims=-1)
+        axis=-1)
     fldj = p_float * np.log(2.) + sum_weighted_log_diag
 
     return fldj
diff --git a/tensorflow/contrib/distributions/python/ops/shape.py b/tensorflow/contrib/distributions/python/ops/shape.py
index bac0b79d59..6a7f28713a 100644
--- a/tensorflow/contrib/distributions/python/ops/shape.py
+++ b/tensorflow/contrib/distributions/python/ops/shape.py
@@ -439,7 +439,7 @@ class _DistributionShape(object):
           if self._batch_ndims_is_0 and expand_batch_dim:
             squeeze_dims += [1]
           if squeeze_dims:
-            x = array_ops.squeeze(x, squeeze_dims=squeeze_dims)
+            x = array_ops.squeeze(x, axis=squeeze_dims)
             # x.shape: [prod(S)]+B+E
         _, batch_shape, event_shape = self.get_shape(x)
       else:
-- 
GitLab


From c1544d1c34dac9aa01ed2de84bc850f8d1bfe919 Mon Sep 17 00:00:00 2001
From: Yifei Feng 
Date: Sun, 22 Apr 2018 19:08:21 -0700
Subject: [PATCH 790/791] Update tuple for cuda version with auto as it was
 removed in #18434.

---
 tensorflow/core/kernels/conv_ops_gpu.h | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/core/kernels/conv_ops_gpu.h b/tensorflow/core/kernels/conv_ops_gpu.h
index 7f9cfec981..bbd5a53660 100644
--- a/tensorflow/core/kernels/conv_ops_gpu.h
+++ b/tensorflow/core/kernels/conv_ops_gpu.h
@@ -143,8 +143,7 @@ class ConvParameters {
   bool ShouldIncludeWinogradNonfusedAlgo(
       perftools::gputools::StreamExecutor* stream_exec) const {
     // Skip this check for cuDNN 7 and newer.
-    perftools::gputools::port::StatusOr> version =
-        stream_exec->AsDnn()->GetVersion();
+    auto version = stream_exec->AsDnn()->GetVersion();
     if (version.ok() && std::get<0>(version.ValueOrDie()) >= 7) {
       return true;
     }
-- 
GitLab


From e5cfbd0eceb4dca98b388b13acff499a5420f863 Mon Sep 17 00:00:00 2001
From: Yifei Feng 
Date: Sun, 22 Apr 2018 20:00:54 -0700
Subject: [PATCH 791/791] Fix more for cuda version check.

---
 tensorflow/core/kernels/conv_ops_gpu.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/conv_ops_gpu.h b/tensorflow/core/kernels/conv_ops_gpu.h
index bbd5a53660..e8da5298e6 100644
--- a/tensorflow/core/kernels/conv_ops_gpu.h
+++ b/tensorflow/core/kernels/conv_ops_gpu.h
@@ -144,7 +144,7 @@ class ConvParameters {
       perftools::gputools::StreamExecutor* stream_exec) const {
     // Skip this check for cuDNN 7 and newer.
     auto version = stream_exec->AsDnn()->GetVersion();
-    if (version.ok() && std::get<0>(version.ValueOrDie()) >= 7) {
+    if (version.ok() && version.ValueOrDie().major_version() >= 7) {
       return true;
     }
     return ShouldIncludeWinogradNonfusedAlgoPreCudnn7();
-- 
GitLab